def setUp(self): # create a test user self.user = User.objects.create_user(username='******', email='', password='******') # create a public experiment self.experiment1 = Experiment(title='Experiment 1', created_by=self.user, public=True) self.experiment1.save() # create a non-public experiment self.experiment2 = Experiment(title='Experiment 2', created_by=self.user, public=False) self.experiment2.save() # dataset1 belongs to experiment1 self.dataset1 = Dataset(experiment=self.experiment1) self.dataset1.save() # dataset2 belongs to experiment2 self.dataset2 = Dataset(experiment=self.experiment2) self.dataset2.save() # absolute path first filename = 'testfile.txt' self.dest1 = abspath( join(settings.FILE_STORE_PATH, '%s/%s/' % (self.experiment1.id, self.dataset1.id))) self.dest2 = abspath( join(settings.FILE_STORE_PATH, '%s/%s/' % (self.experiment2.id, self.dataset2.id))) if not exists(self.dest1): makedirs(self.dest1) if not exists(self.dest2): makedirs(self.dest2) testfile1 = abspath(join(self.dest1, filename)) f = open(testfile1, 'w') f.write("Hello World!\n") f.close() testfile2 = abspath(join(self.dest2, filename)) f = open(testfile2, 'w') f.write("Hello World!\n") f.close() self.dataset_file1 = Dataset_File(dataset=self.dataset1, filename=filename, protocol='tardis', url='tardis://%s' % filename) self.dataset_file1.save() self.dataset_file2 = Dataset_File(dataset=self.dataset2, filename=basename(filename), protocol='tardis', url='tardis://%s' % filename) self.dataset_file2.save()
def testRemoteFile(self): content = urandom(1024) with NamedTemporaryFile() as f: # Create new Datafile datafile = Dataset_File(dataset=self.dataset) datafile.filename = 'background_task_testfile' datafile.size = len(content) datafile.sha512sum = hashlib.sha512(content).hexdigest() datafile.url = 'file://' + path.abspath(f.name) datafile.save() def get_datafile(datafile): return Dataset_File.objects.get(id=datafile.id) # Check that it won't verify as it stands expect(get_datafile(datafile).verified).to_be(False) verify_files() expect(get_datafile(datafile).verified).to_be(False) expect(get_datafile(datafile).is_local()).to_be(False) # Fill in the content f.write(content) f.flush() # Check it now verifies verify_files() expect(get_datafile(datafile).verified).to_be(True) expect(get_datafile(datafile).is_local()).to_be(True)
def add_datafile_to_dataset(dataset, filepath, size): """ Adds datafile metadata to a dataset :param dataset: dataset who's directory to be written to :type dataset: :class:`tardis.tardis_portal.models.Dataset` :param filepath: The full os path to the file :type filepath: string :param size: The file size in bytes :type size: string :rtype: The new datafile object """ from tardis.tardis_portal.models import Dataset_File experiment_path = path.join(settings.FILE_STORE_PATH, str(dataset.experiment.id)) dataset_path = path.join(experiment_path, str(dataset.id)) urlpath = 'tardis:/' + filepath[len(dataset_path):] filename = urlpath.rpartition('/')[2] datafile = Dataset_File(dataset=dataset, filename=filename, url=urlpath, size=size, protocol='tardis') datafile.save() return datafile
def create_datafile(index): testfile = path.join(path.dirname(__file__), 'fixtures', 'jeol_sem_test%d.txt' % index) size, sha512sum = get_size_and_sha512sum(testfile) datafile = Dataset_File(dataset=dataset, filename=path.basename(testfile), size=size, sha512sum=sha512sum) datafile.save() base_url = 'file://' + path.abspath(path.dirname(testfile)) location = Location.load_location({ 'name': 'test-jeol', 'url': base_url, 'type': 'external', 'priority': 10, 'transfer_provider': 'local' }) replica = Replica(datafile=datafile, url='file://' + path.abspath(testfile), protocol='file', location=location) replica.verify() replica.save() return Dataset_File.objects.get(pk=datafile.pk)
def process_enclosure(self, dataset, enclosure): filename = getattr(enclosure, 'title', basename(enclosure.href)) datafile = Dataset_File(filename=filename, dataset=dataset) try: datafile.mimetype = enclosure.mime except AttributeError: pass try: datafile.size = enclosure.length except AttributeError: pass try: hash = enclosure.hash # Split on white space, then ':' to get tuples to feed into dict hashdict = dict([s.partition(':')[::2] for s in hash.split()]) # Set SHA-512 sum datafile.sha512sum = hashdict['sha-512'] except AttributeError: pass datafile.save() url = enclosure.href # This means we will allow the atom feed to feed us any enclosure # URL that matches a registered location. Maybe we should restrict # this to a specific location. location = Location.get_location_for_url(url) if not location: logger.error('Rejected ingestion for unknown location %s' % url) return replica = Replica(datafile=datafile, url=url, location=location) replica.protocol = enclosure.href.partition('://')[0] replica.save() self.make_local_copy(replica)
def _create_test_dataset(nosDatafiles): ds_ = Dataset(description='happy snaps of plumage') ds_.save() for i in range(0, nosDatafiles): df_ = Dataset_File(dataset=ds_, url='http://planet-python.org/' + str(_next_id())) df_.save() ds_.save() return ds_
def _make_dataset(self, exp, filenames): dataset = Dataset(experiment=exp) dataset.save() for filename in filenames: df = Dataset_File(dataset=dataset, size=41, protocol='file') df.filename = filename df.url = 'file://' + path.join(path.dirname(__file__), 'data', df.filename) df.save()
def _build(dataset, filename, url, protocol): from tardis.tardis_portal.models import \ Dataset_File, Replica, Location datafile = Dataset_File(dataset=dataset, filename=filename) datafile.save() replica = Replica(datafile=datafile, url=url, protocol=protocol, location=Location.get_default_location()) replica.save() return datafile
def _create_test_dataset(nosDatafiles): ds_ = Dataset(description='happy snaps of plumage') ds_.save() for i in range(0, nosDatafiles): df_ = Dataset_File(dataset=ds_, size='21', sha512sum='bogus') df_.save() rep_ = Replica(datafile=df_, url='http://planet-python.org/' + str(_next_id()), location=Location.get_default_location()) rep_.save() ds_.save() return ds_
def create_datafile(index): testfile = path.join(path.dirname(__file__), 'fixtures', 'jeol_sem_test%d.txt' % index) size, sha512sum = get_size_and_sha512sum(testfile) datafile = Dataset_File(dataset=dataset, filename=path.basename(testfile), url='file://'+path.abspath(testfile), protocol='file', size=size, sha512sum=sha512sum) datafile.verify() datafile.save() return datafile
def generate_datafile(path, dataset, content=None, size=-1, verify=True, verified=True, verify_checksums_req=False): '''Generates a datafile AND a replica to hold its contents''' from tardis.tardis_portal.models import Dataset_File, Replica, Location saved = settings.REQUIRE_DATAFILE_CHECKSUMS settings.REQUIRE_DATAFILE_CHECKSUMS = False try: datafile = Dataset_File() if content: datafile.size = str(len(content)) else: datafile.size = str(size) # Normally we use any old string for the datafile path, but some # tests require the path to be the same as what 'staging' would use if path == None: datafile.dataset_id = dataset.id datafile.save() path = "%s/%s/%s" % (dataset.get_first_experiment().id, dataset.id, datafile.id) filepath = os.path.normpath(settings.FILE_STORE_PATH + '/' + path) if content: try: os.makedirs(os.path.dirname(filepath)) os.remove(filepath) except: pass gen_file = open(filepath, 'wb+') gen_file.write(content) gen_file.close() datafile.mimetype = "application/unspecified" datafile.filename = os.path.basename(filepath) datafile.dataset_id = dataset.id datafile.save() settings.REQUIRE_DATAFILE_CHECKSUMS = verify_checksums_req location = _infer_location(path) replica = Replica(datafile=datafile, url=path, protocol='', location=location) if verify and content: if not replica.verify(): raise RuntimeError('verify failed!?!') replica.save() replica.verified = verified replica.save(update_fields=['verified']) # force no verification return (datafile, replica) finally: settings.REQUIRE_DATAFILE_CHECKSUMS = saved
def _make_data_file(dataset, filename, content): # TODO: # create datasetfile f = mktemp() print "Inside make data file ", f open(f, "w+b").write(content) df = Dataset_File() df.dataset = dataset df.filename = filename df.url = 'file://'+f df.protocol = "staging" df.size = len(content) df.verify(allowEmptyChecksums=True) df.save() print "Df ---", df
def create_staging_datafile(filepath, username, dataset_id): from tardis.tardis_portal.models import Dataset_File, Dataset, Replica, \ Location dataset = Dataset.objects.get(id=dataset_id) url, size = get_staging_url_and_size(username, filepath) datafile = Dataset_File(dataset=dataset, filename=path.basename(filepath), size=size) replica = Replica(datafile=datafile, protocol='staging', url=url, location=Location.get_location('staging')) replica.verify(allowEmptyChecksums=True) datafile.save() replica.datafile = datafile replica.save()
def test_hrmc_filter(self): """ Make an experiment, lood up grexp file and check dataset schema missing, then loadup grfinal and check dataset schema created """ user = _create_test_user() license = _create_license() exp = _create_test_experiment(user, license) ds = Dataset(description='happy snaps of plumage') ds.save() _create_test_dataset(ds, exp.id, { "output.dat": 'hello', "grexp.dat": '2 5\n6 15\n' }) ds.experiments.add(exp) ds.save() sch = Schema(namespace=self.HRMCSCHEMA, name="hrmc_views", type=Schema.DATASET) sch.save() param = ParameterName(schema=sch, name="plot", full_name="scatterplot", units="image", data_type=ParameterName.FILENAME) param.save() param_sets = get_param_sets(ds) self.assertEquals(list(param_sets), []) _create_test_dataset(ds, exp.id, {'grfinal21.dat': "1 3\n5 14\n"}) df2 = Dataset_File(dataset=ds, url='path/grfinal21.dat') df2.save() h = hrmc.HRMCOutput('HRMC', self.HRMCSCHEMA) h(sender=Dataset_File, instance=df2) param_sets = get_param_sets(ds) self.assertEquals([x.schema.namespace for x in param_sets], [self.HRMCSCHEMA])
def _create_datafile(): user = User.objects.create_user('testuser', '*****@*****.**', 'pwd') user.save() UserProfile(user=user).save() Location.force_initialize() full_access = Experiment.PUBLIC_ACCESS_FULL experiment = Experiment.objects.create(title="IIIF Test", created_by=user, public_access=full_access) experiment.save() ObjectACL(content_object=experiment, pluginId='django_user', entityId=str(user.id), isOwner=True, canRead=True, canWrite=True, canDelete=True, aclOwnershipType=ObjectACL.OWNER_OWNED).save() dataset = Dataset() dataset.save() dataset.experiments.add(experiment) dataset.save() # Create new Datafile tempfile = TemporaryUploadedFile('iiif_stored_file', None, None, None) with Image(filename='magick:rose') as img: img.format = 'tiff' img.save(file=tempfile.file) tempfile.file.flush() datafile = Dataset_File(dataset=dataset, size=os.path.getsize(tempfile.file.name), filename='iiif_named_file') replica = Replica(datafile=datafile, url=write_uploaded_file_to_dataset(dataset, tempfile), location=Location.get_default_location()) replica.verify(allowEmptyChecksums=True) datafile.save() replica.datafile = datafile replica.save() return datafile
def _create_test_dataset(ds, exp_id, fnames): for fname, contents in fnames.items(): dest = os.path.abspath( os.path.join(settings.FILE_STORE_PATH, '%s/%s/' % (exp_id, ds.id))) if not os.path.exists(dest): os.makedirs(dest) testfile = os.path.abspath(os.path.join(dest, fname)) with open(testfile, "w+b") as f: f.write(contents) size, sha512sum = get_size_and_sha512sum(testfile) dataset_file = Dataset_File(dataset=ds, filename=fname, protocol='', size=size, sha512sum=sha512sum, url='%d/%d/%s' % (exp_id, ds.id, fname)) dataset_file.verify() dataset_file.save() return ds
def setUp(self): """ setting up essential objects, copied from tests above """ user = '******' pwd = 'secret' email = '' self.user = User.objects.create_user(user, email, pwd) self.userProfile = UserProfile(user=self.user).save() self.exp = Experiment(title='test exp1', institution_name='monash', created_by=self.user) self.exp.save() self.acl = ObjectACL( pluginId=django_user, entityId=str(self.user.id), content_object=self.exp, canRead=True, isOwner=True, aclOwnershipType=ObjectACL.OWNER_OWNED, ) self.acl.save() self.dataset = Dataset(description='dataset description...') self.dataset.save() self.dataset.experiments.add(self.exp) self.dataset.save() self.dataset_file = Dataset_File(dataset=self.dataset, size=42, filename="foo", md5sum="junk") self.dataset_file.save() self.testschema = Schema(namespace="http://test.com/test/schema", name="Test View", type=Schema.DATAFILE, hidden=True) self.testschema.save() self.dfps = DatafileParameterSet(dataset_file=self.dataset_file, schema=self.testschema) self.dfps.save()
def _build_datafile(self, testfile, filename, dataset, url, protocol='', checksum=None, size=None, mimetype=''): filesize, sha512sum = get_size_and_sha512sum(testfile) datafile = Dataset_File( dataset=dataset, filename=filename, mimetype=mimetype, size=str(size if size != None else filesize), sha512sum=(checksum if checksum else sha512sum)) datafile.save() if urlparse.urlparse(url).scheme == '': location = Location.get_location('local') else: location = Location.get_location_for_url(url) if not location: location = Location.load_location({ 'name': filename, 'url': urlparse.urljoin(url, '.'), 'type': 'external', 'priority': 10, 'transfer_provider': 'local' }) replica = Replica(datafile=datafile, protocol=protocol, url=url, location=location) replica.verify() replica.save() return Dataset_File.objects.get(pk=datafile.pk)
def fpupload(request, dataset_id): """ Uploads all files picked by filepicker to the dataset :param request: a HTTP Request instance :type request: :class:`django.http.HttpRequest` :param dataset_id: the dataset_id :type dataset_id: integer :returns: boolean true if successful :rtype: bool """ dataset = Dataset.objects.get(id=dataset_id) logger.debug('called fpupload') if request.method == 'POST': logger.debug('got POST') for key, val in request.POST.items(): splits = val.split(",") for url in splits: try: fp = FilepickerFile(url) except ValueError: pass else: picked_file = fp.get_file() filepath = write_uploaded_file_to_dataset( dataset, picked_file) datafile = Dataset_File(dataset=dataset, filename=picked_file.name, size=picked_file.size) replica = Replica(datafile=datafile, url=filepath, protocol='', location=Location.get_default_location()) replica.verify(allowEmptyChecksums=True) datafile.save() replica.datafile = datafile replica.save() return HttpResponse(json.dumps({"result": True}))
def testLocalFile(self): content = urandom(1024) cf = ContentFile(content, 'background_task_testfile') # Create new Datafile datafile = Dataset_File(dataset=self.dataset) datafile.filename = cf.name datafile.size = len(content) datafile.sha512sum = hashlib.sha512(content).hexdigest() datafile.url = write_uploaded_file_to_dataset(self.dataset, cf) datafile.save() def get_datafile(datafile): return Dataset_File.objects.get(id=datafile.id) # Check that it's not currently verified expect(get_datafile(datafile).verified).to_be(False) # Check it verifies verify_files() expect(get_datafile(datafile).verified).to_be(True)
def setUp(self): """ setting up essential objects, copied from tests above """ Location.force_initialize() self.location = Location.get_location('local') user = '******' pwd = 'secret' email = '' self.user = User.objects.create_user(user, email, pwd) self.userProfile = UserProfile(user=self.user).save() self.exp = Experiment(title='test exp1', institution_name='monash', created_by=self.user) self.exp.save() self.acl = ObjectACL( pluginId=django_user, entityId=str(self.user.id), content_object=self.exp, canRead=True, isOwner=True, aclOwnershipType=ObjectACL.OWNER_OWNED, ) self.acl.save() self.dataset = Dataset(description='dataset description...') self.dataset.save() self.dataset.experiments.add(self.exp) self.dataset.save() self.dataset_file = Dataset_File(dataset=self.dataset, size=42, filename="foo", md5sum="junk") self.dataset_file.save() self.replica = Replica(datafile=self.dataset_file, url="http://foo", location=self.location, verified=False) self.replica.save()
def testDatasetFile(self): # check registered text file for physical file meta information df = Dataset_File.objects.get(pk=self.dataset_file1.id) try: from magic import Magic self.assertEqual(df.mimetype, 'text/plain; charset=us-ascii') except: # XXX Test disabled because lib magic can't be loaded pass self.assertEqual(df.size, str(13)) self.assertEqual(df.md5sum, '8ddd8be4b179a529afa5f2ffae4b9858') # now check a JPG file filename = abspath(join(dirname(__file__), '../static/images/ands-logo-hi-res.jpg')) dataset = Dataset.objects.get(pk=self.dataset1.id) size, sha512sum = get_size_and_sha512sum(filename) pdf1 = Dataset_File(dataset=dataset, filename=basename(filename), size=str(size), sha512sum=sha512sum, url='file://%s' % filename, protocol='file') pdf1.verify() pdf1.save() try: from magic import Magic self.assertEqual(pdf1.mimetype, 'image/jpeg') except: # XXX Test disabled because lib magic can't be loaded pass self.assertEqual(pdf1.size, str(14232)) self.assertEqual(pdf1.md5sum, 'c450d5126ffe3d14643815204daf1bfb') # now check that we can override the physical file meta information pdf2 = Dataset_File(dataset=dataset, filename=basename(filename), url='file://%s' % filename, protocol='file', mimetype='application/vnd.openxmlformats-officedocument.presentationml.presentation', size=str(0), # Empty string always has the same hash sha512sum='cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e') pdf2.save() try: from magic import Magic self.assertEqual(pdf2.mimetype, 'application/vnd.openxmlformats-officedocument.presentationml.presentation') except: # XXX Test disabled because lib magic can't be loaded pass self.assertEqual(pdf2.size, str(0)) self.assertEqual(pdf2.md5sum, '') pdf2.mimetype = '' pdf2.save() try: from magic import Magic self.assertEqual(pdf2.mimetype, 'application/pdf') except: # XXX Test disabled because lib magic can't be loaded pass
def testDatasetFile(self): # check registered text file for physical file meta information df = Dataset_File.objects.get(pk=self.dataset_file1.id) try: from magic import Magic self.assertEqual(df.mimetype, 'text/plain; charset=us-ascii') except: # XXX Test disabled becuse lib magic can't be loaded pass self.assertEqual(df.size, str(13)) self.assertEqual(df.md5sum, '8ddd8be4b179a529afa5f2ffae4b9858') # now check a pdf file filename = join(abspath(dirname(__file__)), '../static/downloads/DatasetDepositionGuide.pdf') dataset = Dataset.objects.get(pk=self.dataset1.id) pdf1 = Dataset_File(dataset=dataset, filename=basename(filename), url='file://%s' % filename, protocol='file') pdf1.save() try: from magic import Magic self.assertEqual(pdf1.mimetype, 'application/pdf') except: # XXX Test disabled becuse lib magic can't be loaded pass self.assertEqual(pdf1.size, str(1008475)) self.assertEqual(pdf1.md5sum, '9192b3d3e0056412b1d21d3e33562eba') # now check that we can override the physical file meta information pdf2 = Dataset_File(dataset=dataset, filename=basename(filename), url='file://%s' % filename, protocol='file', mimetype='application/vnd.openxmlformats-officedocument.presentationml.presentation', size=str(0), md5sum='md5sum') pdf2.save() try: from magic import Magic self.assertEqual(pdf2.mimetype, 'application/vnd.openxmlformats-officedocument.presentationml.presentation') except: # XXX Test disabled becuse lib magic can't be loaded pass self.assertEqual(pdf2.size, str(0)) self.assertEqual(pdf2.md5sum, 'md5sum') pdf2.mimetype = '' pdf2.save() try: from magic import Magic self.assertEqual(pdf2.mimetype, 'application/pdf') except: # XXX Test disabled becuse lib magic can't be loaded pass
def test_datafile(self): from tardis.tardis_portal.models import Experiment, Dataset def _build(dataset, filename, url, protocol): from tardis.tardis_portal.models import \ Dataset_File, Replica, Location datafile = Dataset_File(dataset=dataset, filename=filename) datafile.save() replica = Replica(datafile=datafile, url=url, protocol=protocol, location=Location.get_default_location()) replica.save() return datafile exp = Experiment(title='test exp1', institution_name='monash', approved=True, created_by=self.user, public_access=Experiment.PUBLIC_ACCESS_NONE) exp.save() dataset = Dataset(description="dataset description...") dataset.save() dataset.experiments.add(exp) dataset.save() save1 = settings.REQUIRE_DATAFILE_SIZES save2 = settings.REQUIRE_DATAFILE_CHECKSUMS try: settings.REQUIRE_DATAFILE_SIZES = False settings.REQUIRE_DATAFILE_CHECKSUMS = False df_file = _build(dataset, 'file.txt', 'path/file.txt', '') self.assertEqual(df_file.filename, 'file.txt') self.assertEqual(df_file.get_preferred_replica().url, 'path/file.txt') self.assertEqual(df_file.get_preferred_replica().protocol, '') self.assertEqual(df_file.dataset, dataset) self.assertEqual(df_file.size, '') self.assertEqual(df_file.get_download_url(), '/test/download/datafile/1/') self.assertTrue(df_file.is_local()) self.assertEqual(df_file.get_absolute_filepath(), os.path.join(settings.FILE_STORE_PATH, 'path/file.txt')) df_file = _build(dataset, 'file1.txt', 'path/file1.txt', 'vbl') self.assertEqual(df_file.filename, 'file1.txt') self.assertEqual(df_file.get_preferred_replica().url, 'path/file1.txt') self.assertEqual(df_file.get_preferred_replica().protocol, 'vbl') self.assertEqual(df_file.dataset, dataset) self.assertEqual(df_file.size, '') self.assertEqual(df_file.get_download_url(), '/test/vbl/download/datafile/2/') self.assertFalse(df_file.is_local()) self.assertEqual(df_file.get_absolute_filepath(), os.path.join(settings.FILE_STORE_PATH, 'path/file1.txt')) df_file = _build(dataset, 'file1.txt', 'path/file1#txt', 'vbl') self.assertEqual(df_file.filename, 'file1.txt') self.assertEqual(df_file.get_preferred_replica().url, 'path/file1#txt') self.assertEqual(df_file.get_preferred_replica().protocol, 'vbl') self.assertEqual(df_file.dataset, dataset) self.assertEqual(df_file.size, '') self.assertEqual(df_file.get_download_url(), '/test/vbl/download/datafile/3/') self.assertFalse(df_file.is_local()) self.assertEqual(df_file.get_absolute_filepath(), os.path.join(settings.FILE_STORE_PATH, 'path/file1#txt')) df_file = _build(dataset, 'f.txt', 'http://localhost:8080/filestore/f.txt', '') self.assertEqual(df_file.filename, 'f.txt') self.assertEqual(df_file.get_preferred_replica().url, 'http://localhost:8080/filestore/f.txt') self.assertEqual(df_file.get_preferred_replica().protocol, '') self.assertEqual(df_file.dataset, dataset) self.assertEqual(df_file.size, '') self.assertEqual(df_file.get_download_url(), '/test/download/datafile/4/') self.assertFalse(df_file.is_local()) self.assertEqual(df_file.get_absolute_filepath(), '') # not local # Now check the 'REQUIRE' config params with self.assertRaises(Exception): settings.REQUIRE_DATAFILE_SIZES = True settings.REQUIRE_DATAFILE_CHECKSUMS = False Dataset_File(dataset=dataset, filename='foo.txt', md5sum='bad') with self.assertRaises(Exception): settings.REQUIRE_DATAFILE_SIZES = False settings.REQUIRE_DATAFILE_CHECKSUMS = True Dataset_File(dataset=dataset, filename='foo.txt', size='1') finally: settings.REQUIRE_DATAFILE_SIZES = save1 settings.REQUIRE_DATAFILE_CHECKSUMS = save2
def setUp(self): # create a test user self.user = User.objects.create_user(username='******', email='', password='******') # create a public experiment self.experiment1 = Experiment(title='Experiment 1', created_by=self.user, public_access=Experiment.PUBLIC_ACCESS_FULL) self.experiment1.save() # create a non-public experiment self.experiment2 = Experiment(title='Experiment 2', created_by=self.user, public_access=Experiment.PUBLIC_ACCESS_NONE) self.experiment2.save() # dataset1 belongs to experiment1 self.dataset1 = Dataset() self.dataset1.save() self.dataset1.experiments.add(self.experiment1) self.dataset1.save() # dataset2 belongs to experiment2 self.dataset2 = Dataset() self.dataset2.save() self.dataset2.experiments.add(self.experiment2) self.dataset2.save() # absolute path first filename1 = 'testfile.txt' filename2 = 'testfile.tiff' self.dest1 = abspath(join(settings.FILE_STORE_PATH, '%s/%s/' % (self.experiment1.id, self.dataset1.id))) self.dest2 = abspath(join(settings.FILE_STORE_PATH, '%s/%s/' % (self.experiment2.id, self.dataset2.id))) if not exists(self.dest1): makedirs(self.dest1) if not exists(self.dest2): makedirs(self.dest2) testfile1 = abspath(join(self.dest1, filename1)) f = open(testfile1, 'w') f.write("Hello World!\n") f.close() testfile2 = abspath(join(self.dest2, filename2)) if IMAGEMAGICK_AVAILABLE: with Image(filename='logo:') as img: img.format = 'tiff' img.save(filename=testfile2) else: # Apparently ImageMagick isn't installed... # Write a "fake" TIFF file f = open(testfile2, 'w') f.write("II\x2a\x00") f.close() size, sha512sum = get_size_and_sha512sum(testfile1) self.dataset_file1 = Dataset_File(dataset=self.dataset1, filename=filename1, protocol='', size=size, sha512sum=sha512sum, url='%d/%d/%s' % (self.experiment1.id, self.dataset1.id, filename1)) self.dataset_file1.verify() self.dataset_file1.save() size, sha512sum = get_size_and_sha512sum(testfile2) self.dataset_file2 = Dataset_File(dataset=self.dataset2, filename=basename(filename2), protocol='', size=size, sha512sum=sha512sum, url='%d/%d/%s' % (self.experiment2.id, self.dataset2.id, filename2)) self.dataset_file2.verify() self.dataset_file2.save()
def setUp(self): # create a test user self.user = User.objects.create_user(username='******', email='', password='******') # create a public experiment self.experiment1 = Experiment(title='Experiment 1', created_by=self.user, public_access=Experiment.PUBLIC_ACCESS_FULL) self.experiment1.save() # create a non-public experiment self.experiment2 = Experiment(title='Experiment 2', created_by=self.user, public_access=Experiment.PUBLIC_ACCESS_NONE) self.experiment2.save() # dataset1 belongs to experiment1 self.dataset1 = Dataset() self.dataset1.save() self.dataset1.experiments.add(self.experiment1) self.dataset1.save() # dataset2 belongs to experiment2 self.dataset2 = Dataset() self.dataset2.save() self.dataset2.experiments.add(self.experiment2) self.dataset2.save() # absolute path first filename1 = 'testfile.txt' filename2 = 'testfile.tiff' self.dest1 = abspath(join(settings.FILE_STORE_PATH, '%s/%s/' % (self.experiment1.id, self.dataset1.id))) self.dest2 = abspath(join(settings.FILE_STORE_PATH, '%s/%s/' % (self.experiment2.id, self.dataset2.id))) if not exists(self.dest1): makedirs(self.dest1) if not exists(self.dest2): makedirs(self.dest2) testfile1 = abspath(join(self.dest1, filename1)) f = open(testfile1, 'w') f.write("Hello World!\n") f.close() testfile2 = abspath(join(self.dest2, filename2)) _generate_test_image(testfile2) size, sha512sum = get_size_and_sha512sum(testfile1) self.dataset_file1 = Dataset_File(dataset=self.dataset1, filename=filename1, protocol='', size=size, sha512sum=sha512sum, url='%d/%d/%s' % (self.experiment1.id, self.dataset1.id, filename1)) self.dataset_file1.verify() self.dataset_file1.save() size, sha512sum = get_size_and_sha512sum(testfile2) self.dataset_file2 = Dataset_File(dataset=self.dataset2, filename=basename(filename2), protocol='', size=size, sha512sum=sha512sum, url='%d/%d/%s' % (self.experiment2.id, self.dataset2.id, filename2)) self.dataset_file2.verify() self.dataset_file2.save()
def addfiles(request): import os from os.path import basename from os import path from tardis.tardis_portal.models import Dataset_File import itertools from tardis.hpctardis.metadata import process_all_experiments from tardis.hpctardis.metadata import process_experimentX if 'username' in request.POST and \ 'password' in request.POST: authMethod = request.POST['authMethod'] user = auth_service.authenticate(authMethod=authMethod, request=request) if user: eid = request.POST['eid'] desc = request.POST['desc'] folder = request.POST['folder'] eid = int(eid) # TODO Use the try and except auth_key = settings.DEFAULT_AUTH try: exp = Experiment.objects.get(pk=eid) author = exp.created_by except Experiment.DoesNotExist: logger.exception( 'Experiment for eid %i in addfiles does not exist' % eid) return HttpResponse("Experiment Not Found") current_user = str(user) created_user = str(author) if current_user == created_user: staging = path.join(settings.STAGING_PATH, str(user), str(eid), str(folder)) filelist = [] ds_desc = {} # import pdb # pdb.set_trace() for root, dirs, files in os.walk(staging): for named in files: filelist.append(named) next = str(filelist) ds_desc[desc] = filelist #TODO If needed for security - Metadata from the folder can be extracted #to check the folder name for d, df in ds_desc.items(): dataset = models.Dataset(description=d, experiment=exp) dataset.save() for f in df: logger.debug('f = %s' % f) filepath = path.join(staging, f) size = path.getsize(filepath) filename = path.basename(filepath) datafile = Dataset_File(dataset=dataset, filename=filename, url=filepath, size=size, protocol='staging') datafile.save() next = next + ' File path :' + staging process_experimentX(exp) next = next + ' The Author is : ' + str( author) + ',' + ' The User is : ' + str(user) return HttpResponse(next) else: next = 'The author of the experiment can only add the files (From Tardis)' return HttpResponse(next) else: return HttpResponse("UnSuccessful")
def testDatasetFile(self): # check registered text file for physical file meta information df = Dataset_File.objects.get(pk=self.dataset_file1.id) try: from magic import Magic self.assertEqual(df.mimetype, 'text/plain; charset=us-ascii') except: # XXX Test disabled becuse lib magic can't be loaded pass self.assertEqual(df.size, str(13)) self.assertEqual(df.md5sum, '8ddd8be4b179a529afa5f2ffae4b9858') # now check a JPG file filename = join(abspath(dirname(__file__)), '../static/images/ands-logo-hi-res.jpg') dataset = Dataset.objects.get(pk=self.dataset1.id) pdf1 = Dataset_File(dataset=dataset, filename=basename(filename), url='file://%s' % filename, protocol='file') pdf1.save() try: from magic import Magic self.assertEqual(pdf1.mimetype, 'image/jpeg') except: # XXX Test disabled becuse lib magic can't be loaded pass self.assertEqual(pdf1.size, str(14232)) self.assertEqual(pdf1.md5sum, 'c450d5126ffe3d14643815204daf1bfb') # now check that we can override the physical file meta information pdf2 = Dataset_File( dataset=dataset, filename=basename(filename), url='file://%s' % filename, protocol='file', mimetype= 'application/vnd.openxmlformats-officedocument.presentationml.presentation', size=str(0), md5sum='md5sum') pdf2.save() try: from magic import Magic self.assertEqual( pdf2.mimetype, 'application/vnd.openxmlformats-officedocument.presentationml.presentation' ) except: # XXX Test disabled becuse lib magic can't be loaded pass self.assertEqual(pdf2.size, str(0)) self.assertEqual(pdf2.md5sum, 'md5sum') pdf2.mimetype = '' pdf2.save() try: from magic import Magic self.assertEqual(pdf2.mimetype, 'application/pdf') except: # XXX Test disabled becuse lib magic can't be loaded pass