def setUp(self): """ setting up essential objects, copied from tests above """ user = '******' pwd = 'secret' email = '' self.user = User.objects.create_user(user, email, pwd) self.userProfile = self.user.userprofile self.exp = Experiment(title='test exp1', institution_name='monash', created_by=self.user) self.exp.save() self.acl = ObjectACL( pluginId=django_user, entityId=str(self.user.id), content_object=self.exp, canRead=True, isOwner=True, aclOwnershipType=ObjectACL.OWNER_OWNED, ) self.acl.save() self.dataset = Dataset(description='dataset description...') self.dataset.save() self.dataset.experiments.add(self.exp) self.dataset.save() self.datafile = DataFile(dataset=self.dataset, size=42, filename="foo", md5sum="junk") self.datafile.save()
def upload(request, dataset_id): """ Uploads a datafile to the store and datafile metadata :param request: a HTTP Request instance :type request: :class:`django.http.HttpRequest` :param dataset_id: the dataset_id :type dataset_id: integer :returns: boolean true if successful :rtype: bool """ dataset = Dataset.objects.get(id=dataset_id) logger.debug('called upload') if request.method == 'POST': logger.debug('got POST') if request.FILES: uploaded_file_post = request.FILES['Filedata'] logger.debug('done upload') datafile = DataFile(dataset=dataset, filename=uploaded_file_post.name, size=uploaded_file_post.size) datafile.save(require_checksums=False) logger.debug('created file') datafile.file_object = uploaded_file_post logger.debug('saved datafile') return HttpResponse('True')
def register_squashfile(exp_id, epn, sq_dir, sq_filename, namespace): ''' example: register_squashfile(456, '1234A', '/srv/squashstore', '1234A.squashfs', 'http://synchrotron.org.au/mx/squashfsarchive/1') ''' dfs = DataFile.objects.filter(filename=sq_filename, dataset__experiments__id=exp_id) if len(dfs) == 1: return dfs[0] e = Experiment.objects.get(id=exp_id) ds = Dataset(description="01 SquashFS Archive") ds.save() ds.experiments.add(e) filepath = os.path.join(sq_dir, sq_filename) try: md5sum = open(filepath + '.md5sum', 'r').read().strip()[:32] except: print 'no md5sum file found' return None size = os.path.getsize(filepath) df = DataFile(md5sum=md5sum, filename=sq_filename, size=str(size), dataset=ds) df.save() schema = Schema.objects.filter(namespace=namespace)[0] ps = DatafileParameterSet(schema=schema, datafile=df) ps.save() ps.set_param('EPN', epn) sbox = StorageBox.objects.get(name='squashstore') dfo = DataFileObject(storage_box=sbox, datafile=df, uri=sq_filename) dfo.save() return df
def test_003_update_df_status_skip_offline(self, mock_stat, mock_df_online): """update_df_status should skip any files that have previously marked as offline.""" df2 = DataFile(dataset=self.dataset, filename="test_df2.jpg") df2.save() dfo2 = DataFileObject(datafile=df2, storage_box=self.sbox1, uri="stream/test_df2.jpg", verified=True) dfo2.save() # df2.verify() schema = Schema.objects.get(namespace=HSM_DATAFILE_NAMESPACE) ps2 = DatafileParameterSet(schema=schema, datafile=df2) ps2.save() param_name = ParameterName.objects.get(schema=schema, name="online") param2 = DatafileParameter(parameterset=ps2, name=param_name) param2.string_value = False param2.save() mock_stat.return_value = Stats(st_size=10000, st_blocks=100, st_mtime=datetime.now()) update_df_status() # assert that the df_online method wasn't called self.assertEquals(mock_df_online.call_count, 0)
def create_dfo(self, top, filename, dataset=None): ''' create dfo and datafile if necessary ''' df, df_data = self.find_datafile(top, filename) if df is None and df_data is None: return True # is a link if df: if dataset is not None and df.dataset.id != dataset.id: # olddataset_id = df.dataset.id df.dataset = dataset df.save() # oldds = Dataset.objects.get(id=olddataset_id) # if oldds.datafile_set.count() == 0: # oldds.delete() elif dataset is None and top.startswith('frames'): prefix = 'Raw data for' prefix_dataset(df.dataset, prefix) self.update_dataset(df.dataset, top) else: if dataset is None: dataset = self.get_or_create_dataset('lost and found') df = DataFile(dataset=dataset, filename=filename, directory=top, **df_data) df.save() dfo = DataFileObject(datafile=df, storage_box=self.s_box, uri=os.path.join(top, filename)) dfo.save() return True
def test_002_no_duplicate_params(self, mock_stat): """Datafile should only ever have one online param""" mock_stat.return_value = Stats(st_size=10000, st_blocks=100, st_mtime=datetime.now()) df1 = DataFile(dataset=self.dataset, filename="test_df.jpg") df1.save() dfo1 = DataFileObject(datafile=df1, storage_box=self.sbox1, uri="stream/test.jpg", verified=True) dfo1.save() df1.verify() param_name = ParameterName.objects.get( schema__namespace=HSM_DATAFILE_NAMESPACE, name="online") paramset = DatafileParameterSet.objects.get( schema__namespace=HSM_DATAFILE_NAMESPACE, datafile=df1) params = DatafileParameter.objects.filter(parameterset=paramset, name=param_name) self.assertEquals(params.count(), 1) self.assertRaises(OnlineParamExistsError, create_df_status(df1, HSM_DATAFILE_NAMESPACE, 500)) params = DatafileParameter.objects.filter(parameterset=paramset, name=param_name) self.assertEquals(params.count(), 1)
def test_001_create_df_status(self, mock_stat): """When a new datafile record is verified, metadata for it's online/offline status should be created and populated with the current online status""" mock_stat.return_value = Stats(st_size=10000, st_blocks=100, st_mtime=datetime.now()) temp = tempfile.NamedTemporaryFile(dir=tempfile.gettempdir()) temp_name = os.path.basename(temp.name) df2 = DataFile(dataset=self.dataset, filename=temp_name) df2.save() dfo2 = DataFileObject(datafile=df2, storage_box=self.sbox1, uri=temp_name) dfo2.save() df2.verify() param_name = ParameterName.objects.get( schema__namespace=HSM_DATAFILE_NAMESPACE, name="online") paramset = DatafileParameterSet.objects.get( schema__namespace=HSM_DATAFILE_NAMESPACE, datafile=df2) param = DatafileParameter.objects.get(parameterset=paramset, name=param_name) self.assertEquals(param.string_value, "True") temp.close()
def fpupload(request, dataset_id): """ Uploads all files picked by filepicker to the dataset :param request: a HTTP Request instance :type request: :class:`django.http.HttpRequest` :param dataset_id: the dataset_id :type dataset_id: integer :returns: boolean true if successful :rtype: bool """ dataset = Dataset.objects.get(id=dataset_id) logger.debug('called fpupload') if request.method == 'POST': logger.debug('got POST') for key, val in request.POST.items(): splits = val.split(",") for url in splits: try: fp = FilepickerFile(url) except ValueError: pass else: picked_file = fp.get_file() datafile = DataFile(dataset=dataset, filename=picked_file.name, size=picked_file.size) datafile.save() datafile.file_object = picked_file return HttpResponse(json.dumps({"result": True}))
def test_000_update_df_status_offline(self, mock_stat): """update_df_status should check the online status of preferred DFOs for all previously online datafiles and update online Parameter to 'False' for any offline files.""" df1 = DataFile(dataset=self.dataset, filename="test_df.jpg") df1.save() dfo1 = DataFileObject(datafile=df1, storage_box=self.sbox1, uri="stream/test.jpg", verified=True) dfo1.save() schema = Schema.objects.get(namespace=HSM_DATAFILE_NAMESPACE) ps = DatafileParameterSet(schema=schema, datafile=df1) ps.save() param_name = ParameterName.objects.get(schema=schema, name="online") param = DatafileParameter(parameterset=ps, name=param_name) param.string_value = True param.save() mock_stat.return_value = Stats(st_size=10000, st_blocks=0, st_mtime=datetime.now()) update_df_status() params = DatafileParameter.objects.filter( parameterset__schema=schema, parameterset__datafile=df1) self.assertEquals(params.count(), 1) self.assertEquals(params[0].string_value, "False")
def setUp(self): """Setup test fixtures if needed.""" self.user = User.objects.create_user("doctor", '', "pwd") self.exp = Experiment(title="Wonderful", institution_name="Monash University", created_by=self.user) self.exp.save() group = Group(name="Group1") group.save() facility = Facility(name="Test Facility", manager_group=group) facility.save() self.inst = Instrument(name="Test Instrument1", facility=facility) self.inst.save() self.dataset = Dataset(description="Dataset1", instrument=self.inst) self.dataset.save() storage_classes = getattr(settings, "HSM_STORAGE_CLASSES", DEFAULT_HSM_CLASSES) self.sbox1 = StorageBox(name="SBOX1", django_storage_class=storage_classes[0], status='online', max_size=256) self.sbox1.save() sbox1_attr = StorageBoxAttribute(storage_box=self.sbox1, key='type', value=StorageBox.DISK) sbox1_attr.save() sbox1_loc_opt = StorageBoxOption(storage_box=self.sbox1, key="location", value="/dummy/path") sbox1_loc_opt.save() self.sbox2 = StorageBox( name="SBOX2", django_storage_class="any.non.disk.StorageSystem", status='offline', max_size=256) self.sbox2.save() sbox2_attr = StorageBoxAttribute(storage_box=self.sbox2, key='type', value=StorageBox.TAPE) sbox2_attr.save() self.df1 = DataFile(dataset=self.dataset, filename="test_df.jpg") self.df1.save() self.dfo1 = DataFileObject(datafile=self.df1, storage_box=self.sbox1, uri="stream/test.jpg", verified=True) self.dfo1.save() self.df1.verify()
def setUp(self): raise SkipTest # temporarily disabling this feature, needs coding from tempfile import mkdtemp, mktemp from django.conf import settings import os # Disconnect post_save signal from django.db.models.signals import post_save from tardis.tardis_portal.models import Experiment, \ staging_hook, Dataset, DataFile, DataFileObject, StorageBox post_save.disconnect(staging_hook, sender=DataFileObject) from django.contrib.auth.models import User user = '******' pwd = 'secret' email = '' self.user = User.objects.create_user(user, email, pwd) try: os.makedirs(settings.GET_FULL_STAGING_PATH_TEST) except OSError: pass self.temp = mkdtemp(dir=settings.GET_FULL_STAGING_PATH_TEST) self.filepath = mktemp(dir=self.temp) content = 'test file' with open(self.filepath, "w+b") as f: f.write(content) # make datafile exp = Experiment(title='test exp1', institution_name='monash', created_by=self.user) exp.save() # make dataset dataset = Dataset(description="dataset description...") dataset.save() dataset.experiments.add(exp) dataset.save() # create datafile df = DataFile(dataset=dataset, size=len(content), filename=path.basename(self.file), md5sum='f20d9f2072bbeb6691c0f9c5099b01f3') df.save() # create replica base_url = settings.GET_FULL_STAGING_PATH_TEST s_box = StorageBox.get_default_storage(location=base_url) dfo = DataFileObject(datafile=df, uri=self.filepath, storage_box=s_box) dfo.save() self.dfo = dfo
def create_staging_datafile(filepath, username, dataset_id): init_filters() from tardis.tardis_portal.models import DataFile, Dataset dataset = Dataset.objects.get(id=dataset_id) url, size = get_staging_url_and_size(username, filepath) datafile = DataFile(dataset=dataset, filename=path.basename(filepath), size=size) datafile.save() datafile.file_object = open(filepath, 'r')
def _create_test_dataset(nosDatafiles): ds_ = Dataset(description='happy snaps of plumage') ds_.save() for i in range(0, nosDatafiles): df_ = DataFile(dataset=ds_, filename='file_%d' % i, size='21', sha512sum='bogus') df_.save() ds_.save() return ds_
def _build(dataset, filename, url): datafile_content = b"\n".join([b'some data %d' % i for i in range(1000)]) filesize = len(datafile_content) datafile = DataFile( dataset=dataset, filename=filename, size=filesize) datafile.save() dfo = DataFileObject( datafile=datafile, storage_box=datafile.get_default_storage_box(), uri=url) dfo.file_object = BytesIO(datafile_content) dfo.save() return datafile
def test_create_index(self): self.exp1 = Experiment(title='test exp1', institution_name='monash', description='Test Description', created_by=self.user) self.exp2 = Experiment(title='test exp2', institution_name='monash', description='Test Description', created_by=self.user) self.exp1.save() self.exp2.save() # get search instance search = ExperimentDocument.search() # query for title(exact matching) query = search.query("match", title='test exp1') result = query.execute(ignore_cache=True) self.assertEqual(result.hits[0].title, 'test exp1') # query for description query = search.query("match", description='Test Description') result = query.execute(ignore_cache=True) self.assertEqual(result.hits[0].description, 'Test Description') # query for created_time query = search.query("match", created_time=self.exp1.created_time) result = query.execute(ignore_cache=True) self.assertEqual(result.hits[0].created_time, self.exp1.created_time) # dataset # dataset1 belongs to experiment1 self.dataset1 = Dataset(description='test_dataset') self.dataset1.save() self.dataset1.experiments.add(self.exp1) self.dataset1.save() # dataset2 belongs to experiment2 self.dataset2 = Dataset(description='test_dataset2') self.dataset2.save() self.dataset2.experiments.add(self.exp2) self.dataset2.save() # search on dataset search = DatasetDocument.search() query = search.query("match", description='test_dataset') result = query.execute(ignore_cache=True) self.assertEqual(result.hits.total.value, 1) # search on datafile settings.REQUIRE_DATAFILE_SIZES = False settings.REQUIRE_DATAFILE_CHECKSUMS = False self.datafile = DataFile(dataset=self.dataset1, filename='test.txt') self.datafile.save() search = DataFileDocument.search() query = search.query("match", filename='test.txt') result = query.execute(ignore_cache=True) self.assertEqual(result.hits[0].filename, self.datafile.filename)
def _build(dataset, filename, url=None): datafile = DataFile(dataset=dataset, filename=filename) datafile.save() if url is None: datafile.file_object = StringIO('bla') return datafile from tardis.tardis_portal.models import \ DataFileObject dfo = DataFileObject( datafile=datafile, storage_box=datafile.get_default_storage_box(), uri=url) dfo.save() return datafile
def test_007_dfo_unverified(self): """df_online and dfo_online should raise Exception for an unverfied DataFile or DataFileObject, respectively""" df2 = DataFile(dataset=self.dataset, filename="test_df.jpg") df2.save() self.assertRaises(DataFileNotVerified, df_online, df2) dfo2 = DataFileObject(datafile=df2, storage_box=self.sbox1, uri="stream/test.jpg", verified=False) dfo2.save() self.assertRaises(DataFileObjectNotVerified, dfo_online, dfo2)
def test_wrong_size_verification(self): content = urandom(1024) cf = ContentFile(content, 'background_task_testfile') # Create new Datafile datafile = DataFile(dataset=self.dataset) datafile.filename = cf.name datafile.size = len(content) - 1 datafile.sha512sum = hashlib.sha512(content).hexdigest() datafile.save() datafile.file_object = cf # verify explicitly to catch Exceptions hidden by celery datafile.verify() self.assertFalse(datafile.file_objects.get().verified)
def test_003_offline_dataset(self, mock_stat): """A dataset should be offline if any datafiles are offline""" mock_stat.return_value = Stats(st_size=10000, st_blocks=0, st_mtime=datetime.now()) ds = Dataset(description="Dataset2", instrument=self.inst) ds.save() df2 = DataFile(dataset=ds, filename="test_file.jpg") df2.save() dfo2 = DataFileObject(datafile=df2, storage_box=self.sbox1, uri=df2.filename) dfo2.save() df2.verify() self.assertFalse(dataset_online(ds))
def _build(dataset, filename, url=None): datafile = DataFile(dataset=dataset, filename=filename) datafile.save() if url is None: datafile.file_object = StringIO(u'bla') return datafile dfo = DataFileObject( datafile=datafile, storage_box=datafile.get_default_storage_box(), uri=url) dfo.save() # Tests are run with CELERY_ALWAYS_EAGER = True, # so saving a DFO will trigger an immediate attempt # to verify the DFO which will trigger an attempt # to apply filters because we are overriding the # USE_FILTERS setting to True in this test: self.assertNotEqual(mock_send_task.call_count, 0) return datafile
def _create_datafile(): user = User.objects.create_user('testuser', '*****@*****.**', 'pwd') user.save() full_access = Experiment.PUBLIC_ACCESS_FULL experiment = Experiment.objects.create(title="IIIF Test", created_by=user, public_access=full_access) experiment.save() ObjectACL(content_object=experiment, pluginId='django_user', entityId=str(user.id), isOwner=True, canRead=True, canWrite=True, canDelete=True, aclOwnershipType=ObjectACL.OWNER_OWNED).save() dataset = Dataset() dataset.save() dataset.experiments.add(experiment) dataset.save() # Create new Datafile tempfile = TemporaryUploadedFile('iiif_stored_file', None, None, None) with Image(filename='magick:rose') as img: img.format = 'tiff' img.save(file=tempfile.file) tempfile.file.flush() datafile = DataFile(dataset=dataset, size=os.path.getsize(tempfile.file.name), filename='iiif_named_file', mimetype='image/tiff') compute_md5 = getattr(settings, 'COMPUTE_MD5', True) compute_sha512 = getattr(settings, 'COMPUTE_SHA512', True) checksums = compute_checksums(open(tempfile.file.name, 'r'), compute_md5=compute_md5, compute_sha512=compute_sha512) if compute_md5: datafile.md5sum = checksums['md5sum'] if compute_sha512: datafile.sha512sum = checksums['sha512sum'] datafile.save() datafile.file_object = tempfile return datafile
def _build_datafile(self, testfile, filename, dataset, checksum=None, size=None, mimetype=''): filesize, sha512sum = get_size_and_sha512sum(testfile) datafile = DataFile(dataset=dataset, filename=filename, mimetype=mimetype, size=size if size is not None else filesize, sha512sum=(checksum if checksum else sha512sum)) datafile.save() dfo = DataFileObject(datafile=datafile, storage_box=datafile.get_default_storage_box()) dfo.save() with open(testfile, 'r') as sourcefile: dfo.file_object = sourcefile return DataFile.objects.get(pk=datafile.pk)
def setUp(self): super(SimpleSearchTest, self).setUp() self.out = StringIO() call_command('search_index', stdout=self.out, action='delete', force=True) call_command('search_index', stdout=self.out, action='rebuild', force=True) # add dataset and datafile to experiment self.dataset1 = Dataset(description='test_dataset') self.dataset1.save() self.dataset1.experiments.add(self.testexp) self.dataset1.save() settings.REQUIRE_DATAFILE_SIZES = False settings.REQUIRE_DATAFILE_CHECKSUMS = False self.datafile = DataFile(dataset=self.dataset1, filename='test.txt') self.datafile.save()
def test_deleting_dfo_without_uri(self): dataset = Dataset(description="dataset description") dataset.save() save1 = settings.REQUIRE_DATAFILE_SIZES save2 = settings.REQUIRE_DATAFILE_CHECKSUMS try: settings.REQUIRE_DATAFILE_SIZES = False settings.REQUIRE_DATAFILE_CHECKSUMS = False datafile = DataFile(dataset=dataset, filename='test1.txt') datafile.save() finally: settings.REQUIRE_DATAFILE_SIZES = save1 settings.REQUIRE_DATAFILE_CHECKSUMS = save2 dfo = DataFileObject( datafile=datafile, storage_box=datafile.get_default_storage_box(), uri=None) dfo.save() self.assertIsNone(dfo.uri) self.assertIsNotNone(dfo.id) dfo.delete() self.assertIsNone(dfo.id)
def testLocalFile(self): content = urandom(1024) cf = ContentFile(content, 'background_task_testfile') # Create new Datafile datafile = DataFile(dataset=self.dataset) datafile.filename = cf.name datafile.size = len(content) datafile.sha512sum = hashlib.sha512(content).hexdigest() datafile.save() datafile.file_object = cf dfo = datafile.file_objects.all()[0] # undo auto-verify: dfo.verified = False dfo.save(update_fields=['verified']) # Check that it's not currently verified expect(datafile.verified).to_be(False) # Check it verifies verify_dfos() expect(datafile.verified).to_be(True)
def test_002_update_df_status_skip_unverified(self, mock_stat, df_online): """update_df_status should skip files that are unverified""" df2 = DataFile(dataset=self.dataset, filename="test_df2.jpg") df2.save() dfo2 = DataFileObject(datafile=df2, storage_box=self.sbox1, uri="stream/test_df2.jpg") dfo2.save() schema = Schema.objects.get(namespace=HSM_DATAFILE_NAMESPACE) ps2 = DatafileParameterSet(schema=schema, datafile=df2) ps2.save() param_name = ParameterName.objects.get(schema=schema, name="online") param2 = DatafileParameter(parameterset=ps2, name=param_name) param2.string_value = True param2.save() mock_stat.return_value = Stats(st_size=10000, st_blocks=100, st_mtime=datetime.now()) update_df_status() df_online.assert_not_called()
def setUp(self): from django.contrib.auth.models import User from tempfile import mkdtemp user = '******' pwd = 'secret' email = '' self.user = User.objects.create_user(user, email, pwd) self.test_dir = mkdtemp() self.exp = Experiment(title='test exp1', institution_name='monash', created_by=self.user) self.exp.save() self.dataset = Dataset(description="dataset description...") self.dataset.save() self.dataset.experiments.add(self.exp) self.dataset.save() self.datafile = DataFile(dataset=self.dataset, filename="testfile.txt", size="42", md5sum='bogus') self.datafile.save() self.dfo = DataFileObject( datafile=self.datafile, storage_box=self.datafile.get_default_storage_box(), uri="1/testfile.txt") self.dfo.save() self.schema = Schema(namespace="http://localhost/psmtest/df/", name="Parameter Set Manager", type=3) self.schema.save() self.parametername1 = ParameterName(schema=self.schema, name="parameter1", full_name="Parameter 1") self.parametername1.save() self.parametername2 = ParameterName(schema=self.schema, name="parameter2", full_name="Parameter 2", data_type=ParameterName.NUMERIC) self.parametername2.save() self.parametername3 = ParameterName(schema=self.schema, name="parameter3", full_name="Parameter 3", data_type=ParameterName.DATETIME) self.parametername3.save() self.datafileparameterset = DatafileParameterSet( schema=self.schema, datafile=self.datafile) self.datafileparameterset.save() self.datafileparameter1 = DatafileParameter( parameterset=self.datafileparameterset, name=self.parametername1, string_value="test1") self.datafileparameter1.save() self.datafileparameter2 = DatafileParameter( parameterset=self.datafileparameterset, name=self.parametername2, numerical_value=2) self.datafileparameter2.save() # Create a ParameterName and Parameter of type LINK to an experiment self.parametername_exp_link = ParameterName( schema=self.schema, name="exp_link", full_name="This parameter is a experiment LINK", data_type=ParameterName.LINK) self.parametername_exp_link.save() self.exp_link_param = DatafileParameter( parameterset=self.datafileparameterset, name=self.parametername_exp_link) exp_url = self.exp.get_absolute_url() # /experiment/view/1/ self.exp_link_param.set_value(exp_url) self.exp_link_param.save() # Create a ParameterName and Parameter of type LINK to a dataset self.parametername_dataset_link = ParameterName( schema=self.schema, name="dataset_link", full_name="This parameter is a dataset LINK", data_type=ParameterName.LINK) self.parametername_dataset_link.save() self.dataset_link_param = DatafileParameter( parameterset=self.datafileparameterset, name=self.parametername_dataset_link) dataset_url = self.dataset.get_absolute_url() # /dataset/1/ self.dataset_link_param.set_value(dataset_url) self.dataset_link_param.save() # Create a ParameterName type LINK to an unresolvable (non-URL) # free-text value self.parametername_unresolvable_link = ParameterName( schema=self.schema, name="freetext_link", full_name="This parameter is a non-URL LINK", data_type=ParameterName.LINK) self.parametername_unresolvable_link.save()
def test_datafile(self, mock_send_task): def _build(dataset, filename, url=None): datafile = DataFile(dataset=dataset, filename=filename) datafile.save() if url is None: datafile.file_object = StringIO(u'bla') return datafile dfo = DataFileObject( datafile=datafile, storage_box=datafile.get_default_storage_box(), uri=url) dfo.save() # Tests are run with CELERY_ALWAYS_EAGER = True, # so saving a DFO will trigger an immediate attempt # to verify the DFO which will trigger an attempt # to apply filters because we are overriding the # USE_FILTERS setting to True in this test: self.assertNotEqual(mock_send_task.call_count, 0) return datafile exp = Experiment(title='test exp1', institution_name='monash', approved=True, created_by=self.user, public_access=Experiment.PUBLIC_ACCESS_NONE) exp.save() acl = ObjectACL( pluginId='django_user', entityId=str(self.user.id), content_object=exp, canRead=True, canWrite=True, aclOwnershipType=ObjectACL.OWNER_OWNED, ) acl.save() dataset = Dataset(description="dataset description...\nwith; issues") dataset.save() dataset.experiments.add(exp) dataset.save() save1 = settings.REQUIRE_DATAFILE_SIZES save2 = settings.REQUIRE_DATAFILE_CHECKSUMS saved_render_image_size_limit = getattr(settings, 'RENDER_IMAGE_SIZE_LIMIT', 0) try: settings.REQUIRE_DATAFILE_SIZES = False settings.REQUIRE_DATAFILE_CHECKSUMS = False df_file = _build(dataset, 'file.txt', 'path/file.txt') first_id = df_file.id self.assertEqual(df_file.filename, 'file.txt') self.assertEqual(df_file.file_objects.all()[0].uri, 'path/file.txt') self.assertEqual(df_file.dataset, dataset) self.assertEqual(df_file.size, None) self.assertEqual( df_file.get_download_url(), '/api/v1/dataset_file/%d/download%s' % (first_id, trailing_slash())) # Test string representation of DataFileObject: dfo = df_file.get_preferred_dfo() self.assertEqual( str(dfo), "Box: %s, URI: %s, verified: %s" % (str(dfo.storage_box), dfo.uri, str(dfo.verified))) # Test constructing absolute file path: self.assertEqual( df_file.get_absolute_filepath(), os.path.join(settings.DEFAULT_STORAGE_BASE_DIR, dfo.uri)) # get_as_temporary_file() doesn't work for a StringIO file object: if not os.path.exists(os.path.dirname(dfo.get_full_path())): os.makedirs(os.path.dirname(dfo.get_full_path())) with open(dfo.get_full_path(), 'w') as file_obj: file_obj.write(u'bla') # Test ability to check out a temporary copy of file: with df_file.get_as_temporary_file() as temp_file_obj: self.assertEqual(temp_file_obj.read().decode(), u'bla') self.assertFalse(df_file.has_image()) # Test checking online status, i.e. whether the DataFile # has at least one verified DataFileObject in a non-tape # storage box: self.assertTrue(df_file.is_online) DataFileObject.objects.get(datafile=df_file).delete() # This behaviour is documented in the is_online property # method's docstring, i.e. is_online is expected to be # True for a DataFile without any DataFileObjects: self.assertTrue(df_file.is_online) # Test method for getting MIME type: self.assertEqual(df_file.get_mimetype(), "text/plain") df_file.mimetype = "" # DataFile's save automatically updates the mimetype, # and we want to test get_mimetype without a mimetype: models.Model.save(df_file) self.assertEqual(df_file.get_mimetype(), "text/plain") df_file.filename = "file.unknown-extension" models.Model.save(df_file) self.assertEqual(df_file.get_mimetype(), "application/octet-stream") # Test method for getting view URL for file types which can # be displayed in the browser. # First test a file of unknown MIME type: self.assertIsNone(df_file.get_view_url()) # Now test for a text/plain file: df_file.filename = "file.txt" df_file.save() self.assertEqual(df_file.mimetype, "text/plain") self.assertEqual(df_file.get_view_url(), "/datafile/view/%s/" % df_file.id) # This setting will prevent files larger than 2 bytes # from being rendered in the browser: settings.RENDER_IMAGE_SIZE_LIMIT = 2 df_file.size = 3 df_file.save() self.assertIsNone(df_file.get_view_url()) df_file = _build(dataset, 'file1.txt', 'path/file1.txt') self.assertEqual(df_file.filename, 'file1.txt') self.assertEqual(df_file.file_objects.all()[0].uri, 'path/file1.txt') self.assertEqual(df_file.dataset, dataset) self.assertEqual(df_file.size, None) self.assertEqual( df_file.get_download_url(), '/api/v1/dataset_file/%d/download%s' % (first_id + 1, trailing_slash())) df_file = _build(dataset, 'file2.txt', 'path/file2#txt') self.assertEqual(df_file.filename, 'file2.txt') self.assertEqual(df_file.dataset, dataset) self.assertEqual(df_file.size, None) self.assertEqual( df_file.get_download_url(), '/api/v1/dataset_file/%d/download%s' % (first_id + 2, trailing_slash())) df_file = _build(dataset, 'f.txt', 'http://localhost:8080/filestore/f.txt') self.assertEqual(df_file.filename, 'f.txt') self.assertEqual(df_file.dataset, dataset) self.assertEqual(df_file.size, None) self.assertEqual( df_file.get_download_url(), '/api/v1/dataset_file/%d/download%s' % (first_id + 3, trailing_slash())) df_file = _build(dataset, 'f-bad-ds.txt') self.assertEqual(df_file.filename, 'f-bad-ds.txt') self.assertEqual(df_file.dataset, dataset) self.assertEqual(df_file.size, None) self.assertEqual( df_file.get_download_url(), '/api/v1/dataset_file/%d/download%s' % (first_id + 4, trailing_slash())) pattern = re.compile('\n|;') self.assertFalse(pattern.search(df_file.file_objects.first().uri)) # check that can't save negative byte sizes with self.assertRaises(Exception): settings.REQUIRE_DATAFILE_SIZES = True DataFile(dataset=dataset, filename='lessthanempty.txt', size=-1).save() # Now check the 'REQUIRE' config params with self.assertRaises(Exception): settings.REQUIRE_DATAFILE_SIZES = True settings.REQUIRE_DATAFILE_CHECKSUMS = False DataFile(dataset=dataset, filename='foo.txt', md5sum='bad').save() with self.assertRaises(Exception): settings.REQUIRE_DATAFILE_SIZES = False settings.REQUIRE_DATAFILE_CHECKSUMS = True DataFile(dataset=dataset, filename='foo.txt', size=1).save() finally: settings.REQUIRE_DATAFILE_SIZES = save1 settings.REQUIRE_DATAFILE_CHECKSUMS = save2 settings.RENDER_IMAGE_SIZE_LIMIT = saved_render_image_size_limit
def test_parameter(self): exp = Experiment( title='test exp1', institution_name='Australian Synchrotron', approved=True, created_by=self.user, public_access=Experiment.PUBLIC_ACCESS_NONE, ) exp.save() dataset = Dataset(description="dataset description") dataset.save() dataset.experiments.add(exp) dataset.save() df_file = DataFile(dataset=dataset, filename='file.txt', size=42, md5sum='bogus') df_file.save() df_schema = Schema(namespace='http://www.cern.ch/felzmann/schema1.xml', type=Schema.DATAFILE) df_schema.save() ds_schema = Schema(namespace='http://www.cern.ch/felzmann/schema2.xml', type=Schema.DATASET) ds_schema.save() exp_schema = Schema( namespace='http://www.cern.ch/felzmann/schema3.xml', type=Schema.EXPERIMENT) exp_schema.save() df_parname = ParameterName(schema=df_schema, name='name', full_name='full_name', units='image/jpg', data_type=ParameterName.FILENAME) df_parname.save() ds_parname = ParameterName(schema=ds_schema, name='name', full_name='full_name', units='image/jpg', data_type=ParameterName.FILENAME) ds_parname.save() exp_parname = ParameterName(schema=exp_schema, name='name', full_name='full_name', units='image/jpg', data_type=ParameterName.FILENAME) exp_parname.save() df_parset = DatafileParameterSet(schema=df_schema, datafile=df_file) df_parset.save() ds_parset = DatasetParameterSet(schema=ds_schema, dataset=dataset) ds_parset.save() exp_parset = ExperimentParameterSet(schema=exp_schema, experiment=exp) exp_parset.save() with self.settings(METADATA_STORE_PATH=os.path.dirname(__file__)): filename = 'test.jpg' df_parameter = DatafileParameter(name=df_parname, parameterset=df_parset, string_value=filename) df_parameter.save() ds_parameter = DatasetParameter(name=ds_parname, parameterset=ds_parset, string_value=filename) ds_parameter.save() exp_parameter = ExperimentParameter(name=exp_parname, parameterset=exp_parset, string_value=filename) exp_parameter.save() self.assertEqual( "<a href='/display/DatafileImage/load/%i/' target='_blank'><img style='width: 300px;' src='/display/DatafileImage/load/%i/' /></a>" % # noqa (df_parameter.id, df_parameter.id), df_parameter.get()) self.assertEqual( "<a href='/display/DatasetImage/load/%i/' target='_blank'><img style='width: 300px;' src='/display/DatasetImage/load/%i/' /></a>" % # noqa (ds_parameter.id, ds_parameter.id), ds_parameter.get()) self.assertEqual( "<a href='/display/ExperimentImage/load/%i/' target='_blank'><img style='width: 300px;' src='/display/ExperimentImage/load/%i/' /></a>" % # noqa (exp_parameter.id, exp_parameter.id), exp_parameter.get())
def test_datafile(self): from tardis.tardis_portal.models import Experiment, Dataset, DataFile def _build(dataset, filename, url=None): datafile = DataFile(dataset=dataset, filename=filename) datafile.save() if url is None: datafile.file_object = StringIO('bla') return datafile from tardis.tardis_portal.models import \ DataFileObject dfo = DataFileObject( datafile=datafile, storage_box=datafile.get_default_storage_box(), uri=url) dfo.save() return datafile exp = Experiment(title='test exp1', institution_name='monash', approved=True, created_by=self.user, public_access=Experiment.PUBLIC_ACCESS_NONE) exp.save() dataset = Dataset(description="dataset description...\nwith; issues") dataset.save() dataset.experiments.add(exp) dataset.save() save1 = settings.REQUIRE_DATAFILE_SIZES save2 = settings.REQUIRE_DATAFILE_CHECKSUMS try: settings.REQUIRE_DATAFILE_SIZES = False settings.REQUIRE_DATAFILE_CHECKSUMS = False df_file = _build(dataset, 'file.txt', 'path/file.txt') first_id = df_file.id self.assertEqual(df_file.filename, 'file.txt') self.assertEqual(df_file.file_objects.all()[0].uri, 'path/file.txt') self.assertEqual(df_file.dataset, dataset) self.assertEqual(df_file.size, None) self.assertEqual( df_file.get_download_url(), '/api/v1/dataset_file/%d/download%s' % (first_id, trailing_slash())) df_file = _build(dataset, 'file1.txt', 'path/file1.txt') self.assertEqual(df_file.filename, 'file1.txt') self.assertEqual(df_file.file_objects.all()[0].uri, 'path/file1.txt') self.assertEqual(df_file.dataset, dataset) self.assertEqual(df_file.size, None) self.assertEqual( df_file.get_download_url(), '/api/v1/dataset_file/%d/download%s' % (first_id + 1, trailing_slash())) df_file = _build(dataset, 'file2.txt', 'path/file2#txt') self.assertEqual(df_file.filename, 'file2.txt') self.assertEqual(df_file.dataset, dataset) self.assertEqual(df_file.size, None) self.assertEqual( df_file.get_download_url(), '/api/v1/dataset_file/%d/download%s' % (first_id + 2, trailing_slash())) df_file = _build(dataset, 'f.txt', 'http://localhost:8080/filestore/f.txt') self.assertEqual(df_file.filename, 'f.txt') self.assertEqual(df_file.dataset, dataset) self.assertEqual(df_file.size, None) self.assertEqual( df_file.get_download_url(), '/api/v1/dataset_file/%d/download%s' % (first_id + 3, trailing_slash())) df_file = _build(dataset, 'f-bad-ds.txt') self.assertEqual(df_file.filename, 'f-bad-ds.txt') self.assertEqual(df_file.dataset, dataset) self.assertEqual(df_file.size, None) self.assertEqual( df_file.get_download_url(), '/api/v1/dataset_file/%d/download%s' % (first_id + 4, trailing_slash())) self.assertNotRegexpMatches(df_file.file_objects.first().uri, '\n|;') # check that can't save negative byte sizes with self.assertRaises(Exception): settings.REQUIRE_DATAFILE_SIZES = True DataFile(dataset=dataset, filename='lessthanempty.txt', size=-1).save() # Now check the 'REQUIRE' config params with self.assertRaises(Exception): settings.REQUIRE_DATAFILE_SIZES = True settings.REQUIRE_DATAFILE_CHECKSUMS = False DataFile(dataset=dataset, filename='foo.txt', md5sum='bad').save() with self.assertRaises(Exception): settings.REQUIRE_DATAFILE_SIZES = False settings.REQUIRE_DATAFILE_CHECKSUMS = True DataFile(dataset=dataset, filename='foo.txt', size=1).save() finally: settings.REQUIRE_DATAFILE_SIZES = save1 settings.REQUIRE_DATAFILE_CHECKSUMS = save2