class MRtaskTestCase(TestCase): def setUp(self): from django.contrib.auth.models import User from tardis.tardis_portal.models import Experiment from tardis.tardis_portal.models import ExperimentACL from tardis.tardis_portal.models import Dataset from tardis.apps.mrtardis.mrtask import MRtask user = '******' pwd = 'secret' email = '' self.user = User.objects.create_user(user, email, pwd) self.experiment = Experiment(approved=True, title="Test Experiment", institution_name="Test Institution", created_by=self.user, public=False) self.experiment.save() acl = ExperimentACL(pluginId="django_user", entityId="1", experiment=self.experiment, canRead=True, canWrite=True, canDelete=True, isOwner=True) acl.save() self.test_dataset = Dataset(experiment=self.experiment, description="test dataset") self.test_dataset.save() self.test_mrtask = MRtask(dataset=self.test_dataset) def test_mrtask(self): self.assertEqual(len(self.test_mrtask.parameters), 0)
def register_squashfile(exp_id, epn, sq_dir, sq_filename, namespace): ''' example: register_squashfile(456, '1234A', '/srv/squashstore', '1234A.squashfs', 'http://synchrotron.org.au/mx/squashfsarchive/1') ''' dfs = DataFile.objects.filter(filename=sq_filename, dataset__experiments__id=exp_id) if len(dfs) == 1: return dfs[0] e = Experiment.objects.get(id=exp_id) ds = Dataset(description="01 SquashFS Archive") ds.save() ds.experiments.add(e) filepath = os.path.join(sq_dir, sq_filename) try: md5sum = open(filepath + '.md5sum', 'r').read().strip()[:32] except: print 'no md5sum file found' return None size = os.path.getsize(filepath) df = DataFile(md5sum=md5sum, filename=sq_filename, size=str(size), dataset=ds) df.save() schema = Schema.objects.filter(namespace=namespace)[0] ps = DatafileParameterSet(schema=schema, datafile=df) ps.save() ps.set_param('EPN', epn) sbox = StorageBox.objects.get(name='squashstore') dfo = DataFileObject(storage_box=sbox, datafile=df, uri=sq_filename) dfo.save() return df
def test_datasetwrapper(self): from mecat.models import DatasetWrapper, Sample sample_desc = "My Description for Sample created in test_datasetwrapper()" sample = Sample(experiment=self.experiment, description=sample_desc) sample.save() self.assertEqual(sample.description, sample_desc) self.assertEqual(sample.experiment, self.experiment) from tardis.tardis_portal.models import Dataset dataset_desc = "My Description for Dataset created in test_datasetwrapper()" dataset = Dataset(description=dataset_desc, experiment=self.experiment) dataset.save() sample_from_db = Sample.objects.get(description=sample_desc) dataset_from_db = Dataset.objects.get(description=dataset_desc) datasetwrapper = DatasetWrapper(sample=sample_from_db, dataset=dataset_from_db) datasetwrapper.save() self.assertEqual(datasetwrapper.sample, sample_from_db) self.assertEqual(datasetwrapper.dataset, dataset_from_db) datasetwrapper_from_db = DatasetWrapper.objects.get(sample__description=sample_desc) self.assertEqual(datasetwrapper_from_db.dataset.pk, dataset_from_db.pk)
def setUp(self): # Create test owner without enough details username, email, password = ('testuser', '*****@*****.**', 'password') user = User.objects.create_user(username, email, password) profile = UserProfile(user=user, isDjangoAccount=True) profile.save() Location.force_initialize() # Create test experiment and make user the owner of it experiment = Experiment(title='Text Experiment', institution_name='Test Uni', created_by=user) experiment.save() acl = ObjectACL( pluginId='django_user', entityId=str(user.id), content_object=experiment, canRead=True, canWrite=True, canDelete=True, isOwner=True, aclOwnershipType=ObjectACL.OWNER_OWNED) acl.save() dataset = Dataset(description='dataset description...') dataset.save() dataset.experiments.add(experiment) dataset.save() def create_datafile(filename): testfile = path.join(path.dirname(__file__), 'fixtures', filename) size, sha512sum = get_size_and_sha512sum(testfile) datafile = Dataset_File(dataset=dataset, filename=path.basename(testfile), size=size, sha512sum=sha512sum) datafile.save() base_url = 'file://' + path.abspath(path.dirname(testfile)) location = Location.load_location({ 'name': 'test-grabber', 'url': base_url, 'type': 'external', 'priority': 10, 'transfer_provider': 'local'}) replica = Replica(datafile=datafile, url='file://'+path.abspath(testfile), protocol='file', location=location) replica.verify() replica.save() return Dataset_File.objects.get(pk=datafile.pk) self.dataset = dataset self.datafiles = [create_datafile('data_grabber_test1.admin'), create_datafile('testfile.txt') ]
def setUp(self): # Create test owner without enough details username, email, password = ('testuser', '*****@*****.**', 'password') user = User.objects.create_user(username, email, password) profile = UserProfile(user=user, isDjangoAccount=True) profile.save() Location.force_initialize() # Create test experiment and make user the owner of it experiment = Experiment(title='Text Experiment', institution_name='Test Uni', created_by=user) experiment.save() acl = ObjectACL( pluginId='django_user', entityId=str(user.id), content_object=experiment, canRead=True, isOwner=True, aclOwnershipType=ObjectACL.OWNER_OWNED, ) acl.save() dataset = Dataset(description='dataset description...') dataset.save() dataset.experiments.add(experiment) dataset.save() def create_datafile(index): testfile = path.join(path.dirname(__file__), 'fixtures', 'jeol_sem_test%d.txt' % index) size, sha512sum = get_size_and_sha512sum(testfile) datafile = Dataset_File(dataset=dataset, filename=path.basename(testfile), size=size, sha512sum=sha512sum) datafile.save() base_url = 'file://' + path.abspath(path.dirname(testfile)) location = Location.load_location({ 'name': 'test-jeol', 'url': base_url, 'type': 'external', 'priority': 10, 'transfer_provider': 'local' }) replica = Replica(datafile=datafile, url='file://' + path.abspath(testfile), protocol='file', location=location) replica.verify() replica.save() return Dataset_File.objects.get(pk=datafile.pk) self.dataset = dataset self.datafiles = [create_datafile(i) for i in (1, 2)]
def _create_test_dataset(nosDatafiles): ds_ = Dataset(description='happy snaps of plumage') ds_.save() for i in range (0, nosDatafiles) : df_ = Dataset_File(dataset=ds_, url='http://planet-python.org/' + str(_next_id())) df_.save() ds_.save() return ds_
def _make_dataset(self, exp, filenames): dataset = Dataset(experiment=exp) dataset.save() for filename in filenames: df = Dataset_File(dataset=dataset, size=41, protocol='file') df.filename = filename df.url = 'file://' + path.join(path.dirname(__file__), 'data', df.filename) df.save()
def _create_test_dataset(nosDatafiles): ds_ = Dataset(description='happy snaps of plumage') ds_.save() for i in range(0, nosDatafiles): df_ = Dataset_File(dataset=ds_, url='http://planet-python.org/' + str(_next_id())) df_.save() ds_.save() return ds_
def _create_test_dataset(nosDatafiles): ds_ = Dataset(description='happy snaps of plumage') ds_.save() for i in range(0, nosDatafiles): df_ = DataFile(dataset=ds_, filename='file_%d' % i, size='21', sha512sum='bogus') df_.save() ds_.save() return ds_
def setUp(self): raise SkipTest # temporarily disabling this feature, needs coding from tempfile import mkdtemp, mktemp from django.conf import settings import os # Disconnect post_save signal from django.db.models.signals import post_save from tardis.tardis_portal.models import Experiment, \ staging_hook, Dataset, DataFile, DataFileObject, StorageBox post_save.disconnect(staging_hook, sender=DataFileObject) from django.contrib.auth.models import User user = '******' pwd = 'secret' email = '' self.user = User.objects.create_user(user, email, pwd) try: os.makedirs(settings.GET_FULL_STAGING_PATH_TEST) except OSError: pass self.temp = mkdtemp(dir=settings.GET_FULL_STAGING_PATH_TEST) self.filepath = mktemp(dir=self.temp) content = 'test file' with open(self.filepath, "w+b") as f: f.write(content) # make datafile exp = Experiment(title='test exp1', institution_name='monash', created_by=self.user) exp.save() # make dataset dataset = Dataset(description="dataset description...") dataset.save() dataset.experiments.add(exp) dataset.save() # create datafile df = DataFile(dataset=dataset, size=len(content), filename=path.basename(self.file), md5sum='f20d9f2072bbeb6691c0f9c5099b01f3') df.save() # create replica base_url = settings.GET_FULL_STAGING_PATH_TEST s_box = StorageBox.get_default_storage(location=base_url) dfo = DataFileObject(datafile=df, uri=self.filepath, storage_box=s_box) dfo.save() self.dfo = dfo
def generate_dataset(datafiles=[], experiments=[]): from tardis.tardis_portal.models import Dataset dataset = Dataset() dataset.save() for df in datafiles: df.dataset_id = dataset.id df.save() for exp in experiments: dataset.experiments.add(exp) dataset.save() return dataset
def _create_test_dataset(nosDatafiles): ds_ = Dataset(description='happy snaps of plumage') ds_.save() for i in range(0, nosDatafiles): df_ = Dataset_File(dataset=ds_, size='21', sha512sum='bogus') df_.save() rep_ = Replica(datafile=df_, url='http://planet-python.org/' + str(_next_id()), location=Location.get_default_location()) rep_.save() ds_.save() return ds_
def setUp(self): # Create test owner without enough details username, email, password = ('testuser', '*****@*****.**', 'password') user = User.objects.create_user(username, email, password) # Create test experiment and make user the owner of it experiment = Experiment(title='Text Experiment', institution_name='Test Uni', created_by=user) experiment.save() acl = ObjectACL( pluginId='django_user', entityId=str(user.id), content_object=experiment, canRead=True, isOwner=True, aclOwnershipType=ObjectACL.OWNER_OWNED, ) acl.save() dataset = Dataset(description='dataset description...') dataset.save() dataset.experiments.add(experiment) dataset.save() base_path = path.join(path.dirname(__file__), 'fixtures') s_box = StorageBox.get_default_storage(location=base_path) def create_datafile(index): testfile = path.join(base_path, 'middleware_test%d.txt' % index) size, sha512sum = get_size_and_sha512sum(testfile) datafile = DataFile(dataset=dataset, filename=path.basename(testfile), size=size, sha512sum=sha512sum) datafile.save() dfo = DataFileObject( datafile=datafile, storage_box=s_box, uri=path.basename(testfile)) dfo.save() if index != 1: dfo.verified = False dfo.save(update_fields=['verified']) return DataFile.objects.get(pk=datafile.pk) self.dataset = dataset self.datafiles = [create_datafile(i) for i in (1, 2)]
def _create_test_dataset(nosDatafiles): ds_ = Dataset(description='happy snaps of plumage') ds_.save() for i in range (0, nosDatafiles) : df_ = Dataset_File(dataset=ds_, size='21', sha512sum='bogus') df_.save() rep_ = Replica(datafile=df_, url='http://planet-python.org/' + str(_next_id()), location=Location.get_default_location()) rep_.save() ds_.save() return ds_
def test_urls_with_some_content(self): # Things that might tend to be in a real live system user = '******' pwd = User.objects.make_random_password() user = User.objects.create(username=user, email='*****@*****.**', first_name="Test", last_name="User") user.set_password(pwd) user.save() experiment = Experiment.objects.create( title="Test Experiment", created_by=user, public_access=Experiment.PUBLIC_ACCESS_FULL) experiment.save() acl = ObjectACL(pluginId=django_user, entityId=str(user.id), content_object=experiment, canRead=True, canWrite=True, canDelete=True, isOwner=True) acl.save() dataset = Dataset(description="test dataset") dataset.save() dataset.experiments.add(experiment) dataset.save() # Test everything works c = Client() c.login(username=user, password=pwd) urls = ['/about/', '/stats/'] urls += ['/experiment/list/%s' % part for part in ('mine', 'shared', 'public')] # urls += ['/experiment/%s/' % part # for part in ('search',)] urls += ['/experiment/view/%d/' % experiment.id] urls += ['/ajax/experiment/%d/%s' % (experiment.id, tabpane) for tabpane in ('description', 'datasets', 'rights')] urls += ['/ajax/datafile_list/%d/' % dataset.id] urls += ['/ajax/dataset_metadata/%d/' % dataset.id] for u in urls: response = c.get(u) ensure(response.status_code, 200, "%s should have returned 200 but returned %d" % (u, response.status_code)) redirect_urls = ['/experiment/list', '/experiment/view/'] for u in redirect_urls: response = c.get(u) expect(response.status_code).to_equal(302)
def _create_dataset(self): user = User.objects.create_user('testuser', '*****@*****.**', 'pwd') user.save() full_access = Experiment.PUBLIC_ACCESS_FULL experiment = Experiment.objects.create(title="Background Test", created_by=user, public_access=full_access) experiment.save() dataset = Dataset() dataset.save() dataset.experiments.add(experiment) dataset.save() return dataset
def setUp(self): # Create test owner without enough details username, email, password = ('testuser', '*****@*****.**', 'password') user = User.objects.create_user(username, email, password) profile = UserProfile(user=user, isDjangoAccount=True) profile.save() # Create test experiment and make user the owner of it experiment = Experiment(title='Text Experiment', institution_name='Test Uni', created_by=user) experiment.save() acl = ExperimentACL( pluginId='django_user', entityId=str(user.id), experiment=experiment, canRead=True, isOwner=True, aclOwnershipType=ExperimentACL.OWNER_OWNED, ) acl.save() dataset = Dataset(description='dataset description...') dataset.save() dataset.experiments.add(experiment) dataset.save() def create_datafile(index): testfile = path.join(path.dirname(__file__), 'fixtures', 'jeol_sem_test%d.txt' % index) size, sha512sum = get_size_and_sha512sum(testfile) datafile = Dataset_File(dataset=dataset, filename=path.basename(testfile), url='file://'+path.abspath(testfile), protocol='file', size=size, sha512sum=sha512sum) datafile.verify() datafile.save() return datafile self.dataset = dataset self.datafiles = [create_datafile(i) for i in (1,2)]
def test_contextual_view(self): """ Given schema on dataset, check that image file created """ user = _create_test_user() license = _create_license() exp = _create_test_experiment(user, license) ds = Dataset(description='happy snaps of plumage') ds.save() ds = _create_test_dataset( ds, exp.id, { "output.dat": 'test data\n', "grexp.dat": '1 2\n2 3\n3 7\n', "grfinal21.dat": '1 2\n 2 4\n4 9\n' }) sch = Schema(namespace=self.HRMCSCHEMA, name="hrmc_views", type=Schema.DATASET) sch.save() param = ParameterName(schema=sch, name="plot", full_name="scatterplot", units="image", data_type=ParameterName.FILENAME) param.save() dps = DatasetParameterSet(schema=sch, dataset=ds) dps.save() ds.experiments.add(exp) ds.save() client = Client() response = client.get('/dataset/%s' % ds.id) self.assertEqual(response.status_code, 200) param_sets = get_param_sets(ds) self.assertTrue(param_sets) dp = DatasetParameter.objects.get(parameterset=param_sets[0], name=param) self.assertTrue(dp) self.assertNotEquals(dp.string_value, "") # ie, it has a filename
def test_003_offline_dataset(self, mock_stat): """A dataset should be offline if any datafiles are offline""" mock_stat.return_value = Stats(st_size=10000, st_blocks=0, st_mtime=datetime.now()) ds = Dataset(description="Dataset2", instrument=self.inst) ds.save() df2 = DataFile(dataset=ds, filename="test_file.jpg") df2.save() dfo2 = DataFileObject(datafile=df2, storage_box=self.sbox1, uri=df2.filename) dfo2.save() df2.verify() self.assertFalse(dataset_online(ds))
def test_hrmc_filter(self): """ Make an experiment, lood up grexp file and check dataset schema missing, then loadup grfinal and check dataset schema created """ user = _create_test_user() license = _create_license() exp = _create_test_experiment(user, license) ds = Dataset(description='happy snaps of plumage') ds.save() _create_test_dataset(ds, exp.id, { "output.dat": 'hello', "grexp.dat": '2 5\n6 15\n' }) ds.experiments.add(exp) ds.save() sch = Schema(namespace=self.HRMCSCHEMA, name="hrmc_views", type=Schema.DATASET) sch.save() param = ParameterName(schema=sch, name="plot", full_name="scatterplot", units="image", data_type=ParameterName.FILENAME) param.save() param_sets = get_param_sets(ds) self.assertEquals(list(param_sets), []) _create_test_dataset(ds, exp.id, {'grfinal21.dat': "1 3\n5 14\n"}) df2 = Dataset_File(dataset=ds, url='path/grfinal21.dat') df2.save() h = hrmc.HRMCOutput('HRMC', self.HRMCSCHEMA) h(sender=Dataset_File, instance=df2) param_sets = get_param_sets(ds) self.assertEquals([x.schema.namespace for x in param_sets], [self.HRMCSCHEMA])
def _create_datafile(): user = User.objects.create_user('testuser', '*****@*****.**', 'pwd') user.save() full_access = Experiment.PUBLIC_ACCESS_FULL experiment = Experiment.objects.create(title="IIIF Test", created_by=user, public_access=full_access) experiment.save() ObjectACL(content_object=experiment, pluginId='django_user', entityId=str(user.id), isOwner=True, canRead=True, canWrite=True, canDelete=True, aclOwnershipType=ObjectACL.OWNER_OWNED).save() dataset = Dataset() dataset.save() dataset.experiments.add(experiment) dataset.save() # Create new Datafile tempfile = TemporaryUploadedFile('iiif_stored_file', None, None, None) with Image(filename='magick:rose') as img: img.format = 'tiff' img.save(file=tempfile.file) tempfile.file.flush() datafile = DataFile(dataset=dataset, size=os.path.getsize(tempfile.file.name), filename='iiif_named_file', mimetype='image/tiff') compute_md5 = getattr(settings, 'COMPUTE_MD5', True) compute_sha512 = getattr(settings, 'COMPUTE_SHA512', True) checksums = compute_checksums(open(tempfile.file.name, 'r'), compute_md5=compute_md5, compute_sha512=compute_sha512) if compute_md5: datafile.md5sum = checksums['md5sum'] if compute_sha512: datafile.sha512sum = checksums['sha512sum'] datafile.save() datafile.file_object = tempfile return datafile
def _create_datafile(): user = User.objects.create_user('testuser', '*****@*****.**', 'pwd') user.save() UserProfile(user=user).save() Location.force_initialize() full_access = Experiment.PUBLIC_ACCESS_FULL experiment = Experiment.objects.create(title="IIIF Test", created_by=user, public_access=full_access) experiment.save() ObjectACL(content_object=experiment, pluginId='django_user', entityId=str(user.id), isOwner=True, canRead=True, canWrite=True, canDelete=True, aclOwnershipType=ObjectACL.OWNER_OWNED).save() dataset = Dataset() dataset.save() dataset.experiments.add(experiment) dataset.save() # Create new Datafile tempfile = TemporaryUploadedFile('iiif_stored_file', None, None, None) with Image(filename='magick:rose') as img: img.format = 'tiff' img.save(file=tempfile.file) tempfile.file.flush() datafile = Dataset_File(dataset=dataset, size=os.path.getsize(tempfile.file.name), filename='iiif_named_file') replica = Replica(datafile=datafile, url=write_uploaded_file_to_dataset(dataset, tempfile), location=Location.get_default_location()) replica.verify(allowEmptyChecksums=True) datafile.save() replica.datafile = datafile replica.save() return datafile
def test_contextual_view(self): """ Given schema on dataset, check that image file created """ user = _create_test_user() license = _create_license() exp = _create_test_experiment(user, license) ds = Dataset(description='happy snaps of plumage') ds.save() ds = _create_test_dataset(ds, exp.id, { "output.dat": 'test data\n', "grexp.dat": '1 2\n2 3\n3 7\n', "grfinal21.dat": '1 2\n 2 4\n4 9\n'}) sch = Schema(namespace=self.HRMCSCHEMA, name="hrmc_views", type=Schema.DATASET) sch.save() param = ParameterName(schema=sch, name="plot", full_name="scatterplot", units="image", data_type=ParameterName.FILENAME ) param.save() dps = DatasetParameterSet(schema=sch, dataset=ds) dps.save() ds.experiments.add(exp) ds.save() client = Client() response = client.get('/dataset/%s' % ds.id) self.assertEqual(response.status_code, 200) param_sets = get_param_sets(ds) self.assertTrue(param_sets) dp = DatasetParameter.objects.get(parameterset=param_sets[0], name=param) self.assertTrue(dp) self.assertNotEquals(dp.string_value, "") # ie, it has a filename
def _create_datafile(): user = User.objects.create_user("testuser", "*****@*****.**", "pwd") user.save() UserProfile(user=user).save() Location.force_initialize() full_access = Experiment.PUBLIC_ACCESS_FULL experiment = Experiment.objects.create(title="IIIF Test", created_by=user, public_access=full_access) experiment.save() ObjectACL( content_object=experiment, pluginId="django_user", entityId=str(user.id), isOwner=True, canRead=True, canWrite=True, canDelete=True, aclOwnershipType=ObjectACL.OWNER_OWNED, ).save() dataset = Dataset() dataset.save() dataset.experiments.add(experiment) dataset.save() # Create new Datafile tempfile = TemporaryUploadedFile("iiif_stored_file", None, None, None) with Image(filename="magick:rose") as img: img.format = "tiff" img.save(file=tempfile.file) tempfile.file.flush() datafile = Dataset_File(dataset=dataset, size=os.path.getsize(tempfile.file.name), filename="iiif_named_file") replica = Replica( datafile=datafile, url=write_uploaded_file_to_dataset(dataset, tempfile), location=Location.get_default_location(), ) replica.verify(allowEmptyChecksums=True) datafile.save() replica.datafile = datafile replica.save() return datafile
def test_hrmc_filter(self): """ Make an experiment, lood up grexp file and check dataset schema missing, then loadup grfinal and check dataset schema created """ user = _create_test_user() license = _create_license() exp = _create_test_experiment(user, license) ds = Dataset(description='happy snaps of plumage') ds.save() _create_test_dataset(ds, exp.id, {"output.dat": 'hello', "grexp.dat": '2 5\n6 15\n'}) ds.experiments.add(exp) ds.save() sch = Schema(namespace=self.HRMCSCHEMA, name="hrmc_views", type=Schema.DATASET) sch.save() param = ParameterName(schema=sch, name="plot", full_name="scatterplot", units="image", data_type=ParameterName.FILENAME ) param.save() param_sets = get_param_sets(ds) self.assertEquals(list(param_sets), []) _create_test_dataset(ds, exp.id, {'grfinal21.dat': "1 3\n5 14\n"}) df2 = Dataset_File(dataset=ds, url='path/grfinal21.dat') df2.save() h = hrmc.HRMCOutput('HRMC', self.HRMCSCHEMA) h(sender=Dataset_File, instance=df2) param_sets = get_param_sets(ds) self.assertEquals([x.schema.namespace for x in param_sets], [self.HRMCSCHEMA])
def test_deleting_dfo_without_uri(self): dataset = Dataset(description="dataset description") dataset.save() save1 = settings.REQUIRE_DATAFILE_SIZES save2 = settings.REQUIRE_DATAFILE_CHECKSUMS try: settings.REQUIRE_DATAFILE_SIZES = False settings.REQUIRE_DATAFILE_CHECKSUMS = False datafile = DataFile(dataset=dataset, filename='test1.txt') datafile.save() finally: settings.REQUIRE_DATAFILE_SIZES = save1 settings.REQUIRE_DATAFILE_CHECKSUMS = save2 dfo = DataFileObject( datafile=datafile, storage_box=datafile.get_default_storage_box(), uri=None) dfo.save() self.assertIsNone(dfo.uri) self.assertIsNotNone(dfo.id) dfo.delete() self.assertIsNone(dfo.id)
def add_dataset(request, experiment_id): if not has_experiment_write(request, experiment_id): return HttpResponseForbidden() # Process form or prepopulate it if request.method == 'POST': form = DatasetForm(request.POST) if form.is_valid(): dataset = Dataset() dataset.description = form.cleaned_data['description'] dataset.save() experiment = Experiment.objects.get(id=experiment_id) dataset.experiments.add(experiment) dataset.save() return _redirect_303('tardis_portal.view_dataset', dataset.id) else: form = DatasetForm() c = {'form': form} return HttpResponse( render_response_index(request, 'tardis_portal/add_or_edit_dataset.html', c))
def add_dataset(request, experiment_id): if not has_experiment_write(request, experiment_id): return HttpResponseForbidden() # Process form or prepopulate it if request.method == 'POST': form = DatasetForm(request.POST) if form.is_valid(): dataset = Dataset() dataset.description = form.cleaned_data['description'] dataset.save() experiment = Experiment.objects.get(id=experiment_id) dataset.experiments.add(experiment) dataset.save() return _redirect_303('tardis_portal.view_dataset', dataset.id) else: form = DatasetForm() c = {'form': form} return HttpResponse(render_response_index(request, 'tardis_portal/add_or_edit_dataset.html', c))
def test_dataset(self): exp = Experiment(title='test exp1', institution_name='monash', created_by=self.user) exp.save() exp2 = Experiment(title='test exp2', institution_name='monash', created_by=self.user) exp2.save() group = Group(name="Test Manager Group") group.save() group.user_set.add(self.user) facility = Facility(name="Test Facility", manager_group=group) facility.save() instrument = Instrument(name="Test Instrument", facility=facility) instrument.save() dataset = Dataset(description='test dataset1') dataset.instrument = instrument dataset.save() dataset.experiments.set([exp, exp2]) dataset.save() dataset_id = dataset.id del dataset dataset = Dataset.objects.get(pk=dataset_id) self.assertEqual(dataset.description, 'test dataset1') self.assertEqual(dataset.experiments.count(), 2) self.assertIn(exp, list(dataset.experiments.iterator())) self.assertIn(exp2, list(dataset.experiments.iterator())) self.assertEqual(instrument, dataset.instrument) target_id = Dataset.objects.first().id self.assertEqual( dataset.get_absolute_url(), '/dataset/%d' % target_id, dataset.get_absolute_url() + ' != /dataset/%d' % target_id)
def get_or_create_dataset(self, name, top=None): ''' returns existing or created dataset given a name returns False if the dataset is not unique by name top is the directory ''' ds = Dataset.objects.filter(description=name, experiments=self.experiment) if len(ds) == 1: return ds[0] elif len(ds) > 1: return False ds = Dataset(description=name) if top is not None: ds.directory = top ds.save() self.tag_user(ds, top) else: ds.save() ds.experiments.add(self.experiment) return ds
def _create_datafile(): user = User.objects.create_user("testuser", "*****@*****.**", "pwd") user.save() full_access = Experiment.PUBLIC_ACCESS_FULL experiment = Experiment.objects.create(title="IIIF Test", created_by=user, public_access=full_access) experiment.save() ObjectACL( content_object=experiment, pluginId="django_user", entityId=str(user.id), isOwner=True, canRead=True, canWrite=True, canDelete=True, aclOwnershipType=ObjectACL.OWNER_OWNED, ).save() dataset = Dataset() dataset.save() dataset.experiments.add(experiment) dataset.save() # Create new Datafile tempfile = TemporaryUploadedFile("iiif_stored_file", None, None, None) with Image(filename="magick:rose") as img: img.format = "tiff" img.save(file=tempfile.file) tempfile.file.flush() datafile = DataFile( dataset=dataset, size=os.path.getsize(tempfile.file.name), filename="iiif_named_file", mimetype="image/tiff" ) checksums = compute_checksums(open(tempfile.file.name, "r")) datafile.md5sum = checksums["md5sum"] datafile.sha512sum = checksums["sha512sum"] datafile.save() datafile.file_object = tempfile return datafile
def _prepare_project(self): project = Project(experiment=self.experiment) project.save() s1 = Sample(experiment=self.experiment, name="S1", description="s1 desc") s1.save() s2 = Sample(experiment=self.experiment, name="S2", description="s2 desc") s2.save() dw1 = DatasetWrapper(sample=s1, name="dw1", description="dw1 desc") dw2 = DatasetWrapper(sample=s1, name="dw2", description="dw2 desc") dw3 = DatasetWrapper(sample=s2, name="dw3", description="dw3 desc") ds1 = Dataset(experiment=self.experiment, description=dw1.description) ds2 = Dataset(experiment=self.experiment, description=dw2.description) ds3 = Dataset(experiment=self.experiment, description=dw3.description) ds1.save() ds2.save() ds3.save() dw1.dataset = ds1 dw1.save() dw2.dataset = ds2 dw2.save() dw3.dataset = ds3 dw3.save() return project
class DownloadTestCase(TestCase): def setUp(self): # create a test user self.user = User.objects.create_user(username='******', email='', password='******') Location.force_initialize() # create a public experiment self.experiment1 = Experiment(title='Experiment 1', created_by=self.user, public_access=Experiment.PUBLIC_ACCESS_FULL) self.experiment1.save() # create a non-public experiment self.experiment2 = Experiment(title='Experiment 2', created_by=self.user, public_access=Experiment.PUBLIC_ACCESS_NONE) self.experiment2.save() # dataset1 belongs to experiment1 self.dataset1 = Dataset() self.dataset1.save() self.dataset1.experiments.add(self.experiment1) self.dataset1.save() # dataset2 belongs to experiment2 self.dataset2 = Dataset() self.dataset2.save() self.dataset2.experiments.add(self.experiment2) self.dataset2.save() # absolute path first filename1 = 'testfile.txt' filename2 = 'testfile.tiff' self.dest1 = abspath(join(settings.FILE_STORE_PATH, '%s/%s/' % (self.experiment1.id, self.dataset1.id))) self.dest2 = abspath(join(settings.FILE_STORE_PATH, '%s/%s/' % (self.experiment2.id, self.dataset2.id))) if not exists(self.dest1): makedirs(self.dest1) if not exists(self.dest2): makedirs(self.dest2) testfile1 = abspath(join(self.dest1, filename1)) f = open(testfile1, 'w') f.write("Hello World!\n") f.close() testfile2 = abspath(join(self.dest2, filename2)) _generate_test_image(testfile2) self.datafile1 = self._build_datafile( \ testfile1, filename1, self.dataset1, '%d/%d/%s' % (self.experiment1.id, self.dataset1.id, filename1)) self.datafile2 = self._build_datafile( \ testfile2, filename2, self.dataset2, '%d/%d/%s' % (self.experiment2.id, self.dataset2.id, filename2)) def _build_datafile(self, testfile, filename, dataset, url, protocol='', checksum=None, size=None, mimetype=''): filesize, sha512sum = get_size_and_sha512sum(testfile) datafile = Dataset_File(dataset=dataset, filename=filename, mimetype=mimetype, size=str(size if size != None else filesize), sha512sum=(checksum if checksum else sha512sum)) datafile.save() if urlparse.urlparse(url).scheme == '': location = Location.get_location('local') else: location = Location.get_location_for_url(url) if not location: location = Location.load_location({ 'name': filename, 'url': urlparse.urljoin(url, '.'), 'type': 'external', 'priority': 10, 'transfer_provider': 'local'}) replica = Replica(datafile=datafile, protocol=protocol, url=url, location=location) replica.verify() replica.save() return Dataset_File.objects.get(pk=datafile.pk) def tearDown(self): self.user.delete() self.experiment1.delete() self.experiment2.delete() rmtree(self.dest1) rmtree(self.dest2) def testView(self): client = Client() # check view of file1 response = client.get('/datafile/view/%i/' % self.datafile1.id) self.assertEqual(response['Content-Disposition'], 'inline; filename="%s"' % self.datafile1.filename) self.assertEqual(response.status_code, 200) self.assertEqual(response.content, 'Hello World!\n') # check view of file2 response = client.get('/datafile/view/%i/' % self.datafile2.id) # Should be forbidden self.assertEqual(response.status_code, 403) self.experiment2.public_access=Experiment.PUBLIC_ACCESS_FULL self.experiment2.save() # check view of file2 again response = client.get('/datafile/view/%i/' % self.datafile2.id) self.assertEqual(response.status_code, 200) # The following behaviour relies on ImageMagick if IMAGEMAGICK_AVAILABLE: # file2 should have a ".png" filename self.assertEqual(response['Content-Disposition'], 'inline; filename="%s"' % (self.datafile2.filename+'.png')) # file2 should be a PNG self.assertEqual(response['Content-Type'], 'image/png') png_signature = "\x89PNG\r\n\x1a\n" self.assertEqual(response.content[0:8], png_signature) else: # file2 should have a ".tiff" filename self.assertEqual(response['Content-Disposition'], 'inline; filename="%s"' % (self.datafile2.filename)) # file2 should be a TIFF self.assertEqual(response['Content-Type'], 'image/tiff') tiff_signature = "II\x2a\x00" self.assertEqual(response.content[0:4], tiff_signature) def _check_tar_file(self, content, rootdir, datafiles, simpleNames=False, noTxt=False): with NamedTemporaryFile('w') as tempfile: tempfile.write(content) tempfile.flush() if getsize(tempfile.name) > 0: expect(is_tarfile(tempfile.name)).to_be_truthy() try: tf = TarFile(tempfile.name, 'r') self._check_names(datafiles, tf.getnames(), rootdir, simpleNames, noTxt) finally: tf.close() else: self._check_names(datafiles, [], rootdir, simpleNames, noTxt) def _check_zip_file(self, content, rootdir, datafiles, simpleNames=False, noTxt=False): with NamedTemporaryFile('w') as tempfile: tempfile.write(content) tempfile.flush() # It should be a zip file expect(is_zipfile(tempfile.name)).to_be_truthy() try: zf = ZipFile(tempfile.name, 'r') self._check_names(datafiles, zf.namelist(), rootdir, simpleNames, noTxt) finally: zf.close() def _check_names(self, datafiles, names, rootdir, simpleNames, noTxt): # SimpleNames says if we expect basenames or pathnames # NoTxt says if we expect '.txt' files to be filtered out if not noTxt: expect(len(names)).to_equal(len(datafiles)) for df in datafiles: if simpleNames: filename = df.filename else: filename = join(rootdir, str(df.dataset.id), df.filename) expect(filename in names).to_be( not (noTxt and filename.endswith('.txt'))) def testDownload(self): client = Client() # check download for experiment1 response = client.get('/download/experiment/%i/zip/' % \ self.experiment1.id) self.assertEqual(response['Content-Disposition'], 'attachment; filename="experiment%s-complete.zip"' % self.experiment1.id) self.assertEqual(response.status_code, 200) self._check_zip_file( response.content, str(self.experiment1.id), reduce(lambda x, y: x + y, [ds.dataset_file_set.all() \ for ds in self.experiment1.datasets.all()])) # check download for experiment1 as tar response = client.get('/download/experiment/%i/tar/' % \ self.experiment1.id) self.assertEqual(response['Content-Disposition'], 'attachment; filename="experiment%s-complete.tar"' % self.experiment1.id) self.assertEqual(response.status_code, 200) self._check_tar_file( response.content, str(self.experiment1.id), reduce(lambda x, y: x + y, [ds.dataset_file_set.all() \ for ds in self.experiment1.datasets.all()])) # check download of file1 response = client.get('/download/datafile/%i/' % self.datafile1.id) self.assertEqual(response['Content-Disposition'], 'attachment; filename="%s"' % self.datafile1.filename) self.assertEqual(response.status_code, 200) self.assertEqual(response.content, 'Hello World!\n') # requesting file2 should be forbidden... response = client.get('/download/datafile/%i/' % self.datafile2.id) self.assertEqual(response.status_code, 403) # check dataset1 download response = client.post('/download/datafiles/', {'expid': self.experiment1.id, 'dataset': [self.dataset1.id], 'datafile': []}) self.assertEqual(response.status_code, 200) self._check_zip_file(response.content, 'datasets', self.dataset1.dataset_file_set.all()) # check dataset1 download as tar response = client.post('/download/datafiles/', {'expid': self.experiment1.id, 'dataset': [self.dataset1.id], 'datafile': [], 'comptype': 'tar'}) self.assertEqual(response.status_code, 200) self._check_tar_file(response.content, 'datasets', self.dataset1.dataset_file_set.all()) # check dataset2 download response = client.post('/download/datafiles/', {'expid': self.experiment2.id, 'dataset': [self.dataset2.id], 'datafile': []}) self.assertEqual(response.status_code, 403) # check datafile1 download via POST response = client.post('/download/datafiles/', {'expid': self.experiment1.id, 'dataset': [], 'datafile': [self.datafile1.id]}) self.assertEqual(response.status_code, 200) self._check_zip_file(response.content, 'datasets', [self.datafile1]) # check datafile2 download via POST response = client.post('/download/datafiles/', {'expid': self.experiment2.id, 'dataset': [], 'datafile': [self.datafile2.id]}) self.assertEqual(response.status_code, 403) # Check datafile2 download with second experiment to "metadata only" self.experiment2.public_access=Experiment.PUBLIC_ACCESS_METADATA self.experiment2.save() response = client.get('/download/datafile/%i/' % self.datafile2.id) # Metadata-only means "no file access"! self.assertEqual(response.status_code, 403) # Check datafile2 download with second experiment to public self.experiment2.public_access=Experiment.PUBLIC_ACCESS_FULL self.experiment2.save() response = client.get('/download/datafile/%i/' % self.datafile2.id) self.assertEqual(response.status_code, 200) # This should be a TIFF (which often starts with "II\x2a\x00") self.assertEqual(response['Content-Type'], 'image/tiff') self.assertEqual(response.content[0:4], "II\x2a\x00") # check experiment zip download with alternative organization response = client.get('/download/experiment/%i/zip/test/' % \ self.experiment1.id) self.assertEqual(response.status_code, 200) self.assertEqual(response['Content-Disposition'], 'attachment; filename="experiment%s-complete.zip"' % self.experiment1.id) self._check_zip_file( response.content, str(self.experiment1.id), reduce(lambda x, y: x + y, [ds.dataset_file_set.all() \ for ds in self.experiment1.datasets.all()]), simpleNames=True) # check experiment tar download with alternative organization response = client.get('/download/experiment/%i/tar/test/' % \ self.experiment1.id) self.assertEqual(response.status_code, 200) self.assertEqual(response['Content-Disposition'], 'attachment; filename="experiment%s-complete.tar"' % self.experiment1.id) self._check_tar_file( response.content, str(self.experiment1.id), reduce(lambda x, y: x + y, [ds.dataset_file_set.all() \ for ds in self.experiment1.datasets.all()]), simpleNames=True) # check experiment1 download with '.txt' filtered out (none left) response = client.get('/download/experiment/%i/tar/test2/' % \ self.experiment1.id) self.assertEqual(response.status_code, 400) # check experiment2 download with '.txt' filtered out response = client.get('/download/experiment/%i/tar/test2/' % \ self.experiment2.id) self.assertEqual(response.status_code, 200) self.assertEqual(response['Content-Disposition'], 'attachment; filename="experiment%s-complete.tar"' % self.experiment2.id) self._check_tar_file( response.content, str(self.experiment2.id), reduce(lambda x, y: x + y, [ds.dataset_file_set.all() \ for ds in self.experiment2.datasets.all()]), simpleNames=True, noTxt=True) # check dataset1 download response = client.post('/download/datafiles/', {'expid': self.experiment1.id, 'dataset': [self.dataset1.id], 'datafile': [], 'comptype': 'zip', 'organization': 'test'}) self.assertEqual(response.status_code, 200) self._check_zip_file(response.content, 'datasets', self.dataset1.dataset_file_set.all(), simpleNames=True) def testDatasetFile(self): # check registered text file for physical file meta information df = Dataset_File.objects.get(pk=self.datafile1.id) try: from magic import Magic self.assertEqual(df.mimetype, 'text/plain; charset=us-ascii') except: # XXX Test disabled because lib magic can't be loaded pass self.assertEqual(df.size, str(13)) self.assertEqual(df.md5sum, '8ddd8be4b179a529afa5f2ffae4b9858') # Now check we can calculate checksums and infer the mime type # for a JPG file. filename = abspath(join(dirname(__file__), '../static/images/ands-logo-hi-res.jpg')) dataset = Dataset.objects.get(pk=self.dataset1.id) pdf1 = self._build_datafile(filename, basename(filename), dataset, 'file://%s' % filename, protocol='file') self.assertEqual(pdf1.get_preferred_replica().verify(), True) pdf1 = Dataset_File.objects.get(pk=pdf1.pk) try: from magic import Magic self.assertEqual(pdf1.mimetype, 'image/jpeg') except: # XXX Test disabled because lib magic can't be loaded pass self.assertEqual(pdf1.size, str(14232)) self.assertEqual(pdf1.md5sum, 'c450d5126ffe3d14643815204daf1bfb') # Now check that we can override the physical file meta information # We are setting size/checksums that don't match the actual file, so # the pdf2 = self._build_datafile( filename, filename, dataset, 'file://%s' % filename, protocol='file', mimetype='application/vnd.openxmlformats-officedocument.presentationml.presentation', size=0, # Empty string always has the same hash checksum='cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e') self.assertEqual(pdf2.size, str(0)) self.assertEqual(pdf2.md5sum, '') self.assertEqual(pdf2.get_preferred_replica().verify(), False) pdf2 = Dataset_File.objects.get(pk=pdf2.pk) try: from magic import Magic self.assertEqual(pdf2.mimetype, 'application/vnd.openxmlformats-officedocument.presentationml.presentation') except: # XXX Test disabled because lib magic can't be loaded pass self.assertEqual(pdf2.size, str(0)) self.assertEqual(pdf2.md5sum, '') pdf2.mimetype = '' pdf2.save() pdf2.get_preferred_replica().save() pdf2 = Dataset_File.objects.get(pk=pdf2.pk) try: from magic import Magic self.assertEqual(pdf2.mimetype, 'application/pdf') except: # XXX Test disabled because lib magic can't be loaded pass
class MyTardisHSMUtilsTestCase(TestCase): """Test cases for the MyTardisHSM utils module""" def setUp(self): """Setup test fixtures if needed.""" self.user = User.objects.create_user("doctor", '', "pwd") self.exp = Experiment(title="Wonderful", institution_name="Monash University", created_by=self.user) self.exp.save() group = Group(name="Group1") group.save() facility = Facility(name="Test Facility", manager_group=group) facility.save() inst = Instrument(name="Test Instrument1", facility=facility) inst.save() self.dataset = Dataset(description="Dataset1", instrument=inst) self.dataset.save() storage_classes = getattr(settings, "HSM_STORAGE_CLASSES", DEFAULT_HSM_CLASSES) self.sbox1 = StorageBox(name="SBOX1", django_storage_class=storage_classes[0], status='online', max_size=256) self.sbox1.save() sbox1_attr = StorageBoxAttribute(storage_box=self.sbox1, key='type', value=StorageBox.DISK) sbox1_attr.save() sbox1_loc_opt = StorageBoxOption(storage_box=self.sbox1, key="location", value="/dummy/path") sbox1_loc_opt.save() self.sbox2 = StorageBox( name="SBOX2", django_storage_class="any.non.disk.StorageSystem", status='offline', max_size=256) self.sbox2.save() sbox2_attr = StorageBoxAttribute(storage_box=self.sbox2, key='type', value=StorageBox.TAPE) sbox2_attr.save() self.df1 = DataFile(dataset=self.dataset, filename="test_df.jpg") self.df1.save() self.dfo1 = DataFileObject(datafile=self.df1, storage_box=self.sbox1, uri="stream/test.jpg") self.dfo1.save() def tearDown(self): """Remove stuff""" def test_000_lock_datafile(self): """We should be able to lock a datafile to prevent concurrent access""" with DatafileLock(self.df1, "dummy_oid1") as lock1: if lock1: with DatafileLock(self.df1, "dummy_oid2") as lock2: self.assertTrue(lock1) self.assertFalse(lock2) def test_001_datafile_lock_expiry(self): """A datafile lock should not release until the expiry has been reached""" with DatafileLock(self.df1, "dummy_oid1", expires=2) as lock1: self.assertTrue(lock1) # If we retry lock right away, lock acquisition should fail because # expiry hasn't been reached with DatafileLock(self.df1, "dummy_oid1_1") as lock1: self.assertFalse(lock1) # wait 2s for lock to release time.sleep(2) # If we retry acquiring the lock now it should succeed with DatafileLock(self.df1, "dummy_oid1_2") as lock1: self.assertTrue(lock1)
class DownloadTestCase(TestCase): def setUp(self): # create a test user self.user = User.objects.create_user(username='******', email='', password='******') # create a public experiment self.experiment1 = Experiment(title='Experiment 1', created_by=self.user, public=True) self.experiment1.save() # create a non-public experiment self.experiment2 = Experiment(title='Experiment 2', created_by=self.user, public=False) self.experiment2.save() # dataset1 belongs to experiment1 self.dataset1 = Dataset(experiment=self.experiment1) self.dataset1.save() # dataset2 belongs to experiment2 self.dataset2 = Dataset(experiment=self.experiment2) self.dataset2.save() # absolute path first filename = 'testfile.txt' self.dest1 = abspath(join(settings.FILE_STORE_PATH, '%s/%s/' % (self.experiment1.id, self.dataset1.id))) self.dest2 = abspath(join(settings.FILE_STORE_PATH, '%s/%s/' % (self.experiment2.id, self.dataset2.id))) if not exists(self.dest1): makedirs(self.dest1) if not exists(self.dest2): makedirs(self.dest2) testfile1 = abspath(join(self.dest1, filename)) f = open(testfile1, 'w') f.write("Hello World!\n") f.close() testfile2 = abspath(join(self.dest2, filename)) f = open(testfile2, 'w') f.write("Hello World!\n") f.close() self.dataset_file1 = Dataset_File(dataset=self.dataset1, filename=filename, protocol='tardis', url='tardis://%s' % filename) self.dataset_file1.save() self.dataset_file2 = Dataset_File(dataset=self.dataset2, filename=basename(filename), protocol='tardis', url='tardis://%s' % filename) self.dataset_file2.save() def tearDown(self): self.user.delete() self.experiment1.delete() self.experiment2.delete() rmtree(self.dest1) rmtree(self.dest2) def testDownload(self): client = Client() # check download for experiment1 response = client.get('/download/experiment/%i/zip/' % self.experiment1.id) self.assertEqual(response['Content-Disposition'], 'attachment; filename="experiment%s-complete.zip"' % self.experiment1.id) self.assertEqual(response.status_code, 200) # check download of file1 response = client.get('/download/datafile/%i/' % self.dataset_file1.id) self.assertEqual(response['Content-Disposition'], 'attachment; filename="%s"' % self.dataset_file2.filename) self.assertEqual(response.status_code, 200) self.assertEqual(response.content, 'Hello World!\n') # requesting file2 should be forbidden... response = client.get('/download/datafile/%i/' % self.dataset_file2.id) self.assertEqual(response.status_code, 403) # check dataset1 download response = client.post('/download/datafiles/', {'expid': self.experiment1.id, 'dataset': [self.dataset1.id], 'datafile': []}) self.assertEqual(response.status_code, 200) # check dataset2 download response = client.post('/download/datafiles/', {'expid': self.experiment2.id, 'dataset': [self.dataset2.id], 'datafile': []}) self.assertEqual(response.status_code, 403) # check datafile1 download via POST response = client.post('/download/datafiles/', {'expid': self.experiment1.id, 'dataset': [], 'datafile': [self.dataset_file1.id]}) self.assertEqual(response.status_code, 200) # check datafile2 download via POST response = client.post('/download/datafiles/', {'expid': self.experiment2.id, 'dataset': [], 'datafile': [self.dataset_file2.id]}) self.assertEqual(response.status_code, 403) def testDatasetFile(self): # check registered text file for physical file meta information df = Dataset_File.objects.get(pk=self.dataset_file1.id) try: from magic import Magic self.assertEqual(df.mimetype, 'text/plain; charset=us-ascii') except: # XXX Test disabled becuse lib magic can't be loaded pass self.assertEqual(df.size, str(13)) self.assertEqual(df.md5sum, '8ddd8be4b179a529afa5f2ffae4b9858') # now check a JPG file filename = join(abspath(dirname(__file__)), '../static/images/ands-logo-hi-res.jpg') dataset = Dataset.objects.get(pk=self.dataset1.id) pdf1 = Dataset_File(dataset=dataset, filename=basename(filename), url='file://%s' % filename, protocol='file') pdf1.save() try: from magic import Magic self.assertEqual(pdf1.mimetype, 'image/jpeg') except: # XXX Test disabled becuse lib magic can't be loaded pass self.assertEqual(pdf1.size, str(14232)) self.assertEqual(pdf1.md5sum, 'c450d5126ffe3d14643815204daf1bfb') # now check that we can override the physical file meta information pdf2 = Dataset_File(dataset=dataset, filename=basename(filename), url='file://%s' % filename, protocol='file', mimetype='application/vnd.openxmlformats-officedocument.presentationml.presentation', size=str(0), md5sum='md5sum') pdf2.save() try: from magic import Magic self.assertEqual(pdf2.mimetype, 'application/vnd.openxmlformats-officedocument.presentationml.presentation') except: # XXX Test disabled becuse lib magic can't be loaded pass self.assertEqual(pdf2.size, str(0)) self.assertEqual(pdf2.md5sum, 'md5sum') pdf2.mimetype = '' pdf2.save() try: from magic import Magic self.assertEqual(pdf2.mimetype, 'application/pdf') except: # XXX Test disabled becuse lib magic can't be loaded pass
class MyTardisHSMTFiltersTestCase(TestCase): """Testing MyTardis HSM App filters""" def setUp(self): """Setup test fixtures if needed.""" self.user = User.objects.create_user("doctor", '', "pwd") self.exp = Experiment(title="Wonderful", institution_name="Monash University", created_by=self.user) self.exp.save() group = Group(name="Group1") group.save() facility = Facility(name="Test Facility", manager_group=group) facility.save() self.inst = Instrument(name="Test Instrument1", facility=facility) self.inst.save() self.dataset = Dataset(description="Dataset1", instrument=self.inst) self.dataset.save() self.dataset.experiments.add(self.exp) storage_classes = getattr(settings, "HSM_STORAGE_CLASSES", DEFAULT_HSM_CLASSES) self.sbox1 = StorageBox(name="SBOX1", django_storage_class=storage_classes[0], status='online', max_size=256) self.sbox1.save() sbox1_attr = StorageBoxAttribute(storage_box=self.sbox1, key='type', value=StorageBox.DISK) sbox1_attr.save() sbox1_loc_opt = StorageBoxOption(storage_box=self.sbox1, key="location", value=tempfile.gettempdir()) sbox1_loc_opt.save() self.sbox2 = StorageBox( name="SBOX2", django_storage_class="any.non.disk.StorageSystem", status='offline', max_size=256) self.sbox2.save() sbox2_attr = StorageBoxAttribute(storage_box=self.sbox2, key='type', value=StorageBox.TAPE) sbox2_attr.save() @mock.patch("os.stat") def test_001_create_df_status(self, mock_stat): """When a new datafile record is verified, metadata for it's online/offline status should be created and populated with the current online status""" mock_stat.return_value = Stats(st_size=10000, st_blocks=100, st_mtime=datetime.now()) temp = tempfile.NamedTemporaryFile(dir=tempfile.gettempdir()) temp_name = os.path.basename(temp.name) df2 = DataFile(dataset=self.dataset, filename=temp_name) df2.save() dfo2 = DataFileObject(datafile=df2, storage_box=self.sbox1, uri=temp_name) dfo2.save() df2.verify() param_name = ParameterName.objects.get( schema__namespace=HSM_DATAFILE_NAMESPACE, name="online") paramset = DatafileParameterSet.objects.get( schema__namespace=HSM_DATAFILE_NAMESPACE, datafile=df2) param = DatafileParameter.objects.get(parameterset=paramset, name=param_name) self.assertEquals(param.string_value, "True") temp.close() @mock.patch("os.stat") def test_002_no_duplicate_params(self, mock_stat): """Datafile should only ever have one online param""" mock_stat.return_value = Stats(st_size=10000, st_blocks=100, st_mtime=datetime.now()) df1 = DataFile(dataset=self.dataset, filename="test_df.jpg") df1.save() dfo1 = DataFileObject(datafile=df1, storage_box=self.sbox1, uri="stream/test.jpg", verified=True) dfo1.save() df1.verify() param_name = ParameterName.objects.get( schema__namespace=HSM_DATAFILE_NAMESPACE, name="online") paramset = DatafileParameterSet.objects.get( schema__namespace=HSM_DATAFILE_NAMESPACE, datafile=df1) params = DatafileParameter.objects.filter(parameterset=paramset, name=param_name) self.assertEquals(params.count(), 1) self.assertRaises(OnlineParamExistsError, create_df_status(df1, HSM_DATAFILE_NAMESPACE, 500)) params = DatafileParameter.objects.filter(parameterset=paramset, name=param_name) self.assertEquals(params.count(), 1) @mock.patch("os.stat") def test_003_offline_dataset(self, mock_stat): """A dataset should be offline if any datafiles are offline""" mock_stat.return_value = Stats(st_size=10000, st_blocks=0, st_mtime=datetime.now()) ds = Dataset(description="Dataset2", instrument=self.inst) ds.save() df2 = DataFile(dataset=ds, filename="test_file.jpg") df2.save() dfo2 = DataFileObject(datafile=df2, storage_box=self.sbox1, uri=df2.filename) dfo2.save() df2.verify() self.assertFalse(dataset_online(ds)) @mock.patch("os.stat") def test_004_offline_experiment(self, mock_stat): """An experiment should be offline if any datafiles are offline""" mock_stat.return_value = Stats(st_size=10000, st_blocks=0, st_mtime=datetime.now()) ds = Dataset(description="Dataset2", instrument=self.inst) ds.save() ds.experiments.add(self.exp) df2 = DataFile(dataset=ds, filename="test_file.jpg") df2.save() dfo2 = DataFileObject(datafile=df2, storage_box=self.sbox1, uri=df2.filename) dfo2.save() df2.verify() self.assertFalse(experiment_online(self.exp))
class ParameterSetManagerTestCase(TestCase): def setUp(self): from django.contrib.auth.models import User from tempfile import mkdtemp user = '******' pwd = 'secret' email = '' self.user = User.objects.create_user(user, email, pwd) self.test_dir = mkdtemp() self.exp = Experiment(title='test exp1', institution_name='monash', created_by=self.user) self.exp.save() self.dataset = Dataset(description="dataset description...") self.dataset.save() self.dataset.experiments.add(self.exp) self.dataset.save() self.datafile = DataFile(dataset=self.dataset, filename="testfile.txt", size="42", md5sum='bogus') self.datafile.save() self.dfo = DataFileObject( datafile=self.datafile, storage_box=self.datafile.get_default_storage_box(), uri="1/testfile.txt") self.dfo.save() self.schema = Schema( namespace="http://localhost/psmtest/df/", name="Parameter Set Manager", type=3) self.schema.save() self.parametername1 = ParameterName( schema=self.schema, name="parameter1", full_name="Parameter 1") self.parametername1.save() self.parametername2 = ParameterName( schema=self.schema, name="parameter2", full_name="Parameter 2", data_type=ParameterName.NUMERIC) self.parametername2.save() self.parametername3 = ParameterName( schema=self.schema, name="parameter3", full_name="Parameter 3", data_type=ParameterName.DATETIME) self.parametername3.save() self.datafileparameterset = DatafileParameterSet( schema=self.schema, datafile=self.datafile) self.datafileparameterset.save() self.datafileparameter1 = DatafileParameter( parameterset=self.datafileparameterset, name=self.parametername1, string_value="test1") self.datafileparameter1.save() self.datafileparameter2 = DatafileParameter( parameterset=self.datafileparameterset, name=self.parametername2, numerical_value=2) self.datafileparameter2.save() # Create a ParameterName and Parameter of type LINK to an experiment self.parametername_exp_link = ParameterName( schema=self.schema, name="exp_link", full_name="This parameter is a experiment LINK", data_type=ParameterName.LINK) self.parametername_exp_link.save() self.exp_link_param = DatafileParameter( parameterset=self.datafileparameterset, name=self.parametername_exp_link) exp_url = self.exp.get_absolute_url() # /experiment/view/1/ self.exp_link_param.set_value(exp_url) self.exp_link_param.save() # Create a ParameterName and Parameter of type LINK to a dataset self.parametername_dataset_link = ParameterName( schema=self.schema, name="dataset_link", full_name="This parameter is a dataset LINK", data_type=ParameterName.LINK) self.parametername_dataset_link.save() self.dataset_link_param = DatafileParameter( parameterset=self.datafileparameterset, name=self.parametername_dataset_link) dataset_url = self.dataset.get_absolute_url() # /dataset/1/ self.dataset_link_param.set_value(dataset_url) self.dataset_link_param.save() # Create a ParameterName type LINK to an unresolvable (non-URL) # free-text value self.parametername_unresolvable_link = ParameterName( schema=self.schema, name="freetext_link", full_name="This parameter is a non-URL LINK", data_type=ParameterName.LINK) self.parametername_unresolvable_link.save() def tearDown(self): self.exp.delete() self.user.delete() self.parametername1.delete() self.parametername2.delete() self.parametername3.delete() self.parametername_exp_link.delete() self.parametername_dataset_link.delete() self.parametername_unresolvable_link.delete() self.schema.delete() def test_existing_parameterset(self): psm = ParameterSetManager(parameterset=self.datafileparameterset) self.assertTrue(psm.get_schema().namespace == "http://localhost/psmtest/df/") self.assertTrue(psm.get_param("parameter1").string_value == "test1") self.assertTrue(psm.get_param("parameter2", True) == 2) def test_new_parameterset(self): psm = ParameterSetManager(parentObject=self.datafile, schema="http://localhost/psmtest/df2/") self.assertTrue(psm.get_schema().namespace == "http://localhost/psmtest/df2/") psm.set_param("newparam1", "test3", "New Parameter 1") self.assertTrue(psm.get_param("newparam1").string_value == "test3") self.assertTrue(psm.get_param("newparam1").name.full_name == "New Parameter 1") psm.new_param("newparam1", "test4") self.assertTrue(len(psm.get_params("newparam1", True)) == 2) psm.set_param_list("newparam2", ("a", "b", "c", "d")) self.assertTrue(len(psm.get_params("newparam2")) == 4) psm.set_params_from_dict( {"newparam2": "test5", "newparam3": 3}) self.assertTrue(psm.get_param("newparam2", True) == "test5") # the newparam3 gets created and '3' is set to a string_value # since once cannot assume that an initial numeric value # will imply continuing numeric type for this new param self.assertTrue(psm.get_param("newparam3").string_value == '3') psm.delete_params("newparam1") self.assertTrue(len(psm.get_params("newparam1", True)) == 0) def test_link_parameter_type(self): """ Test that Parameter.link_gfk (GenericForeignKey) is correctly assigned after using Parameter.set_value(some_url) for a LINK Parameter. """ psm = ParameterSetManager(parameterset=self.datafileparameterset) # Check link to experiment exp_url = self.exp.get_absolute_url() # /experiment/view/1/ self.assertTrue(psm.get_param("exp_link").string_value == exp_url) self.assertTrue(psm.get_param("exp_link").link_id == self.exp.id) exp_ct = ContentType.objects.get(model__iexact="experiment") self.assertTrue(psm.get_param("exp_link").link_ct == exp_ct) self.assertTrue(psm.get_param("exp_link").link_gfk == self.exp) # Check link to dataset dataset_url = self.dataset.get_absolute_url() # /dataset/1/ self.assertTrue(psm.get_param("dataset_link").string_value == dataset_url) self.assertTrue(psm.get_param("dataset_link").link_id == self.dataset.id) dataset_ct = ContentType.objects.get(model__iexact="dataset") self.assertTrue(psm.get_param("dataset_link").link_ct == dataset_ct) self.assertTrue(psm.get_param("dataset_link").link_gfk == self.dataset) def test_link_parameter_type_extra(self): # make a second ParameterSet for testing some variations # in URL values self.datafileparameterset2 = DatafileParameterSet( schema=self.schema, datafile=self.datafile) self.datafileparameterset2.save() psm = ParameterSetManager(parameterset=self.datafileparameterset2) self.dataset_link_param2 = DatafileParameter( parameterset=self.datafileparameterset2, name=self.parametername_dataset_link) # /dataset/1 - no trailing slash dataset_url = self.dataset.get_absolute_url() self.dataset_link_param2.set_value(dataset_url) self.dataset_link_param2.save() # Check link_id/link_ct/link_gfk to dataset self.assertTrue(psm.get_param("dataset_link").link_id == self.dataset.id) dataset_ct = ContentType.objects.get(model__iexact="dataset") self.assertTrue(psm.get_param("dataset_link").link_ct == dataset_ct) self.assertTrue(psm.get_param("dataset_link").link_gfk == self.dataset) # Test links of the form /api/v1/experiment/<experiment_id>/ self.exp_link_param2 = DatafileParameter( parameterset=self.datafileparameterset2, name=self.parametername_exp_link) exp_url = '/api/v1/experiment/%s/' % self.exp.id self.exp_link_param2.set_value(exp_url) self.exp_link_param2.save() # Check link_id/link_ct/link_gfk to experiment self.assertTrue(psm.get_param("exp_link").link_id == self.exp.id) exp_ct = ContentType.objects.get(model__iexact="experiment") self.assertTrue(psm.get_param("exp_link").link_ct == exp_ct) self.assertTrue(psm.get_param("exp_link").link_gfk == self.exp) def test_unresolvable_link_parameter(self): """ Test that LINK Parameters that can't be resolved to a model (including non-URL values) still work. """ self.datafileparameterset3 = DatafileParameterSet( schema=self.schema, datafile=self.datafile) self.datafileparameterset3.save() psm = ParameterSetManager(parameterset=self.datafileparameterset3) # Create a Parameter of type LINK to an unresolvable (non-URL) # free-text value self.freetext_link_param = DatafileParameter( parameterset=self.datafileparameterset3, name=self.parametername_unresolvable_link) self.assertRaises(SuspiciousOperation, lambda: self.freetext_link_param.set_value( "FREETEXT_ID_123")) def test_tz_naive_date_handling(self): """ Ensure that dates are handling in a timezone-aware way. """ psm = ParameterSetManager(parameterset=self.datafileparameterset) psm.new_param("parameter3", str(datetime(1970, 01, 01, 10, 0, 0))) expect(psm.get_param("parameter3", True))\ .to_equal(datetime(1970, 01, 01, 0, 0, 0, tzinfo=pytz.utc)) def test_tz_aware_date_handling(self): """ Ensure that dates are handling in a timezone-aware way. """ psm = ParameterSetManager(parameterset=self.datafileparameterset) psm.new_param("parameter3", '1970-01-01T08:00:00+08:00') expect(psm.get_param("parameter3", True))\ .to_equal(datetime(1970, 01, 01, 0, 0, 0, tzinfo=pytz.utc))
class Task(ParameterSetManager): schema_name = "http://localhost/task/generic" namespace = schema_name dataset = None DPS = None myHPC = None doNotCopyParams = ['TaskStatus', 'jobscript', # many 'jobid', # many 'jobidstatus', # many ] def __init__(self, dataset=None, dataset_id=None, description="", experiment_id=None): """ instantiate new task or existing task :param dataset: optional parameter to instanciate task from metadata, will be tested for completeness and copied into new task if complete :type dataset: Dataset """ if dataset: self.dataset = dataset elif dataset_id: self.dataset = Dataset.objects.get(pk=dataset_id) else: if description == "": raise TypeError("No description given") if not experiment_id: raise TypeError("No experiment id given") self.dataset = Dataset() self.dataset.experiment_id = experiment_id self.dataset.description = description self.dataset.save() try: thisparameterset = DatasetParameterSet.objects.get( schema=self.get_schema(), dataset=self.dataset) except ObjectDoesNotExist: thisparameterset = DatasetParameterSet( schema=self.get_schema(), dataset=self.dataset) thisparameterset.save() super(Task, self).__init__(parameterset=thisparameterset) def get_status(self, value=False): try: return self.get_param("TaskStatus", value) except: return None def set_status(self, status): current_status = self.get_status(value=True) if current_status != status: self.set_param("TaskStatus", status, "Status of task") def get_files(self): return Dataset_File.objects.filter(dataset=self.dataset) def get_by_value(self, value): try: par = self.parameters.get(string_value=value) except ObjectDoesNotExist: try: par = self.parameters.get(numerical_value=value) except (ObjectDoesNotExist, ValueError): return None return par def parseResults(self): """ stub, to be overridden by subclass if needed """ pass def sendMail(self, toName, toAddress, returnURI, type="JobComplete"): from django.core.mail import send_mail subject = "Your Task %s is complete" % self.dataset.description message = "Dear %s,\n" % toName message += "\nYour job %s is complete " % self.dataset.description message += "and the results are stored in myTardis.\n" message += "HOSTNAME\n" message += "\nBest regards,\nYour myTardis\n" send_mail(subject, message, '*****@*****.**', [toAddress]) @classmethod def getTaskList(cls, experiment_id, status="any"): """ Get list of all tasks or specify the type as string :param experiment: the experiment that is being searched for tasks :type experiment: Experiment :param taskclass: the subclass of Task :type type: string yields DatasetParameterSet """ DPSs = DatasetParameterSet.objects.filter( schema__namespace__startswith=cls.schema_name, dataset__experiment__pk=experiment_id) tasklist = [cls(dataset=dps.dataset) for dps in DPSs] if status == "any": return tasklist filteredlist = [] for thistask in tasklist: try: if thistask.get_status(value=True) == status: filteredlist.append(thistask) except ObjectDoesNotExist: continue return filteredlist @classmethod def clone(cls, oldInstance, newDescription, username): newInstance = cls(description=newDescription, experiment_id=oldInstance.dataset.experiment.id) for param in oldInstance.parameters: if param.name.name not in cls.doNotCopyParams: if param.name.isNumeric(): value = param.numerical_value else: value = param.string_value newInstance.new_param(param.name.name, value) import shutil import os for filename in oldInstance.get_params("uploaded_file", value=True): if filename[-8:] != ".jobfile": thisfile = Dataset_File.objects.get( dataset=oldInstance.dataset, filename=filename) shutil.copy(thisfile.get_absolute_filepath(), get_full_staging_path(username)) newfileurl = os.path.join(get_full_staging_path(username), filename) newDatafile = Dataset_File( dataset=newInstance.dataset, url=newfileurl, protocol="staging", mimetype=thisfile.mimetype, ) newDatafile.save() return newInstance
class DownloadTestCase(TestCase): def setUp(self): # create a test user self.user = User.objects.create_user(username='******', email='', password='******') # create a public experiment self.experiment1 = Experiment(title='Experiment 1', created_by=self.user, public_access=Experiment.PUBLIC_ACCESS_FULL) self.experiment1.save() # create a non-public experiment self.experiment2 = Experiment(title='Experiment 2', created_by=self.user, public_access=Experiment.PUBLIC_ACCESS_NONE) self.experiment2.save() # dataset1 belongs to experiment1 self.dataset1 = Dataset() self.dataset1.save() self.dataset1.experiments.add(self.experiment1) self.dataset1.save() # dataset2 belongs to experiment2 self.dataset2 = Dataset() self.dataset2.save() self.dataset2.experiments.add(self.experiment2) self.dataset2.save() # absolute path first filename1 = 'testfile.txt' filename2 = 'testfile.tiff' self.dest1 = abspath(join(settings.FILE_STORE_PATH, '%s/%s/' % (self.experiment1.id, self.dataset1.id))) self.dest2 = abspath(join(settings.FILE_STORE_PATH, '%s/%s/' % (self.experiment2.id, self.dataset2.id))) if not exists(self.dest1): makedirs(self.dest1) if not exists(self.dest2): makedirs(self.dest2) testfile1 = abspath(join(self.dest1, filename1)) f = open(testfile1, 'w') f.write("Hello World!\n") f.close() testfile2 = abspath(join(self.dest2, filename2)) if IMAGEMAGICK_AVAILABLE: with Image(filename='logo:') as img: img.format = 'tiff' img.save(filename=testfile2) else: # Apparently ImageMagick isn't installed... # Write a "fake" TIFF file f = open(testfile2, 'w') f.write("II\x2a\x00") f.close() size, sha512sum = get_size_and_sha512sum(testfile1) self.dataset_file1 = Dataset_File(dataset=self.dataset1, filename=filename1, protocol='', size=size, sha512sum=sha512sum, url='%d/%d/%s' % (self.experiment1.id, self.dataset1.id, filename1)) self.dataset_file1.verify() self.dataset_file1.save() size, sha512sum = get_size_and_sha512sum(testfile2) self.dataset_file2 = Dataset_File(dataset=self.dataset2, filename=basename(filename2), protocol='', size=size, sha512sum=sha512sum, url='%d/%d/%s' % (self.experiment2.id, self.dataset2.id, filename2)) self.dataset_file2.verify() self.dataset_file2.save() def tearDown(self): self.user.delete() self.experiment1.delete() self.experiment2.delete() rmtree(self.dest1) rmtree(self.dest2) def testView(self): client = Client() # check view of file1 response = client.get('/datafile/view/%i/' % self.dataset_file1.id) self.assertEqual(response['Content-Disposition'], 'inline; filename="%s"' % self.dataset_file1.filename) self.assertEqual(response.status_code, 200) self.assertEqual(response.content, 'Hello World!\n') # check view of file2 response = client.get('/datafile/view/%i/' % self.dataset_file2.id) # Should be forbidden self.assertEqual(response.status_code, 403) self.experiment2.public_access=Experiment.PUBLIC_ACCESS_FULL self.experiment2.save() # check view of file2 again response = client.get('/datafile/view/%i/' % self.dataset_file2.id) self.assertEqual(response.status_code, 200) # The following behaviour relies on ImageMagick if IMAGEMAGICK_AVAILABLE: # file2 should have a ".png" filename self.assertEqual(response['Content-Disposition'], 'inline; filename="%s"' % (self.dataset_file2.filename+'.png')) # file2 should be a PNG self.assertEqual(response['Content-Type'], 'image/png') png_signature = "\x89PNG\r\n\x1a\n" self.assertEqual(response.content[0:8], png_signature) else: # file2 should have a ".tiff" filename self.assertEqual(response['Content-Disposition'], 'inline; filename="%s"' % (self.dataset_file2.filename)) # file2 should be a TIFF self.assertEqual(response['Content-Type'], 'image/tiff') tiff_signature = "II\x2a\x00" self.assertEqual(response.content[0:4], tiff_signature) def _check_tar_file(self, content, rootdir, datafiles): # It should be a zip file with NamedTemporaryFile('w') as tempfile: tempfile.write(content) tempfile.flush() with open(tempfile.name, 'r') as zipread: # It should be a zip file (all of which start with "PK") expect(zipread.read(2)).to_equal('PK') expect(is_zipfile(tempfile.name)).to_be_truthy() with ZipFile(tempfile.name, 'r') as zf: expect(len(zf.namelist())).to_equal(len(datafiles)) for df in datafiles: filename = join(rootdir, str(df.dataset.id), df.filename) expect(filename in zf.namelist()).to_be_truthy() def _check_zip_file(self, content, rootdir, datafiles): # It should be a zip file with NamedTemporaryFile('w') as tempfile: tempfile.write(content) tempfile.flush() with open(tempfile.name, 'r') as zipread: # It should be a zip file (all of which start with "PK") expect(zipread.read(2)).to_equal('PK') expect(is_zipfile(tempfile.name)).to_be_truthy() zf = ZipFile(tempfile.name, 'r') expect(len(zf.namelist())).to_equal(len(datafiles)) for df in datafiles: filename = join(rootdir, str(df.dataset.id), df.filename) expect(filename in zf.namelist()).to_be_truthy() zf.close() def testDownload(self): client = Client() # check download for experiment1 response = client.get('/download/experiment/%i/zip/' % self.experiment1.id) self.assertEqual(response['Content-Disposition'], 'attachment; filename="experiment%s-complete.zip"' % self.experiment1.id) self.assertEqual(response.status_code, 200) self._check_zip_file(response.content, str(self.experiment1.id), reduce(lambda x, y: x + y, [ds.dataset_file_set.all() \ for ds in self.experiment1.datasets.all()])) # check download of file1 response = client.get('/download/datafile/%i/' % self.dataset_file1.id) self.assertEqual(response['Content-Disposition'], 'attachment; filename="%s"' % self.dataset_file1.filename) self.assertEqual(response.status_code, 200) self.assertEqual(response.content, 'Hello World!\n') # requesting file2 should be forbidden... response = client.get('/download/datafile/%i/' % self.dataset_file2.id) self.assertEqual(response.status_code, 403) # check dataset1 download response = client.post('/download/datafiles/', {'expid': self.experiment1.id, 'dataset': [self.dataset1.id], 'datafile': []}) self.assertEqual(response.status_code, 200) self._check_zip_file(response.content, 'datasets', self.dataset1.dataset_file_set.all()) # check dataset2 download response = client.post('/download/datafiles/', {'expid': self.experiment2.id, 'dataset': [self.dataset2.id], 'datafile': []}) self.assertEqual(response.status_code, 403) # check datafile1 download via POST response = client.post('/download/datafiles/', {'expid': self.experiment1.id, 'dataset': [], 'datafile': [self.dataset_file1.id]}) self.assertEqual(response.status_code, 200) self._check_zip_file(response.content, 'datasets', [self.dataset_file1]) # check datafile2 download via POST response = client.post('/download/datafiles/', {'expid': self.experiment2.id, 'dataset': [], 'datafile': [self.dataset_file2.id]}) self.assertEqual(response.status_code, 403) # Check datafile2 download with second experiment to "metadata only" self.experiment2.public_access=Experiment.PUBLIC_ACCESS_METADATA self.experiment2.save() response = client.get('/download/datafile/%i/' % self.dataset_file2.id) # Metadata-only means "no file access"! self.assertEqual(response.status_code, 403) # Check datafile2 download with second experiment to public self.experiment2.public_access=Experiment.PUBLIC_ACCESS_FULL self.experiment2.save() response = client.get('/download/datafile/%i/' % self.dataset_file2.id) self.assertEqual(response.status_code, 200) # This should be a TIFF (which often starts with "II\x2a\x00") self.assertEqual(response['Content-Type'], 'image/tiff') self.assertEqual(response.content[0:4], "II\x2a\x00") def testDatasetFile(self): # check registered text file for physical file meta information df = Dataset_File.objects.get(pk=self.dataset_file1.id) try: from magic import Magic self.assertEqual(df.mimetype, 'text/plain; charset=us-ascii') except: # XXX Test disabled becuse lib magic can't be loaded pass self.assertEqual(df.size, str(13)) self.assertEqual(df.md5sum, '8ddd8be4b179a529afa5f2ffae4b9858') # now check a JPG file filename = abspath(join(dirname(__file__), '../static/images/ands-logo-hi-res.jpg')) dataset = Dataset.objects.get(pk=self.dataset1.id) size, sha512sum = get_size_and_sha512sum(filename) pdf1 = Dataset_File(dataset=dataset, filename=basename(filename), size=str(size), sha512sum=sha512sum, url='file://%s' % filename, protocol='file') pdf1.verify() pdf1.save() try: from magic import Magic self.assertEqual(pdf1.mimetype, 'image/jpeg') except: # XXX Test disabled becuse lib magic can't be loaded pass self.assertEqual(pdf1.size, str(14232)) self.assertEqual(pdf1.md5sum, 'c450d5126ffe3d14643815204daf1bfb') # now check that we can override the physical file meta information pdf2 = Dataset_File(dataset=dataset, filename=basename(filename), url='file://%s' % filename, protocol='file', mimetype='application/vnd.openxmlformats-officedocument.presentationml.presentation', size=str(0), # Empty string always has the same hash sha512sum='cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e') pdf2.save() try: from magic import Magic self.assertEqual(pdf2.mimetype, 'application/vnd.openxmlformats-officedocument.presentationml.presentation') except: # XXX Test disabled becuse lib magic can't be loaded pass self.assertEqual(pdf2.size, str(0)) self.assertEqual(pdf2.md5sum, '') pdf2.mimetype = '' pdf2.save() try: from magic import Magic self.assertEqual(pdf2.mimetype, 'application/pdf') except: # XXX Test disabled becuse lib magic can't be loaded pass
class SFTPTest(TestCase): def setUp(self): self.hostname = '127.0.0.1' self.username = '******' self.password = '******' email = '' self.user = User.objects.create_user(self.username, email, self.password) self.exp = Experiment(title='test exp1', institution_name='monash', created_by=self.user) self.exp.save() self.acl = ObjectACL(content_object=self.exp, pluginId='django_user', entityId=str(self.user.id), isOwner=True, canRead=True, canWrite=True, canDelete=True, aclOwnershipType=ObjectACL.OWNER_OWNED) self.acl.save() self.dataset = Dataset(description='test dataset1') self.dataset.save() self.dataset.experiments.set([self.exp]) self.dataset.save() def _build(dataset, filename, url): datafile_content = b"\n".join( [b'some data %d' % i for i in range(1000)]) filesize = len(datafile_content) datafile = DataFile(dataset=dataset, filename=filename, size=filesize) datafile.save() dfo = DataFileObject( datafile=datafile, storage_box=datafile.get_default_storage_box(), uri=url) dfo.file_object = BytesIO(datafile_content) dfo.save() return datafile saved_setting = settings.REQUIRE_DATAFILE_CHECKSUMS try: settings.REQUIRE_DATAFILE_CHECKSUMS = False _build(self.dataset, 'file.txt', 'path/file.txt') finally: settings.REQUIRE_DATAFILE_CHECKSUMS = saved_setting def test_sftp(self): path_mapper = make_mapper(settings.DEFAULT_PATH_MAPPER, rootdir=None) server = flexmock(user=self.user) sftp_interface = MyTSFTPServerInterface(server=server) sftp_interface.session_started() exp_sftp_folders = sftp_interface.list_folder('/home/%s/experiments/' % self.username) exp_sftp_folder_names = sorted( [sftp_folder.filename for sftp_folder in exp_sftp_folders]) exp_folder_names = sorted( [path_mapper(exp) for exp in Experiment.safe.all(self.user)]) self.assertEqual(exp_sftp_folder_names, exp_folder_names) ds_sftp_folders = sftp_interface.list_folder( '/home/%s/experiments/%s/' % (self.username, path_mapper(self.exp))) ds_sftp_folder_names = sorted( [sftp_folder.filename for sftp_folder in ds_sftp_folders]) self.assertEqual( ds_sftp_folder_names, ['00_all_files', path_mapper(self.dataset)]) sftp_files = sftp_interface.list_folder( '/home/%s/experiments/%s/%s/' % (self.username, path_mapper(self.exp), path_mapper(self.dataset))) sftp_filenames = sorted( [sftp_file.filename for sftp_file in sftp_files]) self.assertEqual(sftp_filenames, ['file.txt']) server_interface = MyTServerInterface() self.assertEqual( server_interface.check_auth_password(self.username, self.password), AUTH_SUCCESSFUL) # should fail if user is inactive self.user.is_active = False self.user.save() self.assertEqual( server_interface.check_auth_password(self.username, self.password), AUTH_FAILED) self.user.is_active = True self.user.save() def test_sftp_key_connect(self): server_interface = MyTServerInterface() pub_key_str = ( "AAAAB3NzaC1yc2EAAAADAQABAAAAgQCzvWE391K1pyBvePGpwDWMboSLIp" "5L5sMq+bXPPeJPSLOm9dnm8XexZOpeg14UpsYcmrkzVPeooaqz5PqtaHO46CdK11dS" "cs2a8PLnavGkJRf25/PDXxlHkiZXXbAfW+6t5aVJxSJ4Jt4FV0aDqMaaYxy4ikw6da" "BCkvug2OZQqQ==") priv_key_str = """-----BEGIN RSA PRIVATE KEY----- MIICXgIBAAKBgQCzvWE391K1pyBvePGpwDWMboSLIp5L5sMq+bXPPeJPSLOm9dnm 8XexZOpeg14UpsYcmrkzVPeooaqz5PqtaHO46CdK11dScs2a8PLnavGkJRf25/PD XxlHkiZXXbAfW+6t5aVJxSJ4Jt4FV0aDqMaaYxy4ikw6daBCkvug2OZQqQIDAQAB AoGASpK9XlIQD+wqafWdFpf3368O8QdI9CbnPNJkG3sKhWidmR0R7l6rEX/UOah5 hUn4km+jfWe4ZU/GGmNbmkznDdOWspDKs7eeYl7saeRzuX2CdTVvrdU7qmD5+JLk mXlWWd6rgRIfrFYXYeDVd8p6/kPR4SJe7dTTHuEKKIt9njECQQDhMqjyoNxftpl4 +mwQu0ZDLCZ4afDCGcsf73W3oSmqLyf401vQ6KAp/PmfxqGXY0ewGMzUJn9LFOyP WOGcDFglAkEAzFL/DI3SYmsvLMt6/vK4qwEwSiJU8byUBj3CL3eL0xjn895GXPzb 9CUMu0fz60Tn7UhbohynPLmQ2w6npbZ9NQJBAN+uujGFpl9LuFV6KCzWV4wRJoUk dYfWpvQpnfuvkPsBq+pzxhdTeQM7y5bwbUE509MOTyXKt1WUiwQ3fKDLgiECQQCb Z4zhSYT4ojlRQrqb6pSWS+Mkn5QoAJw9Wv+1BqHsvwa8rxSpaREKUpuqXgGhsdkM 2noHhO+V+jW4xx6vpWr5AkEAgHoSbQUR5uY8ib3N3mNowVi9NhvBN1FkwGStM9W8 QKHf8Ha+rOx3B7Dbljc+Xdpcn9VyRmDlSqzX9aCkr18mNg== -----END RSA PRIVATE KEY-----""" private_key = RSAKey.from_private_key(file_obj=StringIO(priv_key_str)) # Fail if public key not registered self.assertEqual( server_interface.check_auth_publickey(self.username, private_key), AUTH_FAILED) SFTPPublicKey.objects.create(user=self.user, name="TestKey", key_type="ssh-rsa", public_key=pub_key_str) # Succeed if public key is registered self.assertEqual( server_interface.check_auth_publickey(self.username, private_key), AUTH_SUCCESSFUL) # Should fail if user is inactive self.user.is_active = False self.user.save() self.assertEqual( server_interface.check_auth_publickey(self.username, private_key), AUTH_FAILED) self.user.is_active = True self.user.save() @patch('webpack_loader.loader.WebpackLoader.get_bundle') def test_sftp_dynamic_docs_experiment(self, mock_webpack_get_bundle): factory = RequestFactory() request = factory.get( '/sftp_access/?object_type=experiment&object_id=%s' % self.exp.id) request.user = self.user response = sftp_access(request) path_mapper = make_mapper(settings.DEFAULT_PATH_MAPPER, rootdir=None) self.assertIn( b"sftp://tardis_user1@testserver:2200" b"/home/tardis_user1/experiments/%s" % path_mapper(self.exp).encode(), response.content) self.assertNotEqual(mock_webpack_get_bundle.call_count, 0) @patch('webpack_loader.loader.WebpackLoader.get_bundle') def test_sftp_dynamic_docs_dataset(self, mock_webpack_get_bundle): factory = RequestFactory() request = factory.get( '/sftp_access/?object_type=dataset&object_id=%s' % self.dataset.id) request.user = self.user response = sftp_access(request) path_mapper = make_mapper(settings.DEFAULT_PATH_MAPPER, rootdir=None) self.assertIn( b"sftp://tardis_user1@testserver:2200" b"/home/tardis_user1/experiments/%s/%s" % (path_mapper( self.exp).encode(), path_mapper(self.dataset).encode()), response.content) self.assertNotEqual(mock_webpack_get_bundle.call_count, 0) def test_cybderduck_connection_window(self): factory = RequestFactory() request = factory.get('/sftp_access/cyberduck/connection.png') request.user = self.user response = cybderduck_connection_window(request) self.assertEqual(response.status_code, 200)
class UploadTestCase(TestCase): def setUp(self): from os import path, mkdir from tempfile import mkdtemp user = '******' pwd = 'secret' email = '' self.user = User.objects.create_user(user, email, pwd) self.userProfile = UserProfile(user=self.user) self.test_dir = mkdtemp() self.exp = Experiment(title='test exp1', institution_name='monash', created_by=self.user) self.exp.save() acl = ExperimentACL( pluginId=django_user, entityId=str(self.user.id), experiment=self.exp, canRead=True, isOwner=True, aclOwnershipType=ExperimentACL.OWNER_OWNED, ) acl.save() self.dataset = \ Dataset(description='dataset description...', experiment=self.exp) self.dataset.save() self.experiment_path = path.join(settings.FILE_STORE_PATH, str(self.dataset.experiment.id)) self.dataset_path = path.join(self.experiment_path, str(self.dataset.id)) if not path.exists(self.experiment_path): mkdir(self.experiment_path) if not path.exists(self.dataset_path): mkdir(self.dataset_path) # write test file self.filename = 'testfile.txt' self.f1 = open(path.join(self.test_dir, self.filename), 'w') self.f1.write('Test file 1') self.f1.close() self.f1_size = path.getsize(path.join(self.test_dir, self.filename)) self.f1 = open(path.join(self.test_dir, self.filename), 'r') def tearDown(self): from shutil import rmtree self.f1.close() rmtree(self.test_dir) rmtree(self.dataset_path) rmtree(self.experiment_path) self.exp.delete() def testFileUpload(self): from os import path c = Client() c.login(username='******', password='******') session_id = c.session.session_key response = c.post('/upload/' + str(self.dataset.id) + '/', {'Filedata': self.f1, 'session_id': session_id}) test_files_db = \ Dataset_File.objects.filter(dataset__id=self.dataset.id) self.assertTrue(path.exists(path.join(self.dataset_path, self.filename))) self.assertTrue(self.dataset.id == 1) self.assertTrue(test_files_db[0].url == 'tardis://testfile.txt') def testUploadComplete(self): from django.http import QueryDict, HttpRequest from tardis.tardis_portal.views import upload_complete data = [('filesUploaded', '1'), ('speed', 'really fast!'), ('allBytesLoaded', '2'), ('errorCount', '0')] post = QueryDict('&'.join(['%s=%s' % (k, v) for (k, v) in data])) request = HttpRequest() request.POST = post response = upload_complete(request) self.assertTrue('<p>Number: 1</p>' in response.content) self.assertTrue('<p>Errors: 0</p>' in response.content) self.assertTrue('<p>Bytes: 2</p>' in response.content) self.assertTrue('<p>Speed: really fast!</p>' in response.content)
class ViewTemplateContextsTest(TestCase): def setUp(self): """ setting up essential objects, copied from tests above """ Location.force_initialize() self.location = Location.get_location('local') user = '******' pwd = 'secret' email = '' self.user = User.objects.create_user(user, email, pwd) self.userProfile = UserProfile(user=self.user).save() self.exp = Experiment(title='test exp1', institution_name='monash', created_by=self.user) self.exp.save() self.acl = ObjectACL( pluginId=django_user, entityId=str(self.user.id), content_object=self.exp, canRead=True, isOwner=True, aclOwnershipType=ObjectACL.OWNER_OWNED, ) self.acl.save() self.dataset = Dataset(description='dataset description...') self.dataset.save() self.dataset.experiments.add(self.exp) self.dataset.save() self.dataset_file = Dataset_File(dataset=self.dataset, size=42, filename="foo", md5sum="junk") self.dataset_file.save() self.replica = Replica(datafile=self.dataset_file, url="http://foo", location=self.location, verified=False) self.replica.save() def tearDown(self): self.user.delete() self.exp.delete() self.dataset.delete() self.dataset_file.delete() self.acl.delete() def testExperimentView(self): """ test some template context parameters for an experiment view """ from tardis.tardis_portal.views import view_experiment from tardis.tardis_portal.shortcuts import render_response_index from django.http import HttpRequest from django.template import Context import sys # Default behavior views_module = flexmock(sys.modules['tardis.tardis_portal.views']) request = HttpRequest() request.user=self.user request.groups=[] context = {'organization': ['classic', 'test', 'test2'], 'default_organization': 'classic', 'default_format': 'zip', 'protocol': [['zip', '/download/experiment/1/zip/'], ['tar', '/download/experiment/1/tar/']]} views_module.should_call('render_response_index'). \ with_args(_AnyMatcher(), "tardis_portal/view_experiment.html", _ContextMatcher(context)) response = view_experiment(request, experiment_id=self.exp.id) self.assertEqual(response.status_code, 200) # Behavior with USER_AGENT_SENSING enabled and a request.user_agent saved_setting = getattr(settings, "USER_AGENT_SENSING", None) try: setattr(settings, "USER_AGENT_SENSING", True) request = HttpRequest() request.user=self.user request.groups=[] mock_agent = _MiniMock(os=_MiniMock(family="Macintosh")) setattr(request, 'user_agent', mock_agent); context = {'organization': ['classic', 'test', 'test2'], 'default_organization': 'classic', 'default_format': 'tar', 'protocol': [['tar', '/download/experiment/1/tar/']]} views_module.should_call('render_response_index'). \ with_args(_AnyMatcher(), "tardis_portal/view_experiment.html", _ContextMatcher(context)) response = view_experiment(request, experiment_id=self.exp.id) self.assertEqual(response.status_code, 200) finally: if saved_setting != None: setattr(settings, "USER_AGENT_SENSING", saved_setting) else: delattr(settings, "USER_AGENT_SENSING") def testDatasetView(self): """ test some context parameters for a dataset view """ from tardis.tardis_portal.views import view_dataset from tardis.tardis_portal.shortcuts import render_response_index from django.http import HttpRequest from django.template import Context import sys views_module = flexmock(sys.modules['tardis.tardis_portal.views']) request = HttpRequest() request.user=self.user request.groups=[] context = {'default_organization': 'classic', 'default_format': 'zip'} views_module.should_call('render_response_index'). \ with_args(_AnyMatcher(), "tardis_portal/view_dataset.html", _ContextMatcher(context)) response = view_dataset(request, dataset_id=self.dataset.id) self.assertEqual(response.status_code, 200) # Behavior with USER_AGENT_SENSING enabled and a request.user_agent saved_setting = getattr(settings, "USER_AGENT_SENSING", None) try: setattr(settings, "USER_AGENT_SENSING", True) request = HttpRequest() request.user=self.user request.groups=[] mock_agent = _MiniMock(os=_MiniMock(family="Macintosh")) setattr(request, 'user_agent', mock_agent); context = {'default_organization': 'classic', 'default_format': 'tar'} views_module.should_call('render_response_index'). \ with_args(_AnyMatcher(), "tardis_portal/view_dataset.html", _ContextMatcher(context)) response = view_dataset(request, dataset_id=self.dataset.id) self.assertEqual(response.status_code, 200) finally: if saved_setting != None: setattr(settings, "USER_AGENT_SENSING", saved_setting) else: delattr(settings, "USER_AGENT_SENSING")
class ContextualViewTest(TestCase): def setUp(self): """ setting up essential objects, copied from tests above """ user = '******' pwd = 'secret' email = '' self.user = User.objects.create_user(user, email, pwd) self.userProfile = UserProfile(user=self.user).save() self.exp = Experiment(title='test exp1', institution_name='monash', created_by=self.user) self.exp.save() self.acl = ObjectACL( pluginId=django_user, entityId=str(self.user.id), content_object=self.exp, canRead=True, isOwner=True, aclOwnershipType=ObjectACL.OWNER_OWNED, ) self.acl.save() self.dataset = Dataset(description='dataset description...') self.dataset.save() self.dataset.experiments.add(self.exp) self.dataset.save() self.dataset_file = Dataset_File(dataset=self.dataset, size=42, filename="foo", md5sum="junk") self.dataset_file.save() self.testschema = Schema(namespace="http://test.com/test/schema", name="Test View", type=Schema.DATAFILE, hidden=True) self.testschema.save() self.dfps = DatafileParameterSet(dataset_file=self.dataset_file, schema=self.testschema) self.dfps.save() def tearDown(self): self.user.delete() self.exp.delete() self.dataset.delete() self.dataset_file.delete() self.testschema.delete() self.dfps.delete() self.acl.delete() def testDetailsDisplay(self): """ test display of view for an existing schema and no display for an undefined one. """ from tardis.tardis_portal.views import display_datafile_details request = flexmock(user=self.user, groups=[("testgroup",flexmock())]) with self.settings(DATAFILE_VIEWS=[("http://test.com/test/schema", "/test/url"), ("http://does.not.exist", "/false/url")]): response = display_datafile_details(request, dataset_file_id=self.dataset_file.id) self.assertEqual(response.status_code, 200) self.assertTrue("/ajax/parameters/" in response.content) self.assertTrue("/test/url" in response.content) self.assertFalse("/false/url" in response.content)
class StageFilesTestCase(TestCase): def setUp(self): # Create test owner without enough details username, email, password = ('testuser', '*****@*****.**', 'password') user = User.objects.create_user(username, email, password) profile = UserProfile(user=user, isDjangoAccount=True) profile.save() # Need UserAuthentication UserAuthentication(userProfile=profile, username=username, authenticationMethod='localdb').save() # Create staging dir from os import path, makedirs staging_dir = path.join(settings.STAGING_PATH, username) if not path.exists(staging_dir): makedirs(staging_dir) # Ensure that staging dir is set up properly expect(get_full_staging_path(username)).to_be_truthy() Location.force_initialize() # Create test experiment and make user the owner of it experiment = Experiment(title='Text Experiment', institution_name='Test Uni', created_by=user) experiment.save() acl = ObjectACL( pluginId=django_user, entityId=str(user.id), content_object=experiment, canRead=True, isOwner=True, aclOwnershipType=ObjectACL.OWNER_OWNED, ) acl.save() self.dataset = \ Dataset(description='dataset description...') self.dataset.save() self.dataset.experiments.add(experiment) self.dataset.save() self.username, self.password = (username, password) def _get_authenticated_client(self): client = Client() # Login as user login = client.login(username=self.username, password=self.password) self.assertTrue(login) # Return authenticated client return client def _get_staging_url(self): return reverse('tardis.tardis_portal.views.stage_files_to_dataset', args=[str(self.dataset.id)]) def testForbiddenWithoutLogin(self): client = Client() response = client.get(self._get_staging_url()) # Expect a redirect to login expect(response.status_code).to_equal(302) login_url = reverse('tardis.tardis_portal.views.login') ensure(login_url in response['Location'], True, "Redirect URL was not to login.") def testPostOnlyMethodAllowed(self): client = self._get_authenticated_client() for method in (x.lower() for x in ['GET', 'HEAD', 'PUT', 'OPTIONS']): response = getattr(client, method)(self._get_staging_url()) # Expect a 405 Method Not Allowed expect(response.status_code).to_equal(405) # Expect valid "Allow" header response['Allow'] = 'POST' response = client.post(self._get_staging_url()) # Expect something other than a 405 self.assertFalse(response.status_code == 405) def testRequiresJSON(self): client = Client() # Login as user login = client.login(username=self.username, password=self.password) self.assertTrue(login) response = client.post(self._get_staging_url()) # Expect 400 Bad Request because we didn't have a payload expect(response.status_code).to_equal(400) response = client.post(self._get_staging_url(), data={'files': ['foo', 'bar']}) # Expect 400 Bad Request because we didn't have a JSON payload expect(response.status_code).to_equal(400) response = client.post(self._get_staging_url(), data=json.dumps({'files': ['foo', 'bar']}), content_type='application/octet-stream') # Expect 400 Bad Request because we didn't have a JSON Content-Type expect(response.status_code).to_equal(400) def testStageFile(self): client = self._get_authenticated_client() staging_dir = get_full_staging_path(self.username) from os.path import basename from tempfile import NamedTemporaryFile with NamedTemporaryFile('w', dir=staging_dir) as f: # Write some content f.write('This is just some content') f.flush() data = [ f.name ] content_type = 'application/json; charset=utf-8' response = client.post(self._get_staging_url(), data=json.dumps(data), content_type=content_type) # Expect 201 Created expect(response.status_code).to_equal(201) # Expect to get the email address of # staging user back # Can't test for async file staging emails = json.loads(response.content) expect(len(emails)).to_equal(1)
def test_datafile(self): from tardis.tardis_portal.models import Experiment, Dataset def _build(dataset, filename, url, protocol): from tardis.tardis_portal.models import \ Dataset_File, Replica, Location datafile = Dataset_File(dataset=dataset, filename=filename) datafile.save() replica = Replica(datafile=datafile, url=url, protocol=protocol, location=Location.get_default_location()) replica.save() return datafile exp = Experiment(title='test exp1', institution_name='monash', approved=True, created_by=self.user, public_access=Experiment.PUBLIC_ACCESS_NONE) exp.save() dataset = Dataset(description="dataset description...") dataset.save() dataset.experiments.add(exp) dataset.save() save1 = settings.REQUIRE_DATAFILE_SIZES save2 = settings.REQUIRE_DATAFILE_CHECKSUMS try: settings.REQUIRE_DATAFILE_SIZES = False settings.REQUIRE_DATAFILE_CHECKSUMS = False df_file = _build(dataset, 'file.txt', 'path/file.txt', '') self.assertEqual(df_file.filename, 'file.txt') self.assertEqual(df_file.get_preferred_replica().url, 'path/file.txt') self.assertEqual(df_file.get_preferred_replica().protocol, '') self.assertEqual(df_file.dataset, dataset) self.assertEqual(df_file.size, '') self.assertEqual(df_file.get_download_url(), '/test/download/datafile/1/') self.assertTrue(df_file.is_local()) df_file = _build(dataset, 'file1.txt', 'path/file1.txt', 'vbl') self.assertEqual(df_file.filename, 'file1.txt') self.assertEqual(df_file.get_preferred_replica().url, 'path/file1.txt') self.assertEqual(df_file.get_preferred_replica().protocol, 'vbl') self.assertEqual(df_file.dataset, dataset) self.assertEqual(df_file.size, '') self.assertEqual(df_file.get_download_url(), '/test/vbl/download/datafile/2/') self.assertFalse(df_file.is_local()) df_file = _build(dataset, 'f.txt', 'http://localhost:8080/filestore/f.txt', '') self.assertEqual(df_file.filename, 'f.txt') self.assertEqual(df_file.get_preferred_replica().url, 'http://localhost:8080/filestore/f.txt') self.assertEqual(df_file.get_preferred_replica().protocol, '') self.assertEqual(df_file.dataset, dataset) self.assertEqual(df_file.size, '') self.assertEqual(df_file.get_download_url(), '/test/download/datafile/3/') self.assertFalse(df_file.is_local()) # Now check the 'REQUIRE' config params with self.assertRaises(Exception): settings.REQUIRE_DATAFILE_SIZES = True settings.REQUIRE_DATAFILE_CHECKSUMS = False Dataset_File(dataset=dataset, filename='foo.txt', md5sum='bad') with self.assertRaises(Exception): settings.REQUIRE_DATAFILE_SIZES = False settings.REQUIRE_DATAFILE_CHECKSUMS = True Dataset_File(dataset=dataset, filename='foo.txt', size='1') finally: settings.REQUIRE_DATAFILE_SIZES = save1 settings.REQUIRE_DATAFILE_CHECKSUMS = save2
class SimpleSearchTest(MyTardisResourceTestCase): def setUp(self): super(SimpleSearchTest, self).setUp() self.out = StringIO() call_command('search_index', stdout=self.out, action='delete', force=True) call_command('search_index', stdout=self.out, action='rebuild', force=True) # add dataset and datafile to experiment self.dataset1 = Dataset(description='test_dataset') self.dataset1.save() self.dataset1.experiments.add(self.testexp) self.dataset1.save() settings.REQUIRE_DATAFILE_SIZES = False settings.REQUIRE_DATAFILE_CHECKSUMS = False self.datafile = DataFile(dataset=self.dataset1, filename='test.txt') self.datafile.save() def test_simple_search_authenticated_user(self): response = self.api_client.get( '/api/v1/search_simple-search/?query=test', authentication=self.get_credentials()) data = json.loads(response.content.decode()) self.assertEqual(len(data['objects'][0]['hits']['experiments']), 1) self.assertEqual(len(data['objects'][0]['hits']['datasets']), 1) self.assertEqual(len(data['objects'][0]['hits']['datafiles']), 1) def test_simple_search_unauthenticated_user(self): self.testexp.public_access = 100 self.testexp.save() response = self.api_client.get( '/api/v1/search_simple-search/?query=test') data = json.loads(response.content.decode()) self.assertEqual(len(data['objects'][0]['hits']['experiments']), 1) self.assertEqual(len(data['objects'][0]['hits']['datasets']), 1) self.assertEqual(len(data['objects'][0]['hits']['datafiles']), 1) def test_advance_search_unauthenticated_user(self): self.testexp.public_access = 100 self.testexp.save() response = self.api_client.post( '/api/v1/search_advance-search/', data={ "text": "test", "TypeTag": ["Dataset", "Experiment", "Datafile"] }) data = json.loads(response.content.decode()) self.assertEqual(len(data['hits']['experiments']), 1) self.assertEqual(len(data['hits']['datasets']), 1) self.assertEqual(len(data['hits']['datafiles']), 1) def test_advance_search_authenticated_user(self): response = self.api_client.post( '/api/v1/search_advance-search/', data={ "text": "test", "TypeTag": ["Dataset", "Experiment", "Datafile"] }, authentication=self.get_credentials()) data = json.loads(response.content.decode()) self.assertEqual(len(data['hits']['experiments']), 1) self.assertEqual(len(data['hits']['datasets']), 1) self.assertEqual(len(data['hits']['datafiles']), 1)
class DownloadTestCase(TestCase): def setUp(self): # create a test user self.user = User.objects.create_user(username='******', email='', password='******') # create a public experiment self.experiment1 = Experiment( title='Experiment 1', created_by=self.user, public_access=Experiment.PUBLIC_ACCESS_FULL) self.experiment1.save() # create a non-public experiment self.experiment2 = Experiment( title='Experiment 2', created_by=self.user, public_access=Experiment.PUBLIC_ACCESS_NONE) self.experiment2.save() # dataset1 belongs to experiment1 self.dataset1 = Dataset(description='dangerous;name') self.dataset1.save() self.dataset1.experiments.add(self.experiment1) self.dataset1.save() # dataset2 belongs to experiment2 self.dataset2 = Dataset(description='terrible\nname') self.dataset2.save() self.dataset2.experiments.add(self.experiment2) self.dataset2.save() # absolute path first filename1 = 'testfile.txt' filename2 = 'testfile.tiff' self.dest1 = abspath( join(settings.FILE_STORE_PATH, '%s/%s/' % (self.experiment1.id, self.dataset1.id))) self.dest2 = abspath( join(settings.FILE_STORE_PATH, '%s/%s/' % (self.experiment2.id, self.dataset2.id))) if not exists(self.dest1): makedirs(self.dest1) if not exists(self.dest2): makedirs(self.dest2) testfile1 = abspath(join(self.dest1, filename1)) f = open(testfile1, 'w') f.write("Hello World!\n") f.close() testfile2 = abspath(join(self.dest2, filename2)) _generate_test_image(testfile2) self.datafile1 = self._build_datafile(testfile1, filename1, self.dataset1) self.datafile2 = self._build_datafile(testfile2, filename2, self.dataset2) def _build_datafile(self, testfile, filename, dataset, checksum=None, size=None, mimetype=''): filesize, sha512sum = get_size_and_sha512sum(testfile) datafile = DataFile(dataset=dataset, filename=filename, mimetype=mimetype, size=size if size is not None else filesize, sha512sum=(checksum if checksum else sha512sum)) datafile.save() dfo = DataFileObject(datafile=datafile, storage_box=datafile.get_default_storage_box()) dfo.save() with open(testfile, 'r') as sourcefile: dfo.file_object = sourcefile return DataFile.objects.get(pk=datafile.pk) def tearDown(self): self.user.delete() self.experiment1.delete() self.experiment2.delete() rmtree(self.dest1) rmtree(self.dest2) def testView(self): client = Client() # check view of file1 response = client.get('/datafile/view/%i/' % self.datafile1.id) self.assertEqual(response['Content-Disposition'], 'inline; filename="%s"' % self.datafile1.filename) self.assertEqual(response.status_code, 200) response_content = "" for c in response.streaming_content: response_content += c self.assertEqual(response_content, 'Hello World!\n') # check view of file2 response = client.get('/datafile/view/%i/' % self.datafile2.id) # Should be forbidden self.assertEqual(response.status_code, 403) self.experiment2.public_access = Experiment.PUBLIC_ACCESS_FULL self.experiment2.save() # check view of file2 again response = client.get('/datafile/view/%i/' % self.datafile2.id) self.assertEqual(response.status_code, 200) # The following behaviour relies on ImageMagick if IMAGEMAGICK_AVAILABLE: # file2 should have a ".png" filename self.assertEqual( response['Content-Disposition'], 'inline; filename="%s"' % (self.datafile2.filename + '.png')) # file2 should be a PNG self.assertEqual(response['Content-Type'], 'image/png') png_signature = "\x89PNG\r\n\x1a\n" self.assertEqual(response.content[0:8], png_signature) else: # file2 should have a ".tiff" filename self.assertEqual( response['Content-Disposition'], 'inline; filename="%s"' % (self.datafile2.filename)) # file2 should be a TIFF self.assertEqual(response['Content-Type'], 'image/tiff') tiff_signature = "II\x2a\x00" self.assertEqual(response.content[0:4], tiff_signature) def _check_tar_file(self, content, rootdir, datafiles, simpleNames=False, noTxt=False): with NamedTemporaryFile('w') as tempfile: for c in content: tempfile.write(c) tempfile.flush() if getsize(tempfile.name) > 0: expect(is_tarfile(tempfile.name)).to_be_truthy() try: tf = TarFile(tempfile.name, 'r') self._check_names(datafiles, tf.getnames(), rootdir, simpleNames, noTxt) finally: tf.close() else: self._check_names(datafiles, [], rootdir, simpleNames, noTxt) def _check_zip_file(self, content, rootdir, datafiles, simpleNames=False, noTxt=False): with NamedTemporaryFile('w') as tempfile: for c in content: tempfile.write(c) tempfile.flush() # It should be a zip file expect(is_zipfile(tempfile.name)).to_be_truthy() try: zf = ZipFile(tempfile.name, 'r') self._check_names(datafiles, zf.namelist(), rootdir, simpleNames, noTxt) finally: zf.close() def _check_names(self, datafiles, names, rootdir, simpleNames, noTxt): # SimpleNames says if we expect basenames or pathnames # NoTxt says if we expect '.txt' files to be filtered out for name in names: self.assertNotRegexpMatches(name, '\n|;') expect(len(names)).to_equal(len(datafiles)) def testDownload(self): client = Client() # check download for experiment1 as tar response = client.get('/download/experiment/%i/tar/' % self.experiment1.id) self.assertEqual( response['Content-Disposition'], 'attachment; filename="%s-complete.tar"' % self.experiment1.title.replace(' ', '_')) self.assertEqual(response.status_code, 200) self._check_tar_file( response.streaming_content, str(self.experiment1.title.replace(' ', '_')), reduce(lambda x, y: x + y, [ ds.datafile_set.all() for ds in self.experiment1.datasets.all() ])) # check download of file1 response = client.get('/download/datafile/%i/' % self.datafile1.id) self.assertEqual(response['Content-Disposition'], 'attachment; filename="%s"' % self.datafile1.filename) self.assertEqual(response.status_code, 200) response_content = "" for c in response.streaming_content: response_content += c self.assertEqual(response_content, 'Hello World!\n') # requesting file2 should be forbidden... response = client.get('/download/datafile/%i/' % self.datafile2.id) self.assertEqual(response.status_code, 403) # check dataset1 download as tar response = client.post( '/download/datafiles/', { 'expid': self.experiment1.id, 'dataset': [self.dataset1.id], 'datafile': [], 'comptype': 'tar' }) self.assertEqual(response.status_code, 200) self._check_tar_file(response.streaming_content, 'Experiment 1-selection', self.dataset1.datafile_set.all()) # check dataset2 download response = client.post( '/download/datafiles/', { 'expid': self.experiment2.id, 'dataset': [self.dataset2.id], 'datafile': [] }) self.assertEqual(response.status_code, 403) # check datafile1 download via POST response = client.post( '/download/datafiles/', { 'expid': self.experiment1.id, 'dataset': [], 'datafile': [self.datafile1.id] }) self.assertEqual(response.status_code, 200) self._check_tar_file(response.streaming_content, 'Experiment 1-selection', [self.datafile1]) # check datafile2 download via POST response = client.post( '/download/datafiles/', { 'expid': self.experiment2.id, 'dataset': [], 'datafile': [self.datafile2.id] }) self.assertEqual(response.status_code, 403) # Check datafile2 download with second experiment to "metadata only" self.experiment2.public_access = Experiment.PUBLIC_ACCESS_METADATA self.experiment2.save() response = client.get('/download/datafile/%i/' % self.datafile2.id) # Metadata-only means "no file access"! self.assertEqual(response.status_code, 403) # Check datafile2 download with second experiment to public self.experiment2.public_access = Experiment.PUBLIC_ACCESS_FULL self.experiment2.save() response = client.get('/download/datafile/%i/' % self.datafile2.id) self.assertEqual(response.status_code, 200) # This should be a TIFF (which often starts with "II\x2a\x00") self.assertEqual(response['Content-Type'], 'image/tiff') response_content = "" for c in response.streaming_content: response_content += c self.assertEqual(response_content[0:4], "II\x2a\x00") # check experiment tar download with alternative organization response = client.get('/download/experiment/%i/tar/' % self.experiment1.id) self.assertEqual(response.status_code, 200) self.assertEqual( response['Content-Disposition'], 'attachment; filename="%s-complete.tar"' % self.experiment1.title.replace(' ', '_')) self._check_tar_file(response.streaming_content, str(self.experiment1.id), reduce(lambda x, y: x + y, [ ds.datafile_set.all() for ds in self.experiment1.datasets.all() ]), simpleNames=True) # check experiment1 download with '.txt' filtered out (none left) response = client.get('/download/experiment/%i/tar/' % self.experiment1.id) self.assertEqual(response.status_code, 200) # check experiment2 download with '.txt' filtered out response = client.get('/download/experiment/%i/tar/' % self.experiment2.id) self.assertEqual(response.status_code, 200) self.assertEqual( response['Content-Disposition'], 'attachment; filename="%s-complete.tar"' % self.experiment2.title.replace(' ', '_')) self._check_tar_file(response.streaming_content, str(self.experiment2.id), reduce(lambda x, y: x + y, [ ds.datafile_set.all() for ds in self.experiment2.datasets.all() ]), simpleNames=True, noTxt=True) def testDatasetFile(self): # check registered text file for physical file meta information df = DataFile.objects.get(pk=self.datafile1.id) # skipping test # noqa # pylint: disable=W0101 try: from magic import Magic self.assertEqual(df.mimetype, 'text/plain; charset=us-ascii') except: # XXX Test disabled because lib magic can't be loaded pass self.assertEqual(df.size, 13) self.assertEqual(df.md5sum, '8ddd8be4b179a529afa5f2ffae4b9858') # Now check we can calculate checksums and infer the mime type # for a JPG file. filename = 'tardis/tardis_portal/tests/test_data/ands-logo-hi-res.jpg' dataset = Dataset.objects.get(pk=self.dataset1.id) pdf1 = self._build_datafile(filename, basename(filename), dataset) self.assertEqual(pdf1.file_objects.get().verify(), True) pdf1 = DataFile.objects.get(pk=pdf1.pk) try: from magic import Magic # noqa self.assertEqual(pdf1.mimetype, 'image/jpeg') except: # XXX Test disabled because lib magic can't be loaded pass self.assertEqual(pdf1.size, 14232) self.assertEqual(pdf1.md5sum, 'c450d5126ffe3d14643815204daf1bfb') # Now check that we can override the physical file meta information # We are setting size/checksums that don't match the actual file, so # the pdf2 = self._build_datafile( filename, filename, dataset, checksum= 'cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e', size=0, mimetype= 'application/vnd.openxmlformats-officedocument.presentationml.presentation' ) # noqa self.assertEqual(pdf2.size, 0) self.assertEqual(pdf2.md5sum, '') self.assertEqual(pdf2.file_objects.get().verified, False) pdf2 = DataFile.objects.get(pk=pdf2.pk) try: from magic import Magic # noqa self.assertEqual( pdf2.mimetype, 'application/vnd.openxmlformats-officedocument.presentationml.presentation' ) # noqa except: # XXX Test disabled because lib magic can't be loaded pass self.assertEqual(pdf2.size, 0) self.assertEqual(pdf2.md5sum, '') pdf2.mimetype = '' pdf2.save() pdf2.file_objects.get().save() pdf2 = DataFile.objects.get(pk=pdf2.pk) try: from magic import Magic # noqa self.assertEqual(pdf2.mimetype, 'application/pdf') except: # XXX Test disabled because lib magic can't be loaded pass
class DownloadTestCase(TestCase): def setUp(self): # create a test user self.user = User.objects.create_user(username='******', email='', password='******') # create a public experiment self.experiment1 = Experiment(title='Experiment 1', created_by=self.user, public=True) self.experiment1.save() # create a non-public experiment self.experiment2 = Experiment(title='Experiment 2', created_by=self.user, public=False) self.experiment2.save() # dataset1 belongs to experiment1 self.dataset1 = Dataset(experiment=self.experiment1) self.dataset1.save() # dataset2 belongs to experiment2 self.dataset2 = Dataset(experiment=self.experiment2) self.dataset2.save() # absolute path first filename = 'testfile.txt' self.dest1 = abspath(join(settings.FILE_STORE_PATH, '%s/%s/' % (self.experiment1.id, self.dataset1.id))) self.dest2 = abspath(join(settings.FILE_STORE_PATH, '%s/%s/' % (self.experiment2.id, self.dataset2.id))) if not exists(self.dest1): makedirs(self.dest1) if not exists(self.dest2): makedirs(self.dest2) testfile1 = abspath(join(self.dest1, filename)) f = open(testfile1, 'w') f.write("Hello World!\n") f.close() testfile2 = abspath(join(self.dest2, filename)) f = open(testfile2, 'w') f.write("Hello World!\n") f.close() self.dataset_file1 = Dataset_File(dataset=self.dataset1, filename=filename, protocol='tardis', url='tardis://%s' % filename) self.dataset_file1.save() self.dataset_file2 = Dataset_File(dataset=self.dataset2, filename=basename(filename), protocol='tardis', url='tardis://%s' % filename) self.dataset_file2.save() def tearDown(self): self.user.delete() self.experiment1.delete() self.experiment2.delete() rmtree(self.dest1) rmtree(self.dest2) def testDownload(self): client = Client() # check download for experiment1 response = client.get('/download/experiment/%i/zip/' % self.experiment1.id) self.assertEqual(response['Content-Disposition'], 'attachment; filename="experiment%s-complete.zip"' % self.experiment1.id) self.assertEqual(response.status_code, 200) # check download of file1 response = client.get('/download/datafile/%i/' % self.dataset_file1.id) self.assertEqual(response['Content-Disposition'], 'attachment; filename="%s"' % self.dataset_file2.filename) self.assertEqual(response.status_code, 200) self.assertEqual(response.content, 'Hello World!\n') # requesting file2 should be forbidden... response = client.get('/download/datafile/%i/' % self.dataset_file2.id) self.assertEqual(response.status_code, 403) # check dataset1 download response = client.post('/download/datafiles/', {'expid': self.experiment1.id, 'dataset': [self.dataset1.id], 'datafile': []}) self.assertEqual(response.status_code, 200) # check dataset2 download response = client.post('/download/datafiles/', {'expid': self.experiment2.id, 'dataset': [self.dataset2.id], 'datafile': []}) self.assertEqual(response.status_code, 403) # check datafile1 download via POST response = client.post('/download/datafiles/', {'expid': self.experiment1.id, 'dataset': [], 'datafile': [self.dataset_file1.id]}) self.assertEqual(response.status_code, 200) # check datafile2 download via POST response = client.post('/download/datafiles/', {'expid': self.experiment2.id, 'dataset': [], 'datafile': [self.dataset_file2.id]}) self.assertEqual(response.status_code, 403) def testDatasetFile(self): # check registered text file for physical file meta information df = Dataset_File.objects.get(pk=self.dataset_file1.id) try: from magic import Magic self.assertEqual(df.mimetype, 'text/plain; charset=us-ascii') except: # XXX Test disabled becuse lib magic can't be loaded pass self.assertEqual(df.size, str(13)) self.assertEqual(df.md5sum, '8ddd8be4b179a529afa5f2ffae4b9858') # now check a pdf file filename = join(abspath(dirname(__file__)), '../static/downloads/DatasetDepositionGuide.pdf') dataset = Dataset.objects.get(pk=self.dataset1.id) pdf1 = Dataset_File(dataset=dataset, filename=basename(filename), url='file://%s' % filename, protocol='file') pdf1.save() try: from magic import Magic self.assertEqual(pdf1.mimetype, 'application/pdf') except: # XXX Test disabled becuse lib magic can't be loaded pass self.assertEqual(pdf1.size, str(1008475)) self.assertEqual(pdf1.md5sum, '9192b3d3e0056412b1d21d3e33562eba') # now check that we can override the physical file meta information pdf2 = Dataset_File(dataset=dataset, filename=basename(filename), url='file://%s' % filename, protocol='file', mimetype='application/vnd.openxmlformats-officedocument.presentationml.presentation', size=str(0), md5sum='md5sum') pdf2.save() try: from magic import Magic self.assertEqual(pdf2.mimetype, 'application/vnd.openxmlformats-officedocument.presentationml.presentation') except: # XXX Test disabled becuse lib magic can't be loaded pass self.assertEqual(pdf2.size, str(0)) self.assertEqual(pdf2.md5sum, 'md5sum') pdf2.mimetype = '' pdf2.save() try: from magic import Magic self.assertEqual(pdf2.mimetype, 'application/pdf') except: # XXX Test disabled becuse lib magic can't be loaded pass
class ViewsTestCase(TestCase): def setUp(self): from django.contrib.auth.models import User from tardis.tardis_portal.models import Experiment from tardis.tardis_portal.models import ExperimentACL from tardis.tardis_portal.models import Dataset from tardis.apps.mrtardis.mrtask import MRtask user = '******' pwd = 'secret' email = '' self.user = User.objects.create_user(user, email, pwd) self.client = Client() self.client.login(username=user, password=pwd) self.experiment = Experiment(approved=True, title="Test Experiment", institution_name="Test Institution", created_by=self.user, public=False) self.experiment.save() acl = ExperimentACL(pluginId="django_user", entityId="1", experiment=self.experiment, canRead=True, canWrite=True, canDelete=True, isOwner=True) acl.save() self.test_dataset = Dataset(experiment=self.experiment, description="test dataset") self.test_dataset.save() self.test_mrtask = MRtask(dataset=self.test_dataset) def test_view_index(self): response = self.client.get('/apps/mrtardis/index/%d/' % self.experiment.id, HTTP_X_REQUESTED_WITH='XMLHttpRequest') self.assertEqual(response.status_code, 200) def test_hpc_user_setup(self): import tardis.apps.mrtardis.utils as utils testresult = utils.test_hpc_connection(self.user) self.assertEqual(testresult, False) def test_views_test_user_setup(self): # with POST: posturl = '/apps/mrtardis/test_user_setup/%d/' % self.experiment.id response = self.client.post(posturl, {'hpc_username': '******'}, HTTP_X_REQUESTED_WITH='XMLHttpRequest') self.assertEqual(response.status_code, 302) from tardis.apps.mrtardis.models import HPCUser self.assertEqual(HPCUser.objects.get(user=self.user).hpc_username, 'john') def test_views_MRform(self): posturl = '/apps/mrtardis/MRform/%d/' % self.experiment.id postdataarray = [{'postdata': {'action': 'newDS', 'description': 'who is testing the testers?'}, 'expectedoutcome': ""}, {'postdata': {'action': 'continue', 'dataset': '1'}, 'expectedoutcome': ""}, {'postdata': {'action': 'rerunDS', 'dataset': '1', 'description': "test-rerun"}, 'expectedoutcome': ""}, ] for actiontype in postdataarray: postdata = actiontype['postdata'] response = self.client.post(posturl, postdata, HTTP_X_REQUESTED_WITH='XMLHttpRequest') self.assertEqual(response.status_code, 200) # def test_views_displayResults(self): # posturl = '/apps/mrtardis/displayResults/%d/' % self.experiment.id # response = self.client.post(posturl, {'dataset': 1}, # HTTP_X_REQUESTED_WITH='XMLHttpRequest') # self.assertEqual(response.status_code, 200) def test_views_type_filtered_file_list(self): posturl = '/apps/mrtardis/type_filtered_file_list/%d/' %\ self.test_dataset.id filetypes = [".mtz", ".pdb"] for filetype in filetypes: response = self.client.post(posturl, {'type': filetype}, HTTP_X_REQUESTED_WITH='XMLHttpRequest') self.assertEqual(response.status_code, 200) def test_views_add_pdb_files(self): geturl = '/apps/mrtardis/add_pdb_files/%d/' %\ self.test_dataset.id response = self.client.get(geturl, HTTP_X_REQUESTED_WITH='XMLHttpRequest') self.assertEqual(response.status_code, 200)
class MyTardisHSMTasksTestCase(TestCase): """Tests for mytardis_hsm.tasks""" def setUp(self): """Setup test fixtures if needed.""" self.user = User.objects.create_user("doctor", '', "pwd") self.exp = Experiment(title="Wonderful", institution_name="Monash University", created_by=self.user) self.exp.save() group = Group(name="Group1") group.save() facility = Facility(name="Test Facility", manager_group=group) facility.save() inst = Instrument(name="Test Instrument1", facility=facility) inst.save() self.dataset = Dataset(description="Dataset1", instrument=inst) self.dataset.save() storage_classes = getattr(settings, "HSM_STORAGE_CLASSES", DEFAULT_HSM_CLASSES) self.sbox1 = StorageBox(name="SBOX1", django_storage_class=storage_classes[0], status='online', max_size=256) self.sbox1.save() sbox1_attr = StorageBoxAttribute(storage_box=self.sbox1, key='type', value=StorageBox.DISK) sbox1_attr.save() sbox1_loc_opt = StorageBoxOption(storage_box=self.sbox1, key="location", value=tempfile.gettempdir()) sbox1_loc_opt.save() self.sbox2 = StorageBox( name="SBOX2", django_storage_class="any.non.disk.StorageSystem", status='offline', max_size=256) self.sbox2.save() sbox2_attr = StorageBoxAttribute(storage_box=self.sbox2, key='type', value=StorageBox.TAPE) sbox2_attr.save() @mock.patch("os.stat") def test_000_update_df_status_offline(self, mock_stat): """update_df_status should check the online status of preferred DFOs for all previously online datafiles and update online Parameter to 'False' for any offline files.""" df1 = DataFile(dataset=self.dataset, filename="test_df.jpg") df1.save() dfo1 = DataFileObject(datafile=df1, storage_box=self.sbox1, uri="stream/test.jpg", verified=True) dfo1.save() schema = Schema.objects.get(namespace=HSM_DATAFILE_NAMESPACE) ps = DatafileParameterSet(schema=schema, datafile=df1) ps.save() param_name = ParameterName.objects.get(schema=schema, name="online") param = DatafileParameter(parameterset=ps, name=param_name) param.string_value = True param.save() mock_stat.return_value = Stats(st_size=10000, st_blocks=0, st_mtime=datetime.now()) update_df_status() params = DatafileParameter.objects.filter( parameterset__schema=schema, parameterset__datafile=df1) self.assertEquals(params.count(), 1) self.assertEquals(params[0].string_value, "False") @mock.patch("os.stat") def test_001_update_df_status_online(self, mock_stat): """update_df_status should check the online status of preferred DFOs for all previously online datafiles and leave the online Parameter as 'True' for any online files.""" df1 = DataFile(dataset=self.dataset, filename="test_df.jpg") df1.save() dfo1 = DataFileObject(datafile=df1, storage_box=self.sbox1, uri="stream/test.jpg", verified=True) dfo1.save() # df1.verify() schema = Schema.objects.get(namespace=HSM_DATAFILE_NAMESPACE) ps = DatafileParameterSet(schema=schema, datafile=df1) ps.save() param_name = ParameterName.objects.get(schema=schema, name="online") param = DatafileParameter(parameterset=ps, name=param_name) param.string_value = True param.save() mock_stat.return_value = Stats(st_size=10000, st_blocks=100, st_mtime=datetime.now()) update_df_status() params = DatafileParameter.objects.filter( parameterset__schema__namespace=HSM_DATAFILE_NAMESPACE, parameterset__datafile=df1) self.assertEquals(params.count(), 1) self.assertEquals(params[0].string_value, "True") @mock.patch('mytardis_hsm.mytardis_hsm.df_online') @mock.patch("os.stat") def test_002_update_df_status_skip_unverified(self, mock_stat, df_online): """update_df_status should skip files that are unverified""" df2 = DataFile(dataset=self.dataset, filename="test_df2.jpg") df2.save() dfo2 = DataFileObject(datafile=df2, storage_box=self.sbox1, uri="stream/test_df2.jpg") dfo2.save() schema = Schema.objects.get(namespace=HSM_DATAFILE_NAMESPACE) ps2 = DatafileParameterSet(schema=schema, datafile=df2) ps2.save() param_name = ParameterName.objects.get(schema=schema, name="online") param2 = DatafileParameter(parameterset=ps2, name=param_name) param2.string_value = True param2.save() mock_stat.return_value = Stats(st_size=10000, st_blocks=100, st_mtime=datetime.now()) update_df_status() df_online.assert_not_called() @mock.patch('mytardis_hsm.tasks.df_online', autopec=True) @mock.patch("os.stat") def test_003_update_df_status_skip_offline(self, mock_stat, mock_df_online): """update_df_status should skip any files that have previously marked as offline.""" df2 = DataFile(dataset=self.dataset, filename="test_df2.jpg") df2.save() dfo2 = DataFileObject(datafile=df2, storage_box=self.sbox1, uri="stream/test_df2.jpg", verified=True) dfo2.save() # df2.verify() schema = Schema.objects.get(namespace=HSM_DATAFILE_NAMESPACE) ps2 = DatafileParameterSet(schema=schema, datafile=df2) ps2.save() param_name = ParameterName.objects.get(schema=schema, name="online") param2 = DatafileParameter(parameterset=ps2, name=param_name) param2.string_value = False param2.save() mock_stat.return_value = Stats(st_size=10000, st_blocks=100, st_mtime=datetime.now()) update_df_status() # assert that the df_online method wasn't called self.assertEquals(mock_df_online.call_count, 0)
class ViewTemplateContextsTest(TestCase): def setUp(self): """ setting up essential objects, copied from tests above """ user = '******' pwd = 'secret' email = '' self.user = User.objects.create_user(user, email, pwd) self.userProfile = self.user.userprofile self.exp = Experiment(title='test exp1', institution_name='monash', created_by=self.user) self.exp.save() self.acl = ObjectACL( pluginId=django_user, entityId=str(self.user.id), content_object=self.exp, canRead=True, isOwner=True, aclOwnershipType=ObjectACL.OWNER_OWNED, ) self.acl.save() self.dataset = Dataset(description='dataset description...') self.dataset.save() self.dataset.experiments.add(self.exp) self.dataset.save() self.datafile = DataFile(dataset=self.dataset, size=42, filename="foo", md5sum="junk") self.datafile.save() def tearDown(self): self.user.delete() self.exp.delete() self.dataset.delete() self.datafile.delete() self.acl.delete() def testExperimentView(self): """ test some template context parameters for an experiment view """ from tardis.tardis_portal.views import ExperimentView from django.http import HttpRequest import sys # Default behavior views_module = flexmock(sys.modules['tardis.tardis_portal.views']) request = HttpRequest() request.method = 'GET' request.user = self.user request.groups = [] context = { 'organization': ['test', 'test2'], 'default_organization': 'test', 'default_format': 'tar', 'protocol': [['tgz', '/download/experiment/1/tgz/'], ['tar', '/download/experiment/1/tar/']] } views_module.should_call('render_response_index'). \ with_args(_AnyMatcher(), "tardis_portal/view_experiment.html", _ContextMatcher(context)) view_fn = ExperimentView.as_view() response = view_fn(request, experiment_id=self.exp.id) self.assertEqual(response.status_code, 200) # Behavior with USER_AGENT_SENSING enabled and a request.user_agent saved_setting = getattr(settings, "USER_AGENT_SENSING", None) try: setattr(settings, "USER_AGENT_SENSING", True) request = HttpRequest() request.method = 'GET' request.user = self.user request.groups = [] mock_agent = _MiniMock(os=_MiniMock(family="Macintosh")) setattr(request, 'user_agent', mock_agent) context = { 'organization': ['classic', 'test', 'test2'], 'default_organization': 'classic', 'default_format': 'tar', 'protocol': [['tar', '/download/experiment/1/tar/']] } views_module.should_call('render_response_index'). \ with_args(_AnyMatcher(), "tardis_portal/view_experiment.html", _ContextMatcher(context)) view_fn = ExperimentView.as_view() response = view_fn(request, experiment_id=self.exp.id) self.assertEqual(response.status_code, 200) finally: if saved_setting is not None: setattr(settings, "USER_AGENT_SENSING", saved_setting) else: delattr(settings, "USER_AGENT_SENSING") def testDatasetView(self): """ test some context parameters for a dataset view """ from tardis.tardis_portal.views import DatasetView from django.http import HttpRequest import sys views_module = flexmock(sys.modules['tardis.tardis_portal.views']) request = HttpRequest() request.method = 'GET' request.user = self.user request.groups = [] context = {'default_organization': 'test', 'default_format': 'tar'} views_module.should_call('render_response_index'). \ with_args(_AnyMatcher(), "tardis_portal/view_dataset.html", _ContextMatcher(context)) view_fn = DatasetView.as_view() response = view_fn(request, dataset_id=self.dataset.id) self.assertEqual(response.status_code, 200) # Behavior with USER_AGENT_SENSING enabled and a request.user_agent saved_setting = getattr(settings, "USER_AGENT_SENSING", None) try: setattr(settings, "USER_AGENT_SENSING", True) request = HttpRequest() request.method = 'GET' request.user = self.user request.groups = [] mock_agent = _MiniMock(os=_MiniMock(family="Macintosh")) setattr(request, 'user_agent', mock_agent) context = { 'default_organization': 'classic', 'default_format': 'tar' } views_module.should_call('render_response_index'). \ with_args(_AnyMatcher(), "tardis_portal/view_dataset.html", _ContextMatcher(context)) view_fn = DatasetView.as_view() response = view_fn(request, dataset_id=self.dataset.id) self.assertEqual(response.status_code, 200) finally: if saved_setting is not None: setattr(settings, "USER_AGENT_SENSING", saved_setting) else: delattr(settings, "USER_AGENT_SENSING")
class ContextualViewTest(TestCase): def setUp(self): """ setting up essential objects, copied from tests above """ user = '******' pwd = 'secret' email = '' self.user = User.objects.create_user(user, email, pwd) self.userProfile = self.user.userprofile self.exp = Experiment(title='test exp1', institution_name='monash', created_by=self.user) self.exp.save() self.acl = ObjectACL( pluginId=django_user, entityId=str(self.user.id), content_object=self.exp, canRead=True, isOwner=True, aclOwnershipType=ObjectACL.OWNER_OWNED, ) self.acl.save() self.dataset = Dataset(description='dataset description...') self.dataset.save() self.dataset.experiments.add(self.exp) self.dataset.save() self.datafile = DataFile(dataset=self.dataset, size=42, filename="foo", md5sum="junk") self.datafile.save() self.testschema = Schema(namespace="http://test.com/test/schema", name="Test View", type=Schema.DATAFILE, hidden=True) self.testschema.save() self.dfps = DatafileParameterSet(datafile=self.datafile, schema=self.testschema) self.dfps.save() def tearDown(self): self.user.delete() self.exp.delete() self.dataset.delete() self.datafile.delete() self.testschema.delete() self.dfps.delete() self.acl.delete() def testDetailsDisplay(self): """ test display of view for an existing schema and no display for an undefined one. """ from tardis.tardis_portal.views import display_datafile_details request = flexmock(user=self.user, groups=[("testgroup", flexmock())]) with self.settings(DATAFILE_VIEWS=[( "http://test.com/test/schema", "/test/url"), ("http://does.not.exist", "/false/url")]): response = display_datafile_details(request, datafile_id=self.datafile.id) self.assertEqual(response.status_code, 200) self.assertTrue("/ajax/parameters/" in response.content) self.assertTrue("/test/url" in response.content) self.assertFalse("/false/url" in response.content)
class UploadTestCase(TestCase): def setUp(self): from os import path, mkdir from tempfile import mkdtemp user = '******' pwd = 'secret' email = '' self.user = User.objects.create_user(user, email, pwd) self.userProfile = UserProfile(user=self.user) self.test_dir = mkdtemp() self.exp = Experiment(title='test exp1', institution_name='monash', created_by=self.user) self.exp.save() acl = ExperimentACL( pluginId=django_user, entityId=str(self.user.id), experiment=self.exp, canRead=True, isOwner=True, aclOwnershipType=ExperimentACL.OWNER_OWNED, ) acl.save() self.dataset = \ Dataset(description='dataset description...', experiment=self.exp) self.dataset.save() self.experiment_path = path.join(settings.FILE_STORE_PATH, str(self.dataset.experiment.id)) self.dataset_path = path.join(self.experiment_path, str(self.dataset.id)) if not path.exists(self.experiment_path): mkdir(self.experiment_path) if not path.exists(self.dataset_path): mkdir(self.dataset_path) # write test file self.filename = 'testfile.txt' self.f1 = open(path.join(self.test_dir, self.filename), 'w') self.f1.write('Test file 1') self.f1.close() self.f1_size = path.getsize(path.join(self.test_dir, self.filename)) self.f1 = open(path.join(self.test_dir, self.filename), 'r') def tearDown(self): from shutil import rmtree self.f1.close() rmtree(self.test_dir) rmtree(self.dataset_path) rmtree(self.experiment_path) self.exp.delete() def testFileUpload(self): from os import path c = Client() c.login(username='******', password='******') session_id = c.session.session_key response = c.post('/upload/' + str(self.dataset.id) + '/', { 'Filedata': self.f1, 'session_id': session_id }) test_files_db = \ Dataset_File.objects.filter(dataset__id=self.dataset.id) self.assertTrue( path.exists(path.join(self.dataset_path, self.filename))) self.assertTrue(self.dataset.id == 1) self.assertTrue(test_files_db[0].url == 'tardis://testfile.txt') def testUploadComplete(self): from django.http import QueryDict, HttpRequest from tardis.tardis_portal.views import upload_complete data = [('filesUploaded', '1'), ('speed', 'really fast!'), ('allBytesLoaded', '2'), ('errorCount', '0')] post = QueryDict('&'.join(['%s=%s' % (k, v) for (k, v) in data])) request = HttpRequest() request.POST = post response = upload_complete(request) self.assertTrue('<p>Number: 1</p>' in response.content) self.assertTrue('<p>Errors: 0</p>' in response.content) self.assertTrue('<p>Bytes: 2</p>' in response.content) self.assertTrue('<p>Speed: really fast!</p>' in response.content)
class StageFilesTestCase(TestCase): def setUp(self): # Create test owner without enough details username, email, password = ('testuser', '*****@*****.**', 'password') user = User.objects.create_user(username, email, password) profile = UserProfile(user=user, isDjangoAccount=True) profile.save() # Need UserAuthentication UserAuthentication(userProfile=profile, username=username, authenticationMethod='localdb').save() # Create staging dir from os import path, makedirs staging_dir = path.join(settings.STAGING_PATH, username) if not path.exists(staging_dir): makedirs(staging_dir) # Ensure that staging dir is set up properly expect(get_full_staging_path(username)).to_be_truthy() # Create test experiment and make user the owner of it experiment = Experiment(title='Text Experiment', institution_name='Test Uni', created_by=user) experiment.save() acl = ExperimentACL( pluginId=django_user, entityId=str(user.id), experiment=experiment,\ canRead=True, isOwner=True, aclOwnershipType=ExperimentACL.OWNER_OWNED, ) acl.save() self.dataset = \ Dataset(description='dataset description...') self.dataset.save() self.dataset.experiments.add(experiment) self.dataset.save() self.username, self.password = (username, password) def _get_authenticated_client(self): client = Client() # Login as user login = client.login(username=self.username, password=self.password) self.assertTrue(login) # Return authenticated client return client def _get_staging_url(self): return reverse('tardis.tardis_portal.views.stage_files_to_dataset', args=[str(self.dataset.id)]) def testForbiddenWithoutLogin(self): client = Client() response = client.get(self._get_staging_url()) # Expect a redirect to login expect(response.status_code).to_equal(302) login_url = reverse('tardis.tardis_portal.views.login') ensure(login_url in response['Location'], True, "Redirect URL was not to login.") def testPostOnlyMethodAllowed(self): client = self._get_authenticated_client() for method in (x.lower() for x in ['GET', 'HEAD', 'PUT', 'OPTIONS']): response = getattr(client, method)(self._get_staging_url()) # Expect a 405 Method Not Allowed expect(response.status_code).to_equal(405) # Expect valid "Allow" header response['Allow'] = 'POST' response = client.post(self._get_staging_url()) # Expect something other than a 405 self.assertFalse(response.status_code == 405) def testRequiresJSON(self): client = Client() # Login as user login = client.login(username=self.username, password=self.password) self.assertTrue(login) response = client.post(self._get_staging_url()) # Expect 400 Bad Request because we didn't have a payload expect(response.status_code).to_equal(400) response = client.post(self._get_staging_url(), data={'files': ['foo', 'bar']}) # Expect 400 Bad Request because we didn't have a JSON payload expect(response.status_code).to_equal(400) response = client.post(self._get_staging_url(), data=json.dumps({'files': ['foo', 'bar']}), content_type='application/octet-stream') # Expect 400 Bad Request because we didn't have a JSON Content-Type expect(response.status_code).to_equal(400) def testStageFile(self): client = self._get_authenticated_client() staging_dir = get_full_staging_path(self.username) from os.path import basename from tempfile import NamedTemporaryFile with NamedTemporaryFile('w', dir=staging_dir) as f: # Write some content f.write('This is just some content') f.flush() data = [f.name] content_type = 'application/json; charset=utf-8' response = client.post(self._get_staging_url(), data=json.dumps(data), content_type=content_type) # Expect 201 Created expect(response.status_code).to_equal(201) # Expect to get a list of URLs back urls = json.loads(response.content) expect(len(urls)).to_equal(1) # Should have single staging file dataset = Dataset.objects.get(id=self.dataset.id) expect(dataset.dataset_file_set.count()).to_equal(1) datafile = dataset.dataset_file_set.all()[0] expect(datafile.filename).to_equal(basename(f.name)) expect(urlparse(datafile.url).scheme).to_equal('')