class ModelTestCase(TestCase): def setUp(self): self.test_box = StorageBox(name='test box', status='online', max_size=123) self.test_box.save() def test_storageboxoption(self): string_input = 'some credential' object_input = { 'a dictionary': 'of different things', 'booleans': True, 'and nothing': None } string_option = StorageBoxOption(storage_box=self.test_box, key='an_option', value=string_input) string_option.save() object_option = StorageBoxOption(storage_box=self.test_box, key='optional', value_type=StorageBoxOption.PICKLE) object_option.unpickled_value = object_input object_option.save() options_dict = self.test_box.get_options_as_dict() self.assertEqual(options_dict['an_option'], string_input) self.assertEqual(options_dict['optional'], object_input) def tearDown(self): self.test_box.delete()
def setUp(self): """Setup test fixtures if needed.""" self.user = User.objects.create_user("doctor", '', "pwd") self.exp = Experiment(title="Wonderful", institution_name="Monash University", created_by=self.user) self.exp.save() group = Group(name="Group1") group.save() facility = Facility(name="Test Facility", manager_group=group) facility.save() self.inst = Instrument(name="Test Instrument1", facility=facility) self.inst.save() self.dataset = Dataset(description="Dataset1", instrument=self.inst) self.dataset.save() storage_classes = getattr(settings, "HSM_STORAGE_CLASSES", DEFAULT_HSM_CLASSES) self.sbox1 = StorageBox(name="SBOX1", django_storage_class=storage_classes[0], status='online', max_size=256) self.sbox1.save() sbox1_attr = StorageBoxAttribute(storage_box=self.sbox1, key='type', value=StorageBox.DISK) sbox1_attr.save() sbox1_loc_opt = StorageBoxOption(storage_box=self.sbox1, key="location", value="/dummy/path") sbox1_loc_opt.save() self.sbox2 = StorageBox( name="SBOX2", django_storage_class="any.non.disk.StorageSystem", status='offline', max_size=256) self.sbox2.save() sbox2_attr = StorageBoxAttribute(storage_box=self.sbox2, key='type', value=StorageBox.TAPE) sbox2_attr.save() self.df1 = DataFile(dataset=self.dataset, filename="test_df.jpg") self.df1.save() self.dfo1 = DataFileObject(datafile=self.df1, storage_box=self.sbox1, uri="stream/test.jpg", verified=True) self.dfo1.save() self.df1.verify()
def setUp(self): raise SkipTest # temporarily disabling this feature, needs coding from tempfile import mkdtemp, mktemp from django.conf import settings import os # Disconnect post_save signal from django.db.models.signals import post_save from tardis.tardis_portal.models import Experiment, \ staging_hook, Dataset, DataFile, DataFileObject, StorageBox post_save.disconnect(staging_hook, sender=DataFileObject) from django.contrib.auth.models import User user = '******' pwd = 'secret' email = '' self.user = User.objects.create_user(user, email, pwd) try: os.makedirs(settings.GET_FULL_STAGING_PATH_TEST) except OSError: pass self.temp = mkdtemp(dir=settings.GET_FULL_STAGING_PATH_TEST) self.filepath = mktemp(dir=self.temp) content = 'test file' with open(self.filepath, "w+b") as f: f.write(content) # make datafile exp = Experiment(title='test exp1', institution_name='monash', created_by=self.user) exp.save() # make dataset dataset = Dataset(description="dataset description...") dataset.save() dataset.experiments.add(exp) dataset.save() # create datafile df = DataFile(dataset=dataset, size=len(content), filename=path.basename(self.file), md5sum='f20d9f2072bbeb6691c0f9c5099b01f3') df.save() # create replica base_url = settings.GET_FULL_STAGING_PATH_TEST s_box = StorageBox.get_default_storage(location=base_url) dfo = DataFileObject(datafile=df, uri=self.filepath, storage_box=s_box) dfo.save() self.dfo = dfo
class ModelTestCase(TestCase): def setUp(self): self.test_box = StorageBox(name="test box", status="online", max_size=123) self.test_box.save() def test_storageboxoption(self): string_input = "some credential" object_input = {"a dictionary": "of different things", "booleans": True, "and nothing": None} string_option = StorageBoxOption(storage_box=self.test_box, key="an_option", value=string_input) string_option.save() object_option = StorageBoxOption(storage_box=self.test_box, key="optional", value_type=StorageBoxOption.PICKLE) object_option.unpickled_value = object_input object_option.save() options_dict = self.test_box.get_options_as_dict() self.assertEqual(options_dict["an_option"], string_input) self.assertEqual(options_dict["optional"], object_input) def tearDown(self): self.test_box.delete()
def get_or_create_storage_box(datafile): key_name = 'datafile_id' class_name = 'tardis.tardis_portal.storage.squashfs.SquashFSStorage' try: s_box = StorageBoxOption.objects.get( key=key_name, value=datafile.id, storage_box__django_storage_class=class_name).storage_box except StorageBoxOption.DoesNotExist: s_box = StorageBox( django_storage_class=class_name, max_size=datafile.size, status='empty', name=datafile.filename, description='SquashFS Archive in DataFile id: %d, filename: %s' % (datafile.id, datafile.filename)) s_box.save() StorageBoxOption(key=key_name, value=datafile.id, storage_box=s_box).save() return s_box
def setUp(self): # Create test owner without enough details username, email, password = ('testuser', '*****@*****.**', 'password') user = User.objects.create_user(username, email, password) # Create test experiment and make user the owner of it experiment = Experiment(title='Text Experiment', institution_name='Test Uni', created_by=user) experiment.save() acl = ObjectACL( pluginId='django_user', entityId=str(user.id), content_object=experiment, canRead=True, isOwner=True, aclOwnershipType=ObjectACL.OWNER_OWNED, ) acl.save() dataset = Dataset(description='dataset description...') dataset.save() dataset.experiments.add(experiment) dataset.save() base_path = path.join(path.dirname(__file__), 'fixtures') s_box = StorageBox.get_default_storage(location=base_path) def create_datafile(index): testfile = path.join(base_path, 'middleware_test%d.txt' % index) size, sha512sum = get_size_and_sha512sum(testfile) datafile = DataFile(dataset=dataset, filename=path.basename(testfile), size=size, sha512sum=sha512sum) datafile.save() dfo = DataFileObject( datafile=datafile, storage_box=s_box, uri=path.basename(testfile)) dfo.save() if index != 1: dfo.verified = False dfo.save(update_fields=['verified']) return DataFile.objects.get(pk=datafile.pk) self.dataset = dataset self.datafiles = [create_datafile(i) for i in (1, 2)]
class MyTardisHSMUtilsTestCase(TestCase): """Test cases for the MyTardisHSM utils module""" def setUp(self): """Setup test fixtures if needed.""" self.user = User.objects.create_user("doctor", '', "pwd") self.exp = Experiment(title="Wonderful", institution_name="Monash University", created_by=self.user) self.exp.save() group = Group(name="Group1") group.save() facility = Facility(name="Test Facility", manager_group=group) facility.save() inst = Instrument(name="Test Instrument1", facility=facility) inst.save() self.dataset = Dataset(description="Dataset1", instrument=inst) self.dataset.save() storage_classes = getattr(settings, "HSM_STORAGE_CLASSES", DEFAULT_HSM_CLASSES) self.sbox1 = StorageBox(name="SBOX1", django_storage_class=storage_classes[0], status='online', max_size=256) self.sbox1.save() sbox1_attr = StorageBoxAttribute(storage_box=self.sbox1, key='type', value=StorageBox.DISK) sbox1_attr.save() sbox1_loc_opt = StorageBoxOption(storage_box=self.sbox1, key="location", value="/dummy/path") sbox1_loc_opt.save() self.sbox2 = StorageBox( name="SBOX2", django_storage_class="any.non.disk.StorageSystem", status='offline', max_size=256) self.sbox2.save() sbox2_attr = StorageBoxAttribute(storage_box=self.sbox2, key='type', value=StorageBox.TAPE) sbox2_attr.save() self.df1 = DataFile(dataset=self.dataset, filename="test_df.jpg") self.df1.save() self.dfo1 = DataFileObject(datafile=self.df1, storage_box=self.sbox1, uri="stream/test.jpg") self.dfo1.save() def tearDown(self): """Remove stuff""" def test_000_lock_datafile(self): """We should be able to lock a datafile to prevent concurrent access""" with DatafileLock(self.df1, "dummy_oid1") as lock1: if lock1: with DatafileLock(self.df1, "dummy_oid2") as lock2: self.assertTrue(lock1) self.assertFalse(lock2) def test_001_datafile_lock_expiry(self): """A datafile lock should not release until the expiry has been reached""" with DatafileLock(self.df1, "dummy_oid1", expires=2) as lock1: self.assertTrue(lock1) # If we retry lock right away, lock acquisition should fail because # expiry hasn't been reached with DatafileLock(self.df1, "dummy_oid1_1") as lock1: self.assertFalse(lock1) # wait 2s for lock to release time.sleep(2) # If we retry acquiring the lock now it should succeed with DatafileLock(self.df1, "dummy_oid1_2") as lock1: self.assertTrue(lock1)
class MyTardisHSMTasksTestCase(TestCase): """Tests for mytardis_hsm.tasks""" def setUp(self): """Setup test fixtures if needed.""" self.user = User.objects.create_user("doctor", '', "pwd") self.exp = Experiment(title="Wonderful", institution_name="Monash University", created_by=self.user) self.exp.save() group = Group(name="Group1") group.save() facility = Facility(name="Test Facility", manager_group=group) facility.save() inst = Instrument(name="Test Instrument1", facility=facility) inst.save() self.dataset = Dataset(description="Dataset1", instrument=inst) self.dataset.save() storage_classes = getattr(settings, "HSM_STORAGE_CLASSES", DEFAULT_HSM_CLASSES) self.sbox1 = StorageBox(name="SBOX1", django_storage_class=storage_classes[0], status='online', max_size=256) self.sbox1.save() sbox1_attr = StorageBoxAttribute(storage_box=self.sbox1, key='type', value=StorageBox.DISK) sbox1_attr.save() sbox1_loc_opt = StorageBoxOption(storage_box=self.sbox1, key="location", value=tempfile.gettempdir()) sbox1_loc_opt.save() self.sbox2 = StorageBox( name="SBOX2", django_storage_class="any.non.disk.StorageSystem", status='offline', max_size=256) self.sbox2.save() sbox2_attr = StorageBoxAttribute(storage_box=self.sbox2, key='type', value=StorageBox.TAPE) sbox2_attr.save() @mock.patch("os.stat") def test_000_update_df_status_offline(self, mock_stat): """update_df_status should check the online status of preferred DFOs for all previously online datafiles and update online Parameter to 'False' for any offline files.""" df1 = DataFile(dataset=self.dataset, filename="test_df.jpg") df1.save() dfo1 = DataFileObject(datafile=df1, storage_box=self.sbox1, uri="stream/test.jpg", verified=True) dfo1.save() schema = Schema.objects.get(namespace=HSM_DATAFILE_NAMESPACE) ps = DatafileParameterSet(schema=schema, datafile=df1) ps.save() param_name = ParameterName.objects.get(schema=schema, name="online") param = DatafileParameter(parameterset=ps, name=param_name) param.string_value = True param.save() mock_stat.return_value = Stats(st_size=10000, st_blocks=0, st_mtime=datetime.now()) update_df_status() params = DatafileParameter.objects.filter( parameterset__schema=schema, parameterset__datafile=df1) self.assertEquals(params.count(), 1) self.assertEquals(params[0].string_value, "False") @mock.patch("os.stat") def test_001_update_df_status_online(self, mock_stat): """update_df_status should check the online status of preferred DFOs for all previously online datafiles and leave the online Parameter as 'True' for any online files.""" df1 = DataFile(dataset=self.dataset, filename="test_df.jpg") df1.save() dfo1 = DataFileObject(datafile=df1, storage_box=self.sbox1, uri="stream/test.jpg", verified=True) dfo1.save() # df1.verify() schema = Schema.objects.get(namespace=HSM_DATAFILE_NAMESPACE) ps = DatafileParameterSet(schema=schema, datafile=df1) ps.save() param_name = ParameterName.objects.get(schema=schema, name="online") param = DatafileParameter(parameterset=ps, name=param_name) param.string_value = True param.save() mock_stat.return_value = Stats(st_size=10000, st_blocks=100, st_mtime=datetime.now()) update_df_status() params = DatafileParameter.objects.filter( parameterset__schema__namespace=HSM_DATAFILE_NAMESPACE, parameterset__datafile=df1) self.assertEquals(params.count(), 1) self.assertEquals(params[0].string_value, "True") @mock.patch('mytardis_hsm.mytardis_hsm.df_online') @mock.patch("os.stat") def test_002_update_df_status_skip_unverified(self, mock_stat, df_online): """update_df_status should skip files that are unverified""" df2 = DataFile(dataset=self.dataset, filename="test_df2.jpg") df2.save() dfo2 = DataFileObject(datafile=df2, storage_box=self.sbox1, uri="stream/test_df2.jpg") dfo2.save() schema = Schema.objects.get(namespace=HSM_DATAFILE_NAMESPACE) ps2 = DatafileParameterSet(schema=schema, datafile=df2) ps2.save() param_name = ParameterName.objects.get(schema=schema, name="online") param2 = DatafileParameter(parameterset=ps2, name=param_name) param2.string_value = True param2.save() mock_stat.return_value = Stats(st_size=10000, st_blocks=100, st_mtime=datetime.now()) update_df_status() df_online.assert_not_called() @mock.patch('mytardis_hsm.tasks.df_online', autopec=True) @mock.patch("os.stat") def test_003_update_df_status_skip_offline(self, mock_stat, mock_df_online): """update_df_status should skip any files that have previously marked as offline.""" df2 = DataFile(dataset=self.dataset, filename="test_df2.jpg") df2.save() dfo2 = DataFileObject(datafile=df2, storage_box=self.sbox1, uri="stream/test_df2.jpg", verified=True) dfo2.save() # df2.verify() schema = Schema.objects.get(namespace=HSM_DATAFILE_NAMESPACE) ps2 = DatafileParameterSet(schema=schema, datafile=df2) ps2.save() param_name = ParameterName.objects.get(schema=schema, name="online") param2 = DatafileParameter(parameterset=ps2, name=param_name) param2.string_value = False param2.save() mock_stat.return_value = Stats(st_size=10000, st_blocks=100, st_mtime=datetime.now()) update_df_status() # assert that the df_online method wasn't called self.assertEquals(mock_df_online.call_count, 0)
def setUp(self): self.test_box = StorageBox(name="test box", status="online", max_size=123) self.test_box.save()
class MyTardisHSMTFiltersTestCase(TestCase): """Testing MyTardis HSM App filters""" def setUp(self): """Setup test fixtures if needed.""" self.user = User.objects.create_user("doctor", '', "pwd") self.exp = Experiment(title="Wonderful", institution_name="Monash University", created_by=self.user) self.exp.save() group = Group(name="Group1") group.save() facility = Facility(name="Test Facility", manager_group=group) facility.save() self.inst = Instrument(name="Test Instrument1", facility=facility) self.inst.save() self.dataset = Dataset(description="Dataset1", instrument=self.inst) self.dataset.save() self.dataset.experiments.add(self.exp) storage_classes = getattr(settings, "HSM_STORAGE_CLASSES", DEFAULT_HSM_CLASSES) self.sbox1 = StorageBox(name="SBOX1", django_storage_class=storage_classes[0], status='online', max_size=256) self.sbox1.save() sbox1_attr = StorageBoxAttribute(storage_box=self.sbox1, key='type', value=StorageBox.DISK) sbox1_attr.save() sbox1_loc_opt = StorageBoxOption(storage_box=self.sbox1, key="location", value=tempfile.gettempdir()) sbox1_loc_opt.save() self.sbox2 = StorageBox( name="SBOX2", django_storage_class="any.non.disk.StorageSystem", status='offline', max_size=256) self.sbox2.save() sbox2_attr = StorageBoxAttribute(storage_box=self.sbox2, key='type', value=StorageBox.TAPE) sbox2_attr.save() @mock.patch("os.stat") def test_001_create_df_status(self, mock_stat): """When a new datafile record is verified, metadata for it's online/offline status should be created and populated with the current online status""" mock_stat.return_value = Stats(st_size=10000, st_blocks=100, st_mtime=datetime.now()) temp = tempfile.NamedTemporaryFile(dir=tempfile.gettempdir()) temp_name = os.path.basename(temp.name) df2 = DataFile(dataset=self.dataset, filename=temp_name) df2.save() dfo2 = DataFileObject(datafile=df2, storage_box=self.sbox1, uri=temp_name) dfo2.save() df2.verify() param_name = ParameterName.objects.get( schema__namespace=HSM_DATAFILE_NAMESPACE, name="online") paramset = DatafileParameterSet.objects.get( schema__namespace=HSM_DATAFILE_NAMESPACE, datafile=df2) param = DatafileParameter.objects.get(parameterset=paramset, name=param_name) self.assertEquals(param.string_value, "True") temp.close() @mock.patch("os.stat") def test_002_no_duplicate_params(self, mock_stat): """Datafile should only ever have one online param""" mock_stat.return_value = Stats(st_size=10000, st_blocks=100, st_mtime=datetime.now()) df1 = DataFile(dataset=self.dataset, filename="test_df.jpg") df1.save() dfo1 = DataFileObject(datafile=df1, storage_box=self.sbox1, uri="stream/test.jpg", verified=True) dfo1.save() df1.verify() param_name = ParameterName.objects.get( schema__namespace=HSM_DATAFILE_NAMESPACE, name="online") paramset = DatafileParameterSet.objects.get( schema__namespace=HSM_DATAFILE_NAMESPACE, datafile=df1) params = DatafileParameter.objects.filter(parameterset=paramset, name=param_name) self.assertEquals(params.count(), 1) self.assertRaises(OnlineParamExistsError, create_df_status(df1, HSM_DATAFILE_NAMESPACE, 500)) params = DatafileParameter.objects.filter(parameterset=paramset, name=param_name) self.assertEquals(params.count(), 1) @mock.patch("os.stat") def test_003_offline_dataset(self, mock_stat): """A dataset should be offline if any datafiles are offline""" mock_stat.return_value = Stats(st_size=10000, st_blocks=0, st_mtime=datetime.now()) ds = Dataset(description="Dataset2", instrument=self.inst) ds.save() df2 = DataFile(dataset=ds, filename="test_file.jpg") df2.save() dfo2 = DataFileObject(datafile=df2, storage_box=self.sbox1, uri=df2.filename) dfo2.save() df2.verify() self.assertFalse(dataset_online(ds)) @mock.patch("os.stat") def test_004_offline_experiment(self, mock_stat): """An experiment should be offline if any datafiles are offline""" mock_stat.return_value = Stats(st_size=10000, st_blocks=0, st_mtime=datetime.now()) ds = Dataset(description="Dataset2", instrument=self.inst) ds.save() ds.experiments.add(self.exp) df2 = DataFile(dataset=ds, filename="test_file.jpg") df2.save() dfo2 = DataFileObject(datafile=df2, storage_box=self.sbox1, uri=df2.filename) dfo2.save() df2.verify() self.assertFalse(experiment_online(self.exp))
class TestMytardisHSM(TestCase): """Tests for `mytardis_hsm` package.""" def setUp(self): """Setup test fixtures if needed.""" self.user = User.objects.create_user("doctor", '', "pwd") self.exp = Experiment(title="Wonderful", institution_name="Monash University", created_by=self.user) self.exp.save() group = Group(name="Group1") group.save() facility = Facility(name="Test Facility", manager_group=group) facility.save() self.inst = Instrument(name="Test Instrument1", facility=facility) self.inst.save() self.dataset = Dataset(description="Dataset1", instrument=self.inst) self.dataset.save() storage_classes = getattr(settings, "HSM_STORAGE_CLASSES", DEFAULT_HSM_CLASSES) self.sbox1 = StorageBox(name="SBOX1", django_storage_class=storage_classes[0], status='online', max_size=256) self.sbox1.save() sbox1_attr = StorageBoxAttribute(storage_box=self.sbox1, key='type', value=StorageBox.DISK) sbox1_attr.save() sbox1_loc_opt = StorageBoxOption(storage_box=self.sbox1, key="location", value="/dummy/path") sbox1_loc_opt.save() self.sbox2 = StorageBox( name="SBOX2", django_storage_class="any.non.disk.StorageSystem", status='offline', max_size=256) self.sbox2.save() sbox2_attr = StorageBoxAttribute(storage_box=self.sbox2, key='type', value=StorageBox.TAPE) sbox2_attr.save() self.df1 = DataFile(dataset=self.dataset, filename="test_df.jpg") self.df1.save() self.dfo1 = DataFileObject(datafile=self.df1, storage_box=self.sbox1, uri="stream/test.jpg", verified=True) self.dfo1.save() self.df1.verify() def tearDown(self): """Tear down test fixtures, if any.""" def test_000_number_of_users(self): """Checks the number of users in the database""" user = User.objects.all().count() self.assertEqual(user, 1) def test_001_number_of_experiments(self): """Creates an experiments, saves it and then checks the number of experiments in the database is equal to 1. """ exps = Experiment.objects.all() self.assertEqual(exps.count(), 1) def test_003_number_of_datasets(self): """Checks that number of experiments is equal to 1 """ self.assertEqual(Dataset.objects.all().count(), 1) @mock.patch("os.stat") def test_003_dfo_online(self, mock_stat): """HSM.online should return True when a DFOs underlying file has > 0 blocks""" mock_stat.return_value = Stats(st_size=10000, st_blocks=100, st_mtime=datetime.now()) self.assertTrue(dfo_online(self.dfo1)) @mock.patch("os.stat") def test_004_dfo_offline(self, mock_stat): """HSM.online should return False when a DFOs underlying file is > 350 bytes and 0 blocks""" mock_stat.return_value = Stats(st_size=10000, st_blocks=0, st_mtime=datetime.now()) self.assertFalse(dfo_online(self.dfo1)) def test_005_dfo_non_disk(self): """Files in StorageBoxes with a django_storage_class other than those specified in settings should not be processed""" dfo2 = DataFileObject(datafile=self.df1, storage_box=self.sbox2, uri="stream/test.jpg", verified=True) self.assertRaises(StorageClassNotSupportedError, dfo_online, dfo2) with self.settings(HSM_STORAGE_CLASSES=["random.storage.CLASS"]): self.assertRaises(StorageClassNotSupportedError, dfo_online, self.dfo1) def test_006_hsm_schema(self): """HSM schema should be installed""" schemas = Schema.objects\ .filter(namespace="http://tardis.edu.au/schemas/hsm/datafile/1")\ .count() self.assertEqual(schemas, 1) def test_007_dfo_unverified(self): """df_online and dfo_online should raise Exception for an unverfied DataFile or DataFileObject, respectively""" df2 = DataFile(dataset=self.dataset, filename="test_df.jpg") df2.save() self.assertRaises(DataFileNotVerified, df_online, df2) dfo2 = DataFileObject(datafile=df2, storage_box=self.sbox1, uri="stream/test.jpg", verified=False) dfo2.save() self.assertRaises(DataFileObjectNotVerified, dfo_online, dfo2)
def setUp(self): self.test_box = StorageBox(name='test box', status='online', max_size=123) self.test_box.save()