def test_001_create_df_status(self, mock_stat):
"""When a new datafile record is verified, metadata for it's
online/offline status should be created and populated with the
current online status"""
mock_stat.return_value = Stats(st_size=10000,
st_blocks=100,
st_mtime=datetime.now())
temp = tempfile.NamedTemporaryFile(dir=tempfile.gettempdir())
temp_name = os.path.basename(temp.name)
df2 = DataFile(dataset=self.dataset, filename=temp_name)
df2.save()
dfo2 = DataFileObject(datafile=df2,
storage_box=self.sbox1,
uri=temp_name)
dfo2.save()
df2.verify()
param_name = ParameterName.objects.get(
schema__namespace=HSM_DATAFILE_NAMESPACE, name="online")
paramset = DatafileParameterSet.objects.get(
schema__namespace=HSM_DATAFILE_NAMESPACE, datafile=df2)
param = DatafileParameter.objects.get(parameterset=paramset,
name=param_name)
self.assertEquals(param.string_value, "True")
temp.close()
def test_002_no_duplicate_params(self, mock_stat):
"""Datafile should only ever have one online param"""
mock_stat.return_value = Stats(st_size=10000,
st_blocks=100,
st_mtime=datetime.now())
df1 = DataFile(dataset=self.dataset, filename="test_df.jpg")
df1.save()
dfo1 = DataFileObject(datafile=df1,
storage_box=self.sbox1,
uri="stream/test.jpg",
verified=True)
dfo1.save()
df1.verify()
param_name = ParameterName.objects.get(
schema__namespace=HSM_DATAFILE_NAMESPACE, name="online")
paramset = DatafileParameterSet.objects.get(
schema__namespace=HSM_DATAFILE_NAMESPACE, datafile=df1)
params = DatafileParameter.objects.filter(parameterset=paramset,
name=param_name)
self.assertEquals(params.count(), 1)
self.assertRaises(OnlineParamExistsError,
create_df_status(df1, HSM_DATAFILE_NAMESPACE, 500))
params = DatafileParameter.objects.filter(parameterset=paramset,
name=param_name)
self.assertEquals(params.count(), 1)
def test_wrong_size_verification(self):
content = urandom(1024)
cf = ContentFile(content, 'background_task_testfile')
# Create new Datafile
datafile = DataFile(dataset=self.dataset)
datafile.filename = cf.name
datafile.size = len(content) - 1
datafile.sha512sum = hashlib.sha512(content).hexdigest()
datafile.save()
datafile.file_object = cf
# verify explicitly to catch Exceptions hidden by celery
datafile.verify()
self.assertFalse(datafile.file_objects.get().verified)
def test_wrong_size_verification(self):
content = urandom(1024)
cf = ContentFile(content, 'background_task_testfile')
# Create new Datafile
datafile = DataFile(dataset=self.dataset)
datafile.filename = cf.name
datafile.size = len(content) - 1
datafile.sha512sum = hashlib.sha512(content).hexdigest()
datafile.save()
datafile.file_object = cf
# verify explicitly to catch Exceptions hidden by celery
datafile.verify()
self.assertFalse(datafile.file_objects.get().verified)
def test_003_offline_dataset(self, mock_stat):
"""A dataset should be offline if any datafiles are offline"""
mock_stat.return_value = Stats(st_size=10000,
st_blocks=0,
st_mtime=datetime.now())
ds = Dataset(description="Dataset2", instrument=self.inst)
ds.save()
df2 = DataFile(dataset=ds, filename="test_file.jpg")
df2.save()
dfo2 = DataFileObject(datafile=df2,
storage_box=self.sbox1,
uri=df2.filename)
dfo2.save()
df2.verify()
self.assertFalse(dataset_online(ds))
class TestMytardisHSM(TestCase):
"""Tests for `mytardis_hsm` package."""
def setUp(self):
"""Setup test fixtures if needed."""
self.user = User.objects.create_user("doctor", '', "pwd")
self.exp = Experiment(title="Wonderful",
institution_name="Monash University",
created_by=self.user)
self.exp.save()
group = Group(name="Group1")
group.save()
facility = Facility(name="Test Facility", manager_group=group)
facility.save()
self.inst = Instrument(name="Test Instrument1", facility=facility)
self.inst.save()
self.dataset = Dataset(description="Dataset1", instrument=self.inst)
self.dataset.save()
storage_classes = getattr(settings, "HSM_STORAGE_CLASSES",
DEFAULT_HSM_CLASSES)
self.sbox1 = StorageBox(name="SBOX1",
django_storage_class=storage_classes[0],
status='online',
max_size=256)
self.sbox1.save()
sbox1_attr = StorageBoxAttribute(storage_box=self.sbox1,
key='type',
value=StorageBox.DISK)
sbox1_attr.save()
sbox1_loc_opt = StorageBoxOption(storage_box=self.sbox1,
key="location",
value="/dummy/path")
sbox1_loc_opt.save()
self.sbox2 = StorageBox(
name="SBOX2",
django_storage_class="any.non.disk.StorageSystem",
status='offline',
max_size=256)
self.sbox2.save()
sbox2_attr = StorageBoxAttribute(storage_box=self.sbox2,
key='type',
value=StorageBox.TAPE)
sbox2_attr.save()
self.df1 = DataFile(dataset=self.dataset, filename="test_df.jpg")
self.df1.save()
self.dfo1 = DataFileObject(datafile=self.df1,
storage_box=self.sbox1,
uri="stream/test.jpg",
verified=True)
self.dfo1.save()
self.df1.verify()
def tearDown(self):
"""Tear down test fixtures, if any."""
def test_000_number_of_users(self):
"""Checks the number of users in the database"""
user = User.objects.all().count()
self.assertEqual(user, 1)
def test_001_number_of_experiments(self):
"""Creates an experiments, saves it and then checks the
number of experiments in the database is equal to 1.
"""
exps = Experiment.objects.all()
self.assertEqual(exps.count(), 1)
def test_003_number_of_datasets(self):
"""Checks that number of experiments is equal to 1
"""
self.assertEqual(Dataset.objects.all().count(), 1)
@mock.patch("os.stat")
def test_003_dfo_online(self, mock_stat):
"""HSM.online should return True when a DFOs underlying file
has > 0 blocks"""
mock_stat.return_value = Stats(st_size=10000,
st_blocks=100,
st_mtime=datetime.now())
self.assertTrue(dfo_online(self.dfo1))
@mock.patch("os.stat")
def test_004_dfo_offline(self, mock_stat):
"""HSM.online should return False when a DFOs underlying file
is > 350 bytes and 0 blocks"""
mock_stat.return_value = Stats(st_size=10000,
st_blocks=0,
st_mtime=datetime.now())
self.assertFalse(dfo_online(self.dfo1))
def test_005_dfo_non_disk(self):
"""Files in StorageBoxes with a django_storage_class other than
those specified in settings should not be processed"""
dfo2 = DataFileObject(datafile=self.df1,
storage_box=self.sbox2,
uri="stream/test.jpg",
verified=True)
self.assertRaises(StorageClassNotSupportedError, dfo_online, dfo2)
with self.settings(HSM_STORAGE_CLASSES=["random.storage.CLASS"]):
self.assertRaises(StorageClassNotSupportedError, dfo_online,
self.dfo1)
def test_006_hsm_schema(self):
"""HSM schema should be installed"""
schemas = Schema.objects\
.filter(namespace="http://tardis.edu.au/schemas/hsm/datafile/1")\
.count()
self.assertEqual(schemas, 1)
def test_007_dfo_unverified(self):
"""df_online and dfo_online should raise Exception for an unverfied DataFile or
DataFileObject, respectively"""
df2 = DataFile(dataset=self.dataset, filename="test_df.jpg")
df2.save()
self.assertRaises(DataFileNotVerified, df_online, df2)
dfo2 = DataFileObject(datafile=df2,
storage_box=self.sbox1,
uri="stream/test.jpg",
verified=False)
dfo2.save()
self.assertRaises(DataFileObjectNotVerified, dfo_online, dfo2)
