class ModelTestCase(TestCase):
def setUp(self):
self.test_box = StorageBox(name='test box',
status='online',
max_size=123)
self.test_box.save()
def test_storageboxoption(self):
string_input = 'some credential'
object_input = {
'a dictionary': 'of different things',
'booleans': True,
'and nothing': None
}
string_option = StorageBoxOption(storage_box=self.test_box,
key='an_option',
value=string_input)
string_option.save()
object_option = StorageBoxOption(storage_box=self.test_box,
key='optional',
value_type=StorageBoxOption.PICKLE)
object_option.unpickled_value = object_input
object_option.save()
options_dict = self.test_box.get_options_as_dict()
self.assertEqual(options_dict['an_option'], string_input)
self.assertEqual(options_dict['optional'], object_input)
def tearDown(self):
self.test_box.delete()
def setUp(self):
"""Setup test fixtures if needed."""
self.user = User.objects.create_user("doctor", '', "pwd")
self.exp = Experiment(title="Wonderful",
institution_name="Monash University",
created_by=self.user)
self.exp.save()
group = Group(name="Group1")
group.save()
facility = Facility(name="Test Facility", manager_group=group)
facility.save()
self.inst = Instrument(name="Test Instrument1", facility=facility)
self.inst.save()
self.dataset = Dataset(description="Dataset1", instrument=self.inst)
self.dataset.save()
storage_classes = getattr(settings, "HSM_STORAGE_CLASSES",
DEFAULT_HSM_CLASSES)
self.sbox1 = StorageBox(name="SBOX1",
django_storage_class=storage_classes[0],
status='online',
max_size=256)
self.sbox1.save()
sbox1_attr = StorageBoxAttribute(storage_box=self.sbox1,
key='type',
value=StorageBox.DISK)
sbox1_attr.save()
sbox1_loc_opt = StorageBoxOption(storage_box=self.sbox1,
key="location",
value="/dummy/path")
sbox1_loc_opt.save()
self.sbox2 = StorageBox(
name="SBOX2",
django_storage_class="any.non.disk.StorageSystem",
status='offline',
max_size=256)
self.sbox2.save()
sbox2_attr = StorageBoxAttribute(storage_box=self.sbox2,
key='type',
value=StorageBox.TAPE)
sbox2_attr.save()
self.df1 = DataFile(dataset=self.dataset, filename="test_df.jpg")
self.df1.save()
self.dfo1 = DataFileObject(datafile=self.df1,
storage_box=self.sbox1,
uri="stream/test.jpg",
verified=True)
self.dfo1.save()
self.df1.verify()
def setUp(self):
raise SkipTest # temporarily disabling this feature, needs coding
from tempfile import mkdtemp, mktemp
from django.conf import settings
import os
# Disconnect post_save signal
from django.db.models.signals import post_save
from tardis.tardis_portal.models import Experiment, \
staging_hook, Dataset, DataFile, DataFileObject, StorageBox
post_save.disconnect(staging_hook, sender=DataFileObject)
from django.contrib.auth.models import User
user = '******'
pwd = 'secret'
email = ''
self.user = User.objects.create_user(user, email, pwd)
try:
os.makedirs(settings.GET_FULL_STAGING_PATH_TEST)
except OSError:
pass
self.temp = mkdtemp(dir=settings.GET_FULL_STAGING_PATH_TEST)
self.filepath = mktemp(dir=self.temp)
content = 'test file'
with open(self.filepath, "w+b") as f:
f.write(content)
# make datafile
exp = Experiment(title='test exp1',
institution_name='monash',
created_by=self.user)
exp.save()
# make dataset
dataset = Dataset(description="dataset description...")
dataset.save()
dataset.experiments.add(exp)
dataset.save()
# create datafile
df = DataFile(dataset=dataset, size=len(content),
filename=path.basename(self.file),
md5sum='f20d9f2072bbeb6691c0f9c5099b01f3')
df.save()
# create replica
base_url = settings.GET_FULL_STAGING_PATH_TEST
s_box = StorageBox.get_default_storage(location=base_url)
dfo = DataFileObject(datafile=df,
uri=self.filepath,
storage_box=s_box)
dfo.save()
self.dfo = dfo
class ModelTestCase(TestCase):
def setUp(self):
self.test_box = StorageBox(name="test box", status="online", max_size=123)
self.test_box.save()
def test_storageboxoption(self):
string_input = "some credential"
object_input = {"a dictionary": "of different things", "booleans": True, "and nothing": None}
string_option = StorageBoxOption(storage_box=self.test_box, key="an_option", value=string_input)
string_option.save()
object_option = StorageBoxOption(storage_box=self.test_box, key="optional", value_type=StorageBoxOption.PICKLE)
object_option.unpickled_value = object_input
object_option.save()
options_dict = self.test_box.get_options_as_dict()
self.assertEqual(options_dict["an_option"], string_input)
self.assertEqual(options_dict["optional"], object_input)
def tearDown(self):
self.test_box.delete()
def get_or_create_storage_box(datafile):
key_name = 'datafile_id'
class_name = 'tardis.tardis_portal.storage.squashfs.SquashFSStorage'
try:
s_box = StorageBoxOption.objects.get(
key=key_name,
value=datafile.id,
storage_box__django_storage_class=class_name).storage_box
except StorageBoxOption.DoesNotExist:
s_box = StorageBox(
django_storage_class=class_name,
max_size=datafile.size,
status='empty',
name=datafile.filename,
description='SquashFS Archive in DataFile id: %d, filename: %s' %
(datafile.id, datafile.filename))
s_box.save()
StorageBoxOption(key=key_name, value=datafile.id,
storage_box=s_box).save()
return s_box
def setUp(self):
# Create test owner without enough details
username, email, password = ('testuser',
'*****@*****.**',
'password')
user = User.objects.create_user(username, email, password)
# Create test experiment and make user the owner of it
experiment = Experiment(title='Text Experiment',
institution_name='Test Uni',
created_by=user)
experiment.save()
acl = ObjectACL(
pluginId='django_user',
entityId=str(user.id),
content_object=experiment,
canRead=True,
isOwner=True,
aclOwnershipType=ObjectACL.OWNER_OWNED,
)
acl.save()
dataset = Dataset(description='dataset description...')
dataset.save()
dataset.experiments.add(experiment)
dataset.save()
base_path = path.join(path.dirname(__file__), 'fixtures')
s_box = StorageBox.get_default_storage(location=base_path)
def create_datafile(index):
testfile = path.join(base_path, 'middleware_test%d.txt' % index)
size, sha512sum = get_size_and_sha512sum(testfile)
datafile = DataFile(dataset=dataset,
filename=path.basename(testfile),
size=size,
sha512sum=sha512sum)
datafile.save()
dfo = DataFileObject(
datafile=datafile,
storage_box=s_box,
uri=path.basename(testfile))
dfo.save()
if index != 1:
dfo.verified = False
dfo.save(update_fields=['verified'])
return DataFile.objects.get(pk=datafile.pk)
self.dataset = dataset
self.datafiles = [create_datafile(i) for i in (1, 2)]
class MyTardisHSMUtilsTestCase(TestCase):
"""Test cases for the MyTardisHSM utils module"""
def setUp(self):
"""Setup test fixtures if needed."""
self.user = User.objects.create_user("doctor", '', "pwd")
self.exp = Experiment(title="Wonderful",
institution_name="Monash University",
created_by=self.user)
self.exp.save()
group = Group(name="Group1")
group.save()
facility = Facility(name="Test Facility", manager_group=group)
facility.save()
inst = Instrument(name="Test Instrument1", facility=facility)
inst.save()
self.dataset = Dataset(description="Dataset1", instrument=inst)
self.dataset.save()
storage_classes = getattr(settings, "HSM_STORAGE_CLASSES",
DEFAULT_HSM_CLASSES)
self.sbox1 = StorageBox(name="SBOX1",
django_storage_class=storage_classes[0],
status='online',
max_size=256)
self.sbox1.save()
sbox1_attr = StorageBoxAttribute(storage_box=self.sbox1,
key='type',
value=StorageBox.DISK)
sbox1_attr.save()
sbox1_loc_opt = StorageBoxOption(storage_box=self.sbox1,
key="location",
value="/dummy/path")
sbox1_loc_opt.save()
self.sbox2 = StorageBox(
name="SBOX2",
django_storage_class="any.non.disk.StorageSystem",
status='offline',
max_size=256)
self.sbox2.save()
sbox2_attr = StorageBoxAttribute(storage_box=self.sbox2,
key='type',
value=StorageBox.TAPE)
sbox2_attr.save()
self.df1 = DataFile(dataset=self.dataset, filename="test_df.jpg")
self.df1.save()
self.dfo1 = DataFileObject(datafile=self.df1,
storage_box=self.sbox1,
uri="stream/test.jpg")
self.dfo1.save()
def tearDown(self):
"""Remove stuff"""
def test_000_lock_datafile(self):
"""We should be able to lock a datafile to prevent concurrent access"""
with DatafileLock(self.df1, "dummy_oid1") as lock1:
if lock1:
with DatafileLock(self.df1, "dummy_oid2") as lock2:
self.assertTrue(lock1)
self.assertFalse(lock2)
def test_001_datafile_lock_expiry(self):
"""A datafile lock should not release until the expiry has been
reached"""
with DatafileLock(self.df1, "dummy_oid1", expires=2) as lock1:
self.assertTrue(lock1)
# If we retry lock right away, lock acquisition should fail because
# expiry hasn't been reached
with DatafileLock(self.df1, "dummy_oid1_1") as lock1:
self.assertFalse(lock1)
# wait 2s for lock to release
time.sleep(2)
# If we retry acquiring the lock now it should succeed
with DatafileLock(self.df1, "dummy_oid1_2") as lock1:
self.assertTrue(lock1)
class MyTardisHSMTasksTestCase(TestCase):
"""Tests for mytardis_hsm.tasks"""
def setUp(self):
"""Setup test fixtures if needed."""
self.user = User.objects.create_user("doctor", '',
"pwd")
self.exp = Experiment(title="Wonderful",
institution_name="Monash University",
created_by=self.user)
self.exp.save()
group = Group(name="Group1")
group.save()
facility = Facility(name="Test Facility",
manager_group=group)
facility.save()
inst = Instrument(name="Test Instrument1",
facility=facility)
inst.save()
self.dataset = Dataset(description="Dataset1",
instrument=inst)
self.dataset.save()
storage_classes = getattr(settings,
"HSM_STORAGE_CLASSES",
DEFAULT_HSM_CLASSES)
self.sbox1 = StorageBox(name="SBOX1",
django_storage_class=storage_classes[0],
status='online', max_size=256)
self.sbox1.save()
sbox1_attr = StorageBoxAttribute(storage_box=self.sbox1,
key='type',
value=StorageBox.DISK)
sbox1_attr.save()
sbox1_loc_opt = StorageBoxOption(storage_box=self.sbox1,
key="location",
value=tempfile.gettempdir())
sbox1_loc_opt.save()
self.sbox2 = StorageBox(
name="SBOX2",
django_storage_class="any.non.disk.StorageSystem",
status='offline', max_size=256)
self.sbox2.save()
sbox2_attr = StorageBoxAttribute(storage_box=self.sbox2,
key='type',
value=StorageBox.TAPE)
sbox2_attr.save()
@mock.patch("os.stat")
def test_000_update_df_status_offline(self, mock_stat):
"""update_df_status should check the online status of
preferred DFOs for all previously online datafiles and
update online Parameter to 'False' for any offline files."""
df1 = DataFile(dataset=self.dataset,
filename="test_df.jpg")
df1.save()
dfo1 = DataFileObject(datafile=df1,
storage_box=self.sbox1,
uri="stream/test.jpg",
verified=True)
dfo1.save()
schema = Schema.objects.get(namespace=HSM_DATAFILE_NAMESPACE)
ps = DatafileParameterSet(schema=schema, datafile=df1)
ps.save()
param_name = ParameterName.objects.get(schema=schema, name="online")
param = DatafileParameter(parameterset=ps, name=param_name)
param.string_value = True
param.save()
mock_stat.return_value = Stats(st_size=10000,
st_blocks=0,
st_mtime=datetime.now())
update_df_status()
params = DatafileParameter.objects.filter(
parameterset__schema=schema,
parameterset__datafile=df1)
self.assertEquals(params.count(), 1)
self.assertEquals(params[0].string_value, "False")
@mock.patch("os.stat")
def test_001_update_df_status_online(self, mock_stat):
"""update_df_status should check the online status of
preferred DFOs for all previously online datafiles and
leave the online Parameter as 'True' for any online files."""
df1 = DataFile(dataset=self.dataset,
filename="test_df.jpg")
df1.save()
dfo1 = DataFileObject(datafile=df1,
storage_box=self.sbox1,
uri="stream/test.jpg",
verified=True)
dfo1.save()
# df1.verify()
schema = Schema.objects.get(namespace=HSM_DATAFILE_NAMESPACE)
ps = DatafileParameterSet(schema=schema, datafile=df1)
ps.save()
param_name = ParameterName.objects.get(schema=schema, name="online")
param = DatafileParameter(parameterset=ps, name=param_name)
param.string_value = True
param.save()
mock_stat.return_value = Stats(st_size=10000,
st_blocks=100,
st_mtime=datetime.now())
update_df_status()
params = DatafileParameter.objects.filter(
parameterset__schema__namespace=HSM_DATAFILE_NAMESPACE,
parameterset__datafile=df1)
self.assertEquals(params.count(), 1)
self.assertEquals(params[0].string_value, "True")
@mock.patch('mytardis_hsm.mytardis_hsm.df_online')
@mock.patch("os.stat")
def test_002_update_df_status_skip_unverified(self, mock_stat, df_online):
"""update_df_status should skip files that are unverified"""
df2 = DataFile(dataset=self.dataset,
filename="test_df2.jpg")
df2.save()
dfo2 = DataFileObject(datafile=df2,
storage_box=self.sbox1,
uri="stream/test_df2.jpg")
dfo2.save()
schema = Schema.objects.get(namespace=HSM_DATAFILE_NAMESPACE)
ps2 = DatafileParameterSet(schema=schema, datafile=df2)
ps2.save()
param_name = ParameterName.objects.get(schema=schema, name="online")
param2 = DatafileParameter(parameterset=ps2, name=param_name)
param2.string_value = True
param2.save()
mock_stat.return_value = Stats(st_size=10000,
st_blocks=100,
st_mtime=datetime.now())
update_df_status()
df_online.assert_not_called()
@mock.patch('mytardis_hsm.tasks.df_online', autopec=True)
@mock.patch("os.stat")
def test_003_update_df_status_skip_offline(self, mock_stat, mock_df_online):
"""update_df_status should skip any files that have previously
marked as offline."""
df2 = DataFile(dataset=self.dataset,
filename="test_df2.jpg")
df2.save()
dfo2 = DataFileObject(datafile=df2,
storage_box=self.sbox1,
uri="stream/test_df2.jpg",
verified=True)
dfo2.save()
# df2.verify()
schema = Schema.objects.get(namespace=HSM_DATAFILE_NAMESPACE)
ps2 = DatafileParameterSet(schema=schema, datafile=df2)
ps2.save()
param_name = ParameterName.objects.get(schema=schema, name="online")
param2 = DatafileParameter(parameterset=ps2, name=param_name)
param2.string_value = False
param2.save()
mock_stat.return_value = Stats(st_size=10000,
st_blocks=100,
st_mtime=datetime.now())
update_df_status()
# assert that the df_online method wasn't called
self.assertEquals(mock_df_online.call_count, 0)
def setUp(self):
self.test_box = StorageBox(name="test box", status="online", max_size=123)
self.test_box.save()
class MyTardisHSMTFiltersTestCase(TestCase):
"""Testing MyTardis HSM App filters"""
def setUp(self):
"""Setup test fixtures if needed."""
self.user = User.objects.create_user("doctor", '', "pwd")
self.exp = Experiment(title="Wonderful",
institution_name="Monash University",
created_by=self.user)
self.exp.save()
group = Group(name="Group1")
group.save()
facility = Facility(name="Test Facility", manager_group=group)
facility.save()
self.inst = Instrument(name="Test Instrument1", facility=facility)
self.inst.save()
self.dataset = Dataset(description="Dataset1", instrument=self.inst)
self.dataset.save()
self.dataset.experiments.add(self.exp)
storage_classes = getattr(settings, "HSM_STORAGE_CLASSES",
DEFAULT_HSM_CLASSES)
self.sbox1 = StorageBox(name="SBOX1",
django_storage_class=storage_classes[0],
status='online',
max_size=256)
self.sbox1.save()
sbox1_attr = StorageBoxAttribute(storage_box=self.sbox1,
key='type',
value=StorageBox.DISK)
sbox1_attr.save()
sbox1_loc_opt = StorageBoxOption(storage_box=self.sbox1,
key="location",
value=tempfile.gettempdir())
sbox1_loc_opt.save()
self.sbox2 = StorageBox(
name="SBOX2",
django_storage_class="any.non.disk.StorageSystem",
status='offline',
max_size=256)
self.sbox2.save()
sbox2_attr = StorageBoxAttribute(storage_box=self.sbox2,
key='type',
value=StorageBox.TAPE)
sbox2_attr.save()
@mock.patch("os.stat")
def test_001_create_df_status(self, mock_stat):
"""When a new datafile record is verified, metadata for it's
online/offline status should be created and populated with the
current online status"""
mock_stat.return_value = Stats(st_size=10000,
st_blocks=100,
st_mtime=datetime.now())
temp = tempfile.NamedTemporaryFile(dir=tempfile.gettempdir())
temp_name = os.path.basename(temp.name)
df2 = DataFile(dataset=self.dataset, filename=temp_name)
df2.save()
dfo2 = DataFileObject(datafile=df2,
storage_box=self.sbox1,
uri=temp_name)
dfo2.save()
df2.verify()
param_name = ParameterName.objects.get(
schema__namespace=HSM_DATAFILE_NAMESPACE, name="online")
paramset = DatafileParameterSet.objects.get(
schema__namespace=HSM_DATAFILE_NAMESPACE, datafile=df2)
param = DatafileParameter.objects.get(parameterset=paramset,
name=param_name)
self.assertEquals(param.string_value, "True")
temp.close()
@mock.patch("os.stat")
def test_002_no_duplicate_params(self, mock_stat):
"""Datafile should only ever have one online param"""
mock_stat.return_value = Stats(st_size=10000,
st_blocks=100,
st_mtime=datetime.now())
df1 = DataFile(dataset=self.dataset, filename="test_df.jpg")
df1.save()
dfo1 = DataFileObject(datafile=df1,
storage_box=self.sbox1,
uri="stream/test.jpg",
verified=True)
dfo1.save()
df1.verify()
param_name = ParameterName.objects.get(
schema__namespace=HSM_DATAFILE_NAMESPACE, name="online")
paramset = DatafileParameterSet.objects.get(
schema__namespace=HSM_DATAFILE_NAMESPACE, datafile=df1)
params = DatafileParameter.objects.filter(parameterset=paramset,
name=param_name)
self.assertEquals(params.count(), 1)
self.assertRaises(OnlineParamExistsError,
create_df_status(df1, HSM_DATAFILE_NAMESPACE, 500))
params = DatafileParameter.objects.filter(parameterset=paramset,
name=param_name)
self.assertEquals(params.count(), 1)
@mock.patch("os.stat")
def test_003_offline_dataset(self, mock_stat):
"""A dataset should be offline if any datafiles are offline"""
mock_stat.return_value = Stats(st_size=10000,
st_blocks=0,
st_mtime=datetime.now())
ds = Dataset(description="Dataset2", instrument=self.inst)
ds.save()
df2 = DataFile(dataset=ds, filename="test_file.jpg")
df2.save()
dfo2 = DataFileObject(datafile=df2,
storage_box=self.sbox1,
uri=df2.filename)
dfo2.save()
df2.verify()
self.assertFalse(dataset_online(ds))
@mock.patch("os.stat")
def test_004_offline_experiment(self, mock_stat):
"""An experiment should be offline if any datafiles are offline"""
mock_stat.return_value = Stats(st_size=10000,
st_blocks=0,
st_mtime=datetime.now())
ds = Dataset(description="Dataset2", instrument=self.inst)
ds.save()
ds.experiments.add(self.exp)
df2 = DataFile(dataset=ds, filename="test_file.jpg")
df2.save()
dfo2 = DataFileObject(datafile=df2,
storage_box=self.sbox1,
uri=df2.filename)
dfo2.save()
df2.verify()
self.assertFalse(experiment_online(self.exp))
class TestMytardisHSM(TestCase):
"""Tests for `mytardis_hsm` package."""
def setUp(self):
"""Setup test fixtures if needed."""
self.user = User.objects.create_user("doctor", '', "pwd")
self.exp = Experiment(title="Wonderful",
institution_name="Monash University",
created_by=self.user)
self.exp.save()
group = Group(name="Group1")
group.save()
facility = Facility(name="Test Facility", manager_group=group)
facility.save()
self.inst = Instrument(name="Test Instrument1", facility=facility)
self.inst.save()
self.dataset = Dataset(description="Dataset1", instrument=self.inst)
self.dataset.save()
storage_classes = getattr(settings, "HSM_STORAGE_CLASSES",
DEFAULT_HSM_CLASSES)
self.sbox1 = StorageBox(name="SBOX1",
django_storage_class=storage_classes[0],
status='online',
max_size=256)
self.sbox1.save()
sbox1_attr = StorageBoxAttribute(storage_box=self.sbox1,
key='type',
value=StorageBox.DISK)
sbox1_attr.save()
sbox1_loc_opt = StorageBoxOption(storage_box=self.sbox1,
key="location",
value="/dummy/path")
sbox1_loc_opt.save()
self.sbox2 = StorageBox(
name="SBOX2",
django_storage_class="any.non.disk.StorageSystem",
status='offline',
max_size=256)
self.sbox2.save()
sbox2_attr = StorageBoxAttribute(storage_box=self.sbox2,
key='type',
value=StorageBox.TAPE)
sbox2_attr.save()
self.df1 = DataFile(dataset=self.dataset, filename="test_df.jpg")
self.df1.save()
self.dfo1 = DataFileObject(datafile=self.df1,
storage_box=self.sbox1,
uri="stream/test.jpg",
verified=True)
self.dfo1.save()
self.df1.verify()
def tearDown(self):
"""Tear down test fixtures, if any."""
def test_000_number_of_users(self):
"""Checks the number of users in the database"""
user = User.objects.all().count()
self.assertEqual(user, 1)
def test_001_number_of_experiments(self):
"""Creates an experiments, saves it and then checks the
number of experiments in the database is equal to 1.
"""
exps = Experiment.objects.all()
self.assertEqual(exps.count(), 1)
def test_003_number_of_datasets(self):
"""Checks that number of experiments is equal to 1
"""
self.assertEqual(Dataset.objects.all().count(), 1)
@mock.patch("os.stat")
def test_003_dfo_online(self, mock_stat):
"""HSM.online should return True when a DFOs underlying file
has > 0 blocks"""
mock_stat.return_value = Stats(st_size=10000,
st_blocks=100,
st_mtime=datetime.now())
self.assertTrue(dfo_online(self.dfo1))
@mock.patch("os.stat")
def test_004_dfo_offline(self, mock_stat):
"""HSM.online should return False when a DFOs underlying file
is > 350 bytes and 0 blocks"""
mock_stat.return_value = Stats(st_size=10000,
st_blocks=0,
st_mtime=datetime.now())
self.assertFalse(dfo_online(self.dfo1))
def test_005_dfo_non_disk(self):
"""Files in StorageBoxes with a django_storage_class other than
those specified in settings should not be processed"""
dfo2 = DataFileObject(datafile=self.df1,
storage_box=self.sbox2,
uri="stream/test.jpg",
verified=True)
self.assertRaises(StorageClassNotSupportedError, dfo_online, dfo2)
with self.settings(HSM_STORAGE_CLASSES=["random.storage.CLASS"]):
self.assertRaises(StorageClassNotSupportedError, dfo_online,
self.dfo1)
def test_006_hsm_schema(self):
"""HSM schema should be installed"""
schemas = Schema.objects\
.filter(namespace="http://tardis.edu.au/schemas/hsm/datafile/1")\
.count()
self.assertEqual(schemas, 1)
def test_007_dfo_unverified(self):
"""df_online and dfo_online should raise Exception for an unverfied DataFile or
DataFileObject, respectively"""
df2 = DataFile(dataset=self.dataset, filename="test_df.jpg")
df2.save()
self.assertRaises(DataFileNotVerified, df_online, df2)
dfo2 = DataFileObject(datafile=df2,
storage_box=self.sbox1,
uri="stream/test.jpg",
verified=False)
dfo2.save()
self.assertRaises(DataFileObjectNotVerified, dfo_online, dfo2)
def setUp(self):
self.test_box = StorageBox(name='test box',
status='online',
max_size=123)
self.test_box.save()
