def register_squashfile(exp_id, epn, sq_dir, sq_filename, namespace):
'''
example:
register_squashfile(456, '1234A', '/srv/squashstore', '1234A.squashfs',
'http://synchrotron.org.au/mx/squashfsarchive/1')
'''
dfs = DataFile.objects.filter(filename=sq_filename,
dataset__experiments__id=exp_id)
if len(dfs) == 1:
return dfs[0]
e = Experiment.objects.get(id=exp_id)
ds = Dataset(description="01 SquashFS Archive")
ds.save()
ds.experiments.add(e)
filepath = os.path.join(sq_dir, sq_filename)
try:
md5sum = open(filepath + '.md5sum', 'r').read().strip()[:32]
except:
print 'no md5sum file found'
return None
size = os.path.getsize(filepath)
df = DataFile(md5sum=md5sum,
filename=sq_filename,
size=str(size),
dataset=ds)
df.save()
schema = Schema.objects.filter(namespace=namespace)[0]
ps = DatafileParameterSet(schema=schema, datafile=df)
ps.save()
ps.set_param('EPN', epn)
sbox = StorageBox.objects.get(name='squashstore')
dfo = DataFileObject(storage_box=sbox, datafile=df, uri=sq_filename)
dfo.save()
return df
def setUp(self):
# create a test user
self.user = User.objects.create_user(username='******',
email='',
password='******')
# create a public experiment
self.experiment1 = Experiment(title='Experiment 1',
created_by=self.user,
public=True)
self.experiment1.save()
# create a non-public experiment
self.experiment2 = Experiment(title='Experiment 2',
created_by=self.user,
public=False)
self.experiment2.save()
# dataset1 belongs to experiment1
self.dataset1 = Dataset(experiment=self.experiment1)
self.dataset1.save()
# dataset2 belongs to experiment2
self.dataset2 = Dataset(experiment=self.experiment2)
self.dataset2.save()
# absolute path first
filename = 'testfile.txt'
self.dest1 = abspath(
join(settings.FILE_STORE_PATH,
'%s/%s/' % (self.experiment1.id, self.dataset1.id)))
self.dest2 = abspath(
join(settings.FILE_STORE_PATH,
'%s/%s/' % (self.experiment2.id, self.dataset2.id)))
if not exists(self.dest1):
makedirs(self.dest1)
if not exists(self.dest2):
makedirs(self.dest2)
testfile1 = abspath(join(self.dest1, filename))
f = open(testfile1, 'w')
f.write("Hello World!\n")
f.close()
testfile2 = abspath(join(self.dest2, filename))
f = open(testfile2, 'w')
f.write("Hello World!\n")
f.close()
self.dataset_file1 = Dataset_File(dataset=self.dataset1,
filename=filename,
protocol='tardis',
url='tardis://%s' % filename)
self.dataset_file1.save()
self.dataset_file2 = Dataset_File(dataset=self.dataset2,
filename=basename(filename),
protocol='tardis',
url='tardis://%s' % filename)
self.dataset_file2.save()
def setUp(self):
"""
setting up essential objects, copied from tests above
"""
user = '******'
pwd = 'secret'
email = ''
self.user = User.objects.create_user(user, email, pwd)
self.userProfile = self.user.userprofile
self.exp = Experiment(title='test exp1',
institution_name='monash',
created_by=self.user)
self.exp.save()
self.acl = ObjectACL(
pluginId=django_user,
entityId=str(self.user.id),
content_object=self.exp,
canRead=True,
isOwner=True,
aclOwnershipType=ObjectACL.OWNER_OWNED,
)
self.acl.save()
self.dataset = Dataset(description='dataset description...')
self.dataset.save()
self.dataset.experiments.add(self.exp)
self.dataset.save()
self.datafile = DataFile(dataset=self.dataset,
size=42,
filename="foo",
md5sum="junk")
self.datafile.save()
def test_datasetwrapper(self):
from mecat.models import DatasetWrapper, Sample
sample_desc = "My Description for Sample created in test_datasetwrapper()"
sample = Sample(experiment=self.experiment, description=sample_desc)
sample.save()
self.assertEqual(sample.description, sample_desc)
self.assertEqual(sample.experiment, self.experiment)
from tardis.tardis_portal.models import Dataset
dataset_desc = "My Description for Dataset created in test_datasetwrapper()"
dataset = Dataset(description=dataset_desc, experiment=self.experiment)
dataset.save()
sample_from_db = Sample.objects.get(description=sample_desc)
dataset_from_db = Dataset.objects.get(description=dataset_desc)
datasetwrapper = DatasetWrapper(sample=sample_from_db, dataset=dataset_from_db)
datasetwrapper.save()
self.assertEqual(datasetwrapper.sample, sample_from_db)
self.assertEqual(datasetwrapper.dataset, dataset_from_db)
datasetwrapper_from_db = DatasetWrapper.objects.get(sample__description=sample_desc)
self.assertEqual(datasetwrapper_from_db.dataset.pk, dataset_from_db.pk)
def setUp(self):
# create a test user
self.user = User.objects.create_user(username='******',
email='',
password='******')
# create a public experiment
self.experiment1 = Experiment(
title='Experiment 1',
created_by=self.user,
public_access=Experiment.PUBLIC_ACCESS_FULL)
self.experiment1.save()
# create a non-public experiment
self.experiment2 = Experiment(
title='Experiment 2',
created_by=self.user,
public_access=Experiment.PUBLIC_ACCESS_NONE)
self.experiment2.save()
# dataset1 belongs to experiment1
self.dataset1 = Dataset()
self.dataset1.save()
self.dataset1.experiments.add(self.experiment1)
self.dataset1.save()
# dataset2 belongs to experiment2
self.dataset2 = Dataset()
self.dataset2.save()
self.dataset2.experiments.add(self.experiment2)
self.dataset2.save()
# absolute path first
filename1 = 'testfile.txt'
filename2 = 'testfile.tiff'
self.dest1 = abspath(join(settings.FILE_STORE_PATH, '%s/%s/'
% (self.experiment1.id,
self.dataset1.id)))
self.dest2 = abspath(join(settings.FILE_STORE_PATH,
'%s/%s/'
% (self.experiment2.id,
self.dataset2.id)))
if not exists(self.dest1):
makedirs(self.dest1)
if not exists(self.dest2):
makedirs(self.dest2)
testfile1 = abspath(join(self.dest1, filename1))
f = open(testfile1, 'w')
f.write("Hello World!\n")
f.close()
testfile2 = abspath(join(self.dest2, filename2))
_generate_test_image(testfile2)
self.datafile1 = self._build_datafile(testfile1, filename1,
self.dataset1)
self.datafile2 = self._build_datafile(testfile2, filename2,
self.dataset2)
class MRtaskTestCase(TestCase):
def setUp(self):
from django.contrib.auth.models import User
from tardis.tardis_portal.models import Experiment
from tardis.tardis_portal.models import ExperimentACL
from tardis.tardis_portal.models import Dataset
from tardis.apps.mrtardis.mrtask import MRtask
user = '******'
pwd = 'secret'
email = ''
self.user = User.objects.create_user(user, email, pwd)
self.experiment = Experiment(approved=True,
title="Test Experiment",
institution_name="Test Institution",
created_by=self.user,
public=False)
self.experiment.save()
acl = ExperimentACL(pluginId="django_user",
entityId="1",
experiment=self.experiment,
canRead=True,
canWrite=True,
canDelete=True,
isOwner=True)
acl.save()
self.test_dataset = Dataset(experiment=self.experiment,
description="test dataset")
self.test_dataset.save()
self.test_mrtask = MRtask(dataset=self.test_dataset)
def test_mrtask(self):
self.assertEqual(len(self.test_mrtask.parameters), 0)
def setUp(self):
# Create test owner without enough details
username, email, password = ('testuser',
'*****@*****.**',
'password')
user = User.objects.create_user(username, email, password)
profile = UserProfile(user=user, isDjangoAccount=True)
profile.save()
Location.force_initialize()
# Create test experiment and make user the owner of it
experiment = Experiment(title='Text Experiment',
institution_name='Test Uni',
created_by=user)
experiment.save()
acl = ObjectACL(
pluginId='django_user',
entityId=str(user.id),
content_object=experiment,
canRead=True,
canWrite=True,
canDelete=True,
isOwner=True,
aclOwnershipType=ObjectACL.OWNER_OWNED)
acl.save()
dataset = Dataset(description='dataset description...')
dataset.save()
dataset.experiments.add(experiment)
dataset.save()
def create_datafile(filename):
testfile = path.join(path.dirname(__file__), 'fixtures',
filename)
size, sha512sum = get_size_and_sha512sum(testfile)
datafile = Dataset_File(dataset=dataset,
filename=path.basename(testfile),
size=size,
sha512sum=sha512sum)
datafile.save()
base_url = 'file://' + path.abspath(path.dirname(testfile))
location = Location.load_location({
'name': 'test-grabber', 'url': base_url, 'type': 'external',
'priority': 10, 'transfer_provider': 'local'})
replica = Replica(datafile=datafile,
url='file://'+path.abspath(testfile),
protocol='file',
location=location)
replica.verify()
replica.save()
return Dataset_File.objects.get(pk=datafile.pk)
self.dataset = dataset
self.datafiles = [create_datafile('data_grabber_test1.admin'),
create_datafile('testfile.txt')
]
def setUp(self):
from os import path, mkdir
from tempfile import mkdtemp
user = '******'
pwd = 'secret'
email = ''
self.user = User.objects.create_user(user, email, pwd)
self.userProfile = UserProfile(user=self.user).save()
self.test_dir = mkdtemp()
Location.force_initialize()
self.exp = Experiment(title='test exp1',
institution_name='monash',
created_by=self.user)
self.exp.save()
acl = ObjectACL(
pluginId=django_user,
entityId=str(self.user.id),
content_object=self.exp,
canRead=True,
isOwner=True,
aclOwnershipType=ObjectACL.OWNER_OWNED,
)
acl.save()
self.dataset = \
Dataset(description='dataset description...')
self.dataset.save()
self.dataset.experiments.add(self.exp)
self.dataset.save()
self.experiment_path = path.join(
settings.FILE_STORE_PATH,
str(self.dataset.get_first_experiment().id))
self.dataset_path = path.join(self.experiment_path,
str(self.dataset.id))
if not path.exists(self.experiment_path):
mkdir(self.experiment_path)
if not path.exists(self.dataset_path):
mkdir(self.dataset_path)
# write test file
self.filename = 'testfile.txt'
self.f1 = open(path.join(self.test_dir, self.filename), 'w')
self.f1.write('Test file 1')
self.f1.close()
self.f1_size = path.getsize(path.join(self.test_dir, self.filename))
self.f1 = open(path.join(self.test_dir, self.filename), 'r')
def setUp(self):
# create a test user
self.user = User.objects.create_user(username='******',
email='',
password='******')
# create a public experiment
self.experiment1 = Experiment(
title='Experiment 1',
created_by=self.user,
public_access=Experiment.PUBLIC_ACCESS_FULL)
self.experiment1.save()
# create a non-public experiment
self.experiment2 = Experiment(
title='Experiment 2',
created_by=self.user,
public_access=Experiment.PUBLIC_ACCESS_NONE)
self.experiment2.save()
# dataset1 belongs to experiment1
self.dataset1 = Dataset(description='dangerous;name')
self.dataset1.save()
self.dataset1.experiments.add(self.experiment1)
self.dataset1.save()
# dataset2 belongs to experiment2
self.dataset2 = Dataset(description='terrible\nname')
self.dataset2.save()
self.dataset2.experiments.add(self.experiment2)
self.dataset2.save()
# absolute path first
filename1 = 'testfile.txt'
filename2 = 'testfile.tiff'
self.dest1 = abspath(
join(settings.FILE_STORE_PATH,
'%s/%s/' % (self.experiment1.id, self.dataset1.id)))
self.dest2 = abspath(
join(settings.FILE_STORE_PATH,
'%s/%s/' % (self.experiment2.id, self.dataset2.id)))
if not exists(self.dest1):
makedirs(self.dest1)
if not exists(self.dest2):
makedirs(self.dest2)
testfile1 = abspath(join(self.dest1, filename1))
f = open(testfile1, 'w')
f.write("Hello World!\n")
f.close()
testfile2 = abspath(join(self.dest2, filename2))
_generate_test_image(testfile2)
self.datafile1 = self._build_datafile(testfile1, filename1,
self.dataset1)
self.datafile2 = self._build_datafile(testfile2, filename2,
self.dataset2)
def _create_test_dataset(nosDatafiles):
ds_ = Dataset(description='happy snaps of plumage')
ds_.save()
for i in range (0, nosDatafiles) :
df_ = Dataset_File(dataset=ds_, url='http://planet-python.org/' + str(_next_id()))
df_.save()
ds_.save()
return ds_
def _make_dataset(self, exp, filenames):
dataset = Dataset(experiment=exp)
dataset.save()
for filename in filenames:
df = Dataset_File(dataset=dataset, size=41, protocol='file')
df.filename = filename
df.url = 'file://' + path.join(path.dirname(__file__), 'data', df.filename)
df.save()
def setUp(self):
# create a test user
self.user = User.objects.create_user(username='******',
email='',
password='******')
# create a public experiment
self.experiment1 = Experiment(title='Experiment 1',
created_by=self.user,
public=True)
self.experiment1.save()
# create a non-public experiment
self.experiment2 = Experiment(title='Experiment 2',
created_by=self.user,
public=False)
self.experiment2.save()
# dataset1 belongs to experiment1
self.dataset1 = Dataset(experiment=self.experiment1)
self.dataset1.save()
# dataset2 belongs to experiment2
self.dataset2 = Dataset(experiment=self.experiment2)
self.dataset2.save()
# absolute path first
filename = 'testfile.txt'
self.dest1 = abspath(join(settings.FILE_STORE_PATH, '%s'
% self.experiment1.id))
self.dest2 = abspath(join(settings.FILE_STORE_PATH, '%s'
% self.experiment2.id))
if not exists(self.dest1):
mkdir(self.dest1)
if not exists(self.dest2):
mkdir(self.dest2)
testfile1 = abspath(join(self.dest1, filename))
f = open(testfile1, 'w')
f.write("Hello World!\n")
f.close()
testfile2 = abspath(join(self.dest2, filename))
f = open(testfile2, 'w')
f.write("Hello World!\n")
f.close()
self.dataset_file1 = Dataset_File(dataset=self.dataset1,
filename=filename,
protocol='tardis',
url='tardis://%s' % filename)
self.dataset_file1.save()
self.dataset_file2 = Dataset_File(dataset=self.dataset2,
filename=basename(filename),
protocol='tardis',
url='tardis://%s' % filename)
self.dataset_file2.save()
def _make_dataset(self, exp, filenames):
dataset = Dataset(experiment=exp)
dataset.save()
for filename in filenames:
df = Dataset_File(dataset=dataset, size=41, protocol='file')
df.filename = filename
df.url = 'file://' + path.join(path.dirname(__file__), 'data',
df.filename)
df.save()
def _create_test_dataset(nosDatafiles):
ds_ = Dataset(description='happy snaps of plumage')
ds_.save()
for i in range(0, nosDatafiles):
df_ = DataFile(dataset=ds_, filename='file_%d' % i, size='21',
sha512sum='bogus')
df_.save()
ds_.save()
return ds_
def setUp(self):
"""Setup test fixtures if needed."""
self.user = User.objects.create_user("doctor", '', "pwd")
self.exp = Experiment(title="Wonderful",
institution_name="Monash University",
created_by=self.user)
self.exp.save()
group = Group(name="Group1")
group.save()
facility = Facility(name="Test Facility", manager_group=group)
facility.save()
self.inst = Instrument(name="Test Instrument1", facility=facility)
self.inst.save()
self.dataset = Dataset(description="Dataset1", instrument=self.inst)
self.dataset.save()
storage_classes = getattr(settings, "HSM_STORAGE_CLASSES",
DEFAULT_HSM_CLASSES)
self.sbox1 = StorageBox(name="SBOX1",
django_storage_class=storage_classes[0],
status='online',
max_size=256)
self.sbox1.save()
sbox1_attr = StorageBoxAttribute(storage_box=self.sbox1,
key='type',
value=StorageBox.DISK)
sbox1_attr.save()
sbox1_loc_opt = StorageBoxOption(storage_box=self.sbox1,
key="location",
value="/dummy/path")
sbox1_loc_opt.save()
self.sbox2 = StorageBox(
name="SBOX2",
django_storage_class="any.non.disk.StorageSystem",
status='offline',
max_size=256)
self.sbox2.save()
sbox2_attr = StorageBoxAttribute(storage_box=self.sbox2,
key='type',
value=StorageBox.TAPE)
sbox2_attr.save()
self.df1 = DataFile(dataset=self.dataset, filename="test_df.jpg")
self.df1.save()
self.dfo1 = DataFileObject(datafile=self.df1,
storage_box=self.sbox1,
uri="stream/test.jpg",
verified=True)
self.dfo1.save()
self.df1.verify()
def generate_dataset(datafiles=[], experiments=[]):
from tardis.tardis_portal.models import Dataset
dataset = Dataset()
dataset.save()
for df in datafiles:
df.dataset_id = dataset.id
df.save()
for exp in experiments:
dataset.experiments.add(exp)
dataset.save()
return dataset
def setUp(self):
# Create test owner without enough details
username, email, password = ('testuser',
'*****@*****.**',
'password')
user = User.objects.create_user(username, email, password)
# Create test experiment and make user the owner of it
experiment = Experiment(title='Text Experiment',
institution_name='Test Uni',
created_by=user)
experiment.save()
acl = ObjectACL(
pluginId='django_user',
entityId=str(user.id),
content_object=experiment,
canRead=True,
isOwner=True,
aclOwnershipType=ObjectACL.OWNER_OWNED,
)
acl.save()
dataset = Dataset(description='dataset description...')
dataset.save()
dataset.experiments.add(experiment)
dataset.save()
base_path = path.join(path.dirname(__file__), 'fixtures')
s_box = StorageBox.get_default_storage(location=base_path)
def create_datafile(index):
testfile = path.join(base_path, 'middleware_test%d.txt' % index)
size, sha512sum = get_size_and_sha512sum(testfile)
datafile = DataFile(dataset=dataset,
filename=path.basename(testfile),
size=size,
sha512sum=sha512sum)
datafile.save()
dfo = DataFileObject(
datafile=datafile,
storage_box=s_box,
uri=path.basename(testfile))
dfo.save()
if index != 1:
dfo.verified = False
dfo.save(update_fields=['verified'])
return DataFile.objects.get(pk=datafile.pk)
self.dataset = dataset
self.datafiles = [create_datafile(i) for i in (1, 2)]
def _create_test_dataset(nosDatafiles):
ds_ = Dataset(description='happy snaps of plumage')
ds_.save()
for i in range (0, nosDatafiles) :
df_ = Dataset_File(dataset=ds_, size='21', sha512sum='bogus')
df_.save()
rep_ = Replica(datafile=df_,
url='http://planet-python.org/' + str(_next_id()),
location=Location.get_default_location())
rep_.save()
ds_.save()
return ds_
def test_urls_with_some_content(self):
# Things that might tend to be in a real live system
user = '******'
pwd = User.objects.make_random_password()
user = User.objects.create(username=user,
email='*****@*****.**',
first_name="Test", last_name="User")
user.set_password(pwd)
user.save()
experiment = Experiment.objects.create(
title="Test Experiment",
created_by=user,
public_access=Experiment.PUBLIC_ACCESS_FULL)
experiment.save()
acl = ObjectACL(pluginId=django_user,
entityId=str(user.id),
content_object=experiment,
canRead=True,
canWrite=True,
canDelete=True,
isOwner=True)
acl.save()
dataset = Dataset(description="test dataset")
dataset.save()
dataset.experiments.add(experiment)
dataset.save()
# Test everything works
c = Client()
c.login(username=user, password=pwd)
urls = ['/about/', '/stats/']
urls += ['/experiment/list/%s' % part
for part in ('mine', 'shared', 'public')]
# urls += ['/experiment/%s/' % part
# for part in ('search',)]
urls += ['/experiment/view/%d/' % experiment.id]
urls += ['/ajax/experiment/%d/%s' % (experiment.id, tabpane)
for tabpane in ('description', 'datasets', 'rights')]
urls += ['/ajax/datafile_list/%d/' % dataset.id]
urls += ['/ajax/dataset_metadata/%d/' % dataset.id]
for u in urls:
response = c.get(u)
ensure(response.status_code, 200,
"%s should have returned 200 but returned %d"
% (u, response.status_code))
redirect_urls = ['/experiment/list', '/experiment/view/']
for u in redirect_urls:
response = c.get(u)
expect(response.status_code).to_equal(302)
def _create_dataset(self):
user = User.objects.create_user('testuser', '*****@*****.**', 'pwd')
user.save()
full_access = Experiment.PUBLIC_ACCESS_FULL
experiment = Experiment.objects.create(title="Background Test",
created_by=user,
public_access=full_access)
experiment.save()
dataset = Dataset()
dataset.save()
dataset.experiments.add(experiment)
dataset.save()
return dataset
def test_create_index(self):
self.exp1 = Experiment(title='test exp1',
institution_name='monash',
description='Test Description',
created_by=self.user)
self.exp2 = Experiment(title='test exp2',
institution_name='monash',
description='Test Description',
created_by=self.user)
self.exp1.save()
self.exp2.save()
# get search instance
search = ExperimentDocument.search()
# query for title(exact matching)
query = search.query("match", title='test exp1')
result = query.execute(ignore_cache=True)
self.assertEqual(result.hits[0].title, 'test exp1')
# query for description
query = search.query("match", description='Test Description')
result = query.execute(ignore_cache=True)
self.assertEqual(result.hits[0].description, 'Test Description')
# query for created_time
query = search.query("match", created_time=self.exp1.created_time)
result = query.execute(ignore_cache=True)
self.assertEqual(result.hits[0].created_time, self.exp1.created_time)
# dataset
# dataset1 belongs to experiment1
self.dataset1 = Dataset(description='test_dataset')
self.dataset1.save()
self.dataset1.experiments.add(self.exp1)
self.dataset1.save()
# dataset2 belongs to experiment2
self.dataset2 = Dataset(description='test_dataset2')
self.dataset2.save()
self.dataset2.experiments.add(self.exp2)
self.dataset2.save()
# search on dataset
search = DatasetDocument.search()
query = search.query("match", description='test_dataset')
result = query.execute(ignore_cache=True)
self.assertEqual(result.hits.total.value, 1)
# search on datafile
settings.REQUIRE_DATAFILE_SIZES = False
settings.REQUIRE_DATAFILE_CHECKSUMS = False
self.datafile = DataFile(dataset=self.dataset1, filename='test.txt')
self.datafile.save()
search = DataFileDocument.search()
query = search.query("match", filename='test.txt')
result = query.execute(ignore_cache=True)
self.assertEqual(result.hits[0].filename, self.datafile.filename)
def setUp(self):
self.hostname = '127.0.0.1'
self.username = '******'
self.password = '******'
email = ''
self.user = User.objects.create_user(self.username, email,
self.password)
self.exp = Experiment(title='test exp1',
institution_name='monash',
created_by=self.user)
self.exp.save()
self.acl = ObjectACL(content_object=self.exp,
pluginId='django_user',
entityId=str(self.user.id),
isOwner=True,
canRead=True,
canWrite=True,
canDelete=True,
aclOwnershipType=ObjectACL.OWNER_OWNED)
self.acl.save()
self.dataset = Dataset(description='test dataset1')
self.dataset.save()
self.dataset.experiments.set([self.exp])
self.dataset.save()
def _build(dataset, filename, url):
datafile_content = b"\n".join(
[b'some data %d' % i for i in range(1000)])
filesize = len(datafile_content)
datafile = DataFile(dataset=dataset,
filename=filename,
size=filesize)
datafile.save()
dfo = DataFileObject(
datafile=datafile,
storage_box=datafile.get_default_storage_box(),
uri=url)
dfo.file_object = BytesIO(datafile_content)
dfo.save()
return datafile
saved_setting = settings.REQUIRE_DATAFILE_CHECKSUMS
try:
settings.REQUIRE_DATAFILE_CHECKSUMS = False
_build(self.dataset, 'file.txt', 'path/file.txt')
finally:
settings.REQUIRE_DATAFILE_CHECKSUMS = saved_setting
def setUp(self):
"""
setting up essential objects, copied from tests above
"""
user = "******"
pwd = "secret"
email = ""
self.user = User.objects.create_user(user, email, pwd)
self.userProfile = self.user.userprofile
self.exp = Experiment(title="test exp1", institution_name="monash", created_by=self.user)
self.exp.save()
self.acl = ObjectACL(
pluginId=django_user,
entityId=str(self.user.id),
content_object=self.exp,
canRead=True,
isOwner=True,
aclOwnershipType=ObjectACL.OWNER_OWNED,
)
self.acl.save()
self.dataset = Dataset(description="dataset description...")
self.dataset.save()
self.dataset.experiments.add(self.exp)
self.dataset.save()
self.datafile = DataFile(dataset=self.dataset, size=42, filename="foo", md5sum="junk")
self.datafile.save()
self.testschema = Schema(
namespace="http://test.com/test/schema", name="Test View", type=Schema.DATAFILE, hidden=True
)
self.testschema.save()
self.dfps = DatafileParameterSet(datafile=self.datafile, schema=self.testschema)
self.dfps.save()
def setUp(self):
# Create test owner without enough details
username, email, password = ("testuser", "*****@*****.**", "password")
user = User.objects.create_user(username, email, password)
# Need UserAuthentication
UserAuthentication(userProfile=user.userprofile, username=username, authenticationMethod="localdb").save()
# Create staging dir
from os import path, makedirs
staging_dir = path.join(settings.STAGING_PATH, username)
if not path.exists(staging_dir):
makedirs(staging_dir)
# Ensure that staging dir is set up properly
expect(get_full_staging_path(username)).to_be_truthy()
# Create test experiment and make user the owner of it
experiment = Experiment(title="Text Experiment", institution_name="Test Uni", created_by=user)
experiment.save()
acl = ObjectACL(
pluginId=django_user,
entityId=str(user.id),
content_object=experiment,
canRead=True,
isOwner=True,
aclOwnershipType=ObjectACL.OWNER_OWNED,
)
acl.save()
self.dataset = Dataset(description="dataset description...")
self.dataset.save()
self.dataset.experiments.add(experiment)
self.dataset.save()
self.username, self.password = (username, password)
def __init__(self, dataset=None, dataset_id=None,
description="", experiment_id=None):
"""
instantiate new task or existing task
:param dataset: optional parameter to instanciate task from
metadata, will be tested for completeness and copied into
new task if complete
:type dataset: Dataset
"""
if dataset:
self.dataset = dataset
elif dataset_id:
self.dataset = Dataset.objects.get(pk=dataset_id)
else:
if description == "":
raise TypeError("No description given")
if not experiment_id:
raise TypeError("No experiment id given")
self.dataset = Dataset()
self.dataset.experiment_id = experiment_id
self.dataset.description = description
self.dataset.save()
try:
thisparameterset = DatasetParameterSet.objects.get(
schema=self.get_schema(),
dataset=self.dataset)
except ObjectDoesNotExist:
thisparameterset = DatasetParameterSet(
schema=self.get_schema(),
dataset=self.dataset)
thisparameterset.save()
super(Task, self).__init__(parameterset=thisparameterset)
def setUp(self):
"""
setting up essential objects, copied from tests above
"""
user = '******'
pwd = 'secret'
email = ''
self.user = User.objects.create_user(user, email, pwd)
self.userProfile = self.user.userprofile
self.exp = Experiment(title='test exp1',
institution_name='monash', created_by=self.user)
self.exp.save()
self.acl = ObjectACL(
pluginId=django_user,
entityId=str(self.user.id),
content_object=self.exp,
canRead=True,
isOwner=True,
aclOwnershipType=ObjectACL.OWNER_OWNED,
)
self.acl.save()
self.dataset = Dataset(description='dataset description...')
self.dataset.save()
self.dataset.experiments.add(self.exp)
self.dataset.save()
self.datafile = DataFile(dataset=self.dataset,
size=42, filename="foo",
md5sum="junk")
self.datafile.save()
def test_003_offline_dataset(self, mock_stat):
"""A dataset should be offline if any datafiles are offline"""
mock_stat.return_value = Stats(st_size=10000,
st_blocks=0,
st_mtime=datetime.now())
ds = Dataset(description="Dataset2", instrument=self.inst)
ds.save()
df2 = DataFile(dataset=ds, filename="test_file.jpg")
df2.save()
dfo2 = DataFileObject(datafile=df2,
storage_box=self.sbox1,
uri=df2.filename)
dfo2.save()
df2.verify()
self.assertFalse(dataset_online(ds))
def setUp(self):
from os import path, mkdir
from tempfile import mkdtemp
user = '******'
pwd = 'secret'
email = ''
self.user = User.objects.create_user(user, email, pwd)
self.userProfile = UserProfile(user=self.user).save()
self.test_dir = mkdtemp()
Location.force_initialize()
self.exp = Experiment(title='test exp1',
institution_name='monash', created_by=self.user)
self.exp.save()
acl = ObjectACL(
pluginId=django_user,
entityId=str(self.user.id),
content_object=self.exp,
canRead=True,
isOwner=True,
aclOwnershipType=ObjectACL.OWNER_OWNED,
)
acl.save()
self.dataset = \
Dataset(description='dataset description...')
self.dataset.save()
self.dataset.experiments.add(self.exp)
self.dataset.save()
self.experiment_path = path.join(settings.FILE_STORE_PATH,
str(self.dataset.get_first_experiment().id))
self.dataset_path = path.join(self.experiment_path,
str(self.dataset.id))
if not path.exists(self.experiment_path):
mkdir(self.experiment_path)
if not path.exists(self.dataset_path):
mkdir(self.dataset_path)
# write test file
self.filename = 'testfile.txt'
self.f1 = open(path.join(self.test_dir, self.filename), 'w')
self.f1.write('Test file 1')
self.f1.close()
self.f1_size = path.getsize(path.join(self.test_dir,
self.filename))
self.f1 = open(path.join(self.test_dir, self.filename), 'r')
def setUp(self):
# Create test owner without enough details
username, email, password = ('testuser', '*****@*****.**',
'password')
user = User.objects.create_user(username, email, password)
profile = UserProfile(user=user, isDjangoAccount=True)
profile.save()
# Need UserAuthentication
UserAuthentication(userProfile=profile,
username=username,
authenticationMethod='localdb').save()
# Create staging dir
from os import path, makedirs
staging_dir = path.join(settings.STAGING_PATH, username)
if not path.exists(staging_dir):
makedirs(staging_dir)
# Ensure that staging dir is set up properly
expect(get_full_staging_path(username)).to_be_truthy()
Location.force_initialize()
# Create test experiment and make user the owner of it
experiment = Experiment(title='Text Experiment',
institution_name='Test Uni',
created_by=user)
experiment.save()
acl = ObjectACL(
pluginId=django_user,
entityId=str(user.id),
content_object=experiment,
canRead=True,
isOwner=True,
aclOwnershipType=ObjectACL.OWNER_OWNED,
)
acl.save()
self.dataset = \
Dataset(description='dataset description...')
self.dataset.save()
self.dataset.experiments.add(experiment)
self.dataset.save()
self.username, self.password = (username, password)
def _create_datafile():
user = User.objects.create_user("testuser", "*****@*****.**", "pwd")
user.save()
UserProfile(user=user).save()
Location.force_initialize()
full_access = Experiment.PUBLIC_ACCESS_FULL
experiment = Experiment.objects.create(title="IIIF Test", created_by=user, public_access=full_access)
experiment.save()
ObjectACL(
content_object=experiment,
pluginId="django_user",
entityId=str(user.id),
isOwner=True,
canRead=True,
canWrite=True,
canDelete=True,
aclOwnershipType=ObjectACL.OWNER_OWNED,
).save()
dataset = Dataset()
dataset.save()
dataset.experiments.add(experiment)
dataset.save()
# Create new Datafile
tempfile = TemporaryUploadedFile("iiif_stored_file", None, None, None)
with Image(filename="magick:rose") as img:
img.format = "tiff"
img.save(file=tempfile.file)
tempfile.file.flush()
datafile = Dataset_File(dataset=dataset, size=os.path.getsize(tempfile.file.name), filename="iiif_named_file")
replica = Replica(
datafile=datafile,
url=write_uploaded_file_to_dataset(dataset, tempfile),
location=Location.get_default_location(),
)
replica.verify(allowEmptyChecksums=True)
datafile.save()
replica.datafile = datafile
replica.save()
return datafile
def test_contextual_view(self):
"""
Given schema on dataset, check that image file created
"""
user = _create_test_user()
license = _create_license()
exp = _create_test_experiment(user, license)
ds = Dataset(description='happy snaps of plumage')
ds.save()
ds = _create_test_dataset(ds, exp.id, {
"output.dat": 'test data\n',
"grexp.dat": '1 2\n2 3\n3 7\n',
"grfinal21.dat": '1 2\n 2 4\n4 9\n'})
sch = Schema(namespace=self.HRMCSCHEMA,
name="hrmc_views", type=Schema.DATASET)
sch.save()
param = ParameterName(schema=sch, name="plot",
full_name="scatterplot", units="image",
data_type=ParameterName.FILENAME
)
param.save()
dps = DatasetParameterSet(schema=sch, dataset=ds)
dps.save()
ds.experiments.add(exp)
ds.save()
client = Client()
response = client.get('/dataset/%s' % ds.id)
self.assertEqual(response.status_code, 200)
param_sets = get_param_sets(ds)
self.assertTrue(param_sets)
dp = DatasetParameter.objects.get(parameterset=param_sets[0],
name=param)
self.assertTrue(dp)
self.assertNotEquals(dp.string_value, "") # ie, it has a filename
def setUp(self):
super(SimpleSearchTest, self).setUp()
self.out = StringIO()
call_command('search_index',
stdout=self.out,
action='delete',
force=True)
call_command('search_index',
stdout=self.out,
action='rebuild',
force=True)
# add dataset and datafile to experiment
self.dataset1 = Dataset(description='test_dataset')
self.dataset1.save()
self.dataset1.experiments.add(self.testexp)
self.dataset1.save()
settings.REQUIRE_DATAFILE_SIZES = False
settings.REQUIRE_DATAFILE_CHECKSUMS = False
self.datafile = DataFile(dataset=self.dataset1, filename='test.txt')
self.datafile.save()
def setUp(self):
"""
setting up essential objects, copied from tests above
"""
user = '******'
pwd = 'secret'
email = ''
self.user = User.objects.create_user(user, email, pwd)
self.userProfile = UserProfile(user=self.user).save()
self.exp = Experiment(title='test exp1',
institution_name='monash',
created_by=self.user)
self.exp.save()
self.acl = ObjectACL(
pluginId=django_user,
entityId=str(self.user.id),
content_object=self.exp,
canRead=True,
isOwner=True,
aclOwnershipType=ObjectACL.OWNER_OWNED,
)
self.acl.save()
self.dataset = Dataset(description='dataset description...')
self.dataset.save()
self.dataset.experiments.add(self.exp)
self.dataset.save()
self.dataset_file = Dataset_File(dataset=self.dataset,
size=42,
filename="foo",
md5sum="junk")
self.dataset_file.save()
self.testschema = Schema(namespace="http://test.com/test/schema",
name="Test View",
type=Schema.DATAFILE,
hidden=True)
self.testschema.save()
self.dfps = DatafileParameterSet(dataset_file=self.dataset_file,
schema=self.testschema)
self.dfps.save()
def test_hrmc_filter(self):
"""
Make an experiment, lood up grexp file and check
dataset schema missing, then loadup grfinal and check dataset schema
created
"""
user = _create_test_user()
license = _create_license()
exp = _create_test_experiment(user, license)
ds = Dataset(description='happy snaps of plumage')
ds.save()
_create_test_dataset(ds, exp.id,
{"output.dat": 'hello', "grexp.dat": '2 5\n6 15\n'})
ds.experiments.add(exp)
ds.save()
sch = Schema(namespace=self.HRMCSCHEMA,
name="hrmc_views", type=Schema.DATASET)
sch.save()
param = ParameterName(schema=sch, name="plot",
full_name="scatterplot", units="image",
data_type=ParameterName.FILENAME
)
param.save()
param_sets = get_param_sets(ds)
self.assertEquals(list(param_sets), [])
_create_test_dataset(ds, exp.id, {'grfinal21.dat': "1 3\n5 14\n"})
df2 = Dataset_File(dataset=ds, url='path/grfinal21.dat')
df2.save()
h = hrmc.HRMCOutput('HRMC', self.HRMCSCHEMA)
h(sender=Dataset_File, instance=df2)
param_sets = get_param_sets(ds)
self.assertEquals([x.schema.namespace for x in param_sets],
[self.HRMCSCHEMA])
def setUp(self):
"""
setting up essential objects, copied from tests above
"""
Location.force_initialize()
self.location = Location.get_location('local')
user = '******'
pwd = 'secret'
email = ''
self.user = User.objects.create_user(user, email, pwd)
self.userProfile = UserProfile(user=self.user).save()
self.exp = Experiment(title='test exp1',
institution_name='monash',
created_by=self.user)
self.exp.save()
self.acl = ObjectACL(
pluginId=django_user,
entityId=str(self.user.id),
content_object=self.exp,
canRead=True,
isOwner=True,
aclOwnershipType=ObjectACL.OWNER_OWNED,
)
self.acl.save()
self.dataset = Dataset(description='dataset description...')
self.dataset.save()
self.dataset.experiments.add(self.exp)
self.dataset.save()
self.dataset_file = Dataset_File(dataset=self.dataset,
size=42,
filename="foo",
md5sum="junk")
self.dataset_file.save()
self.replica = Replica(datafile=self.dataset_file,
url="http://foo",
location=self.location,
verified=False)
self.replica.save()
def test_deleting_dfo_without_uri(self):
dataset = Dataset(description="dataset description")
dataset.save()
save1 = settings.REQUIRE_DATAFILE_SIZES
save2 = settings.REQUIRE_DATAFILE_CHECKSUMS
try:
settings.REQUIRE_DATAFILE_SIZES = False
settings.REQUIRE_DATAFILE_CHECKSUMS = False
datafile = DataFile(dataset=dataset, filename='test1.txt')
datafile.save()
finally:
settings.REQUIRE_DATAFILE_SIZES = save1
settings.REQUIRE_DATAFILE_CHECKSUMS = save2
dfo = DataFileObject(
datafile=datafile,
storage_box=datafile.get_default_storage_box(),
uri=None)
dfo.save()
self.assertIsNone(dfo.uri)
self.assertIsNotNone(dfo.id)
dfo.delete()
self.assertIsNone(dfo.id)
def setUp(self):
# Create test owner without enough details
username, email, password = ('testuser', '*****@*****.**',
'password')
user = User.objects.create_user(username, email, password)
profile = UserProfile(user=user, isDjangoAccount=True)
profile.save()
Location.force_initialize()
# Create test experiment and make user the owner of it
experiment = Experiment(title='Text Experiment',
institution_name='Test Uni',
created_by=user)
experiment.save()
acl = ObjectACL(
pluginId='django_user',
entityId=str(user.id),
content_object=experiment,
canRead=True,
isOwner=True,
aclOwnershipType=ObjectACL.OWNER_OWNED,
)
acl.save()
dataset = Dataset(description='dataset description...')
dataset.save()
dataset.experiments.add(experiment)
dataset.save()
def create_datafile(index):
testfile = path.join(path.dirname(__file__), 'fixtures',
'jeol_sem_test%d.txt' % index)
size, sha512sum = get_size_and_sha512sum(testfile)
datafile = Dataset_File(dataset=dataset,
filename=path.basename(testfile),
size=size,
sha512sum=sha512sum)
datafile.save()
base_url = 'file://' + path.abspath(path.dirname(testfile))
location = Location.load_location({
'name': 'test-jeol',
'url': base_url,
'type': 'external',
'priority': 10,
'transfer_provider': 'local'
})
replica = Replica(datafile=datafile,
url='file://' + path.abspath(testfile),
protocol='file',
location=location)
replica.verify()
replica.save()
return Dataset_File.objects.get(pk=datafile.pk)
self.dataset = dataset
self.datafiles = [create_datafile(i) for i in (1, 2)]
def setUp(self):
from os import path, mkdir
from tempfile import mkdtemp
user = "******"
pwd = "secret"
email = ""
self.user = User.objects.create_user(user, email, pwd)
self.userProfile = self.user.userprofile
self.test_dir = mkdtemp()
self.exp = Experiment(title="test exp1", institution_name="monash", created_by=self.user)
self.exp.save()
acl = ObjectACL(
pluginId=django_user,
entityId=str(self.user.id),
content_object=self.exp,
canRead=True,
isOwner=True,
aclOwnershipType=ObjectACL.OWNER_OWNED,
)
acl.save()
self.dataset = Dataset(description="dataset description...")
self.dataset.save()
self.dataset.experiments.add(self.exp)
self.dataset.save()
path_parts = [
settings.FILE_STORE_PATH,
"%s-%s" % (quote(self.dataset.description, safe="") or "untitled", self.dataset.id),
]
self.dataset_path = path.join(*path_parts)
if not path.exists(self.dataset_path):
mkdir(self.dataset_path)
# write test file
self.filename = "testfile.txt"
self.f1 = open(path.join(self.test_dir, self.filename), "w")
self.f1.write("Test file 1")
self.f1.close()
self.f1_size = path.getsize(path.join(self.test_dir, self.filename))
self.f1 = open(path.join(self.test_dir, self.filename), "r")
def _create_datafile():
user = User.objects.create_user("testuser", "*****@*****.**", "pwd")
user.save()
full_access = Experiment.PUBLIC_ACCESS_FULL
experiment = Experiment.objects.create(title="IIIF Test", created_by=user, public_access=full_access)
experiment.save()
ObjectACL(
content_object=experiment,
pluginId="django_user",
entityId=str(user.id),
isOwner=True,
canRead=True,
canWrite=True,
canDelete=True,
aclOwnershipType=ObjectACL.OWNER_OWNED,
).save()
dataset = Dataset()
dataset.save()
dataset.experiments.add(experiment)
dataset.save()
# Create new Datafile
tempfile = TemporaryUploadedFile("iiif_stored_file", None, None, None)
with Image(filename="magick:rose") as img:
img.format = "tiff"
img.save(file=tempfile.file)
tempfile.file.flush()
datafile = DataFile(
dataset=dataset, size=os.path.getsize(tempfile.file.name), filename="iiif_named_file", mimetype="image/tiff"
)
checksums = compute_checksums(open(tempfile.file.name, "r"))
datafile.md5sum = checksums["md5sum"]
datafile.sha512sum = checksums["sha512sum"]
datafile.save()
datafile.file_object = tempfile
return datafile
def _create_test_dataset(nosDatafiles):
ds_ = Dataset(description='happy snaps of plumage')
ds_.save()
for i in range(0, nosDatafiles):
df_ = Dataset_File(dataset=ds_,
url='http://planet-python.org/' + str(_next_id()))
df_.save()
ds_.save()
return ds_
def setUp(self):
self.hostname = '127.0.0.1'
self.username = '******'
self.password = '******'
email = ''
self.user = User.objects.create_user(
self.username, email, self.password)
self.exp = Experiment(
title='test exp1', institution_name='monash', created_by=self.user)
self.exp.save()
self.acl = ObjectACL(
content_object=self.exp,
pluginId='django_user',
entityId=str(self.user.id),
isOwner=True,
canRead=True,
canWrite=True,
canDelete=True,
aclOwnershipType=ObjectACL.OWNER_OWNED)
self.acl.save()
self.dataset = Dataset(description='test dataset1')
self.dataset.save()
self.dataset.experiments.set([self.exp])
self.dataset.save()
def _build(dataset, filename, url):
datafile_content = u"\n".join([u'some data %d' % i
for i in range(1000)])
filesize = len(datafile_content)
datafile = DataFile(
dataset=dataset, filename=filename, size=filesize)
datafile.save()
dfo = DataFileObject(
datafile=datafile,
storage_box=datafile.get_default_storage_box(),
uri=url)
dfo.file_object = StringIO(datafile_content)
dfo.save()
return datafile
saved_setting = settings.REQUIRE_DATAFILE_CHECKSUMS
try:
settings.REQUIRE_DATAFILE_CHECKSUMS = False
df_file = _build(self.dataset, 'file.txt', 'path/file.txt')
finally:
settings.REQUIRE_DATAFILE_CHECKSUMS = saved_setting
def setUp(self):
raise SkipTest # temporarily disabling this feature, needs coding
from tempfile import mkdtemp, mktemp
from django.conf import settings
import os
# Disconnect post_save signal
from django.db.models.signals import post_save
from tardis.tardis_portal.models import Experiment, \
staging_hook, Dataset, DataFile, DataFileObject, StorageBox
post_save.disconnect(staging_hook, sender=DataFileObject)
from django.contrib.auth.models import User
user = '******'
pwd = 'secret'
email = ''
self.user = User.objects.create_user(user, email, pwd)
try:
os.makedirs(settings.GET_FULL_STAGING_PATH_TEST)
except OSError:
pass
self.temp = mkdtemp(dir=settings.GET_FULL_STAGING_PATH_TEST)
self.filepath = mktemp(dir=self.temp)
content = 'test file'
with open(self.filepath, "w+b") as f:
f.write(content)
# make datafile
exp = Experiment(title='test exp1',
institution_name='monash',
created_by=self.user)
exp.save()
# make dataset
dataset = Dataset(description="dataset description...")
dataset.save()
dataset.experiments.add(exp)
dataset.save()
# create datafile
df = DataFile(dataset=dataset, size=len(content),
filename=path.basename(self.file),
md5sum='f20d9f2072bbeb6691c0f9c5099b01f3')
df.save()
# create replica
base_url = settings.GET_FULL_STAGING_PATH_TEST
s_box = StorageBox.get_default_storage(location=base_url)
dfo = DataFileObject(datafile=df,
uri=self.filepath,
storage_box=s_box)
dfo.save()
self.dfo = dfo
def _create_test_dataset(nosDatafiles):
ds_ = Dataset(description='happy snaps of plumage')
ds_.save()
for i in range(0, nosDatafiles):
df_ = DataFile(dataset=ds_,
filename='file_%d' % i,
size='21',
sha512sum='bogus')
df_.save()
ds_.save()
return ds_
def generate_dataset(datafiles=[], experiments=[]):
from tardis.tardis_portal.models import Dataset
dataset = Dataset()
dataset.save()
for df in datafiles:
df.dataset_id = dataset.id
df.save()
for exp in experiments:
dataset.experiments.add(exp)
dataset.save()
return dataset
def _create_test_dataset(nosDatafiles):
ds_ = Dataset(description='happy snaps of plumage')
ds_.save()
for i in range(0, nosDatafiles):
df_ = Dataset_File(dataset=ds_, size='21', sha512sum='bogus')
df_.save()
rep_ = Replica(datafile=df_,
url='http://planet-python.org/' + str(_next_id()),
location=Location.get_default_location())
rep_.save()
ds_.save()
return ds_
def _prepare_project(self):
project = Project(experiment=self.experiment)
project.save()
s1 = Sample(experiment=self.experiment, name="S1", description="s1 desc")
s1.save()
s2 = Sample(experiment=self.experiment, name="S2", description="s2 desc")
s2.save()
dw1 = DatasetWrapper(sample=s1, name="dw1", description="dw1 desc")
dw2 = DatasetWrapper(sample=s1, name="dw2", description="dw2 desc")
dw3 = DatasetWrapper(sample=s2, name="dw3", description="dw3 desc")
ds1 = Dataset(experiment=self.experiment, description=dw1.description)
ds2 = Dataset(experiment=self.experiment, description=dw2.description)
ds3 = Dataset(experiment=self.experiment, description=dw3.description)
ds1.save()
ds2.save()
ds3.save()
dw1.dataset = ds1
dw1.save()
dw2.dataset = ds2
dw2.save()
dw3.dataset = ds3
dw3.save()
return project
def setUp(self):
# Create test owner without enough details
username, email, password = ('testuser',
'*****@*****.**',
'password')
user = User.objects.create_user(username, email, password)
profile = UserProfile(user=user, isDjangoAccount=True)
profile.save()
# Need UserAuthentication
UserAuthentication(userProfile=profile,
username=username,
authenticationMethod='localdb').save()
# Create staging dir
from os import path, makedirs
staging_dir = path.join(settings.STAGING_PATH, username)
if not path.exists(staging_dir):
makedirs(staging_dir)
# Ensure that staging dir is set up properly
expect(get_full_staging_path(username)).to_be_truthy()
Location.force_initialize()
# Create test experiment and make user the owner of it
experiment = Experiment(title='Text Experiment',
institution_name='Test Uni',
created_by=user)
experiment.save()
acl = ExperimentACL(
pluginId=django_user,
entityId=str(user.id),
experiment=experiment,\
canRead=True,
isOwner=True,
aclOwnershipType=ExperimentACL.OWNER_OWNED,
)
acl.save()
self.dataset = \
Dataset(description='dataset description...')
self.dataset.save()
self.dataset.experiments.add(experiment)
self.dataset.save()
self.username, self.password = (username, password)
def _create_dataset(self):
user = User.objects.create_user('testuser', '*****@*****.**', 'pwd')
user.save()
full_access = Experiment.PUBLIC_ACCESS_FULL
experiment = Experiment.objects.create(title="Background Test",
created_by=user,
public_access=full_access)
experiment.save()
dataset = Dataset()
dataset.save()
dataset.experiments.add(experiment)
dataset.save()
return dataset
def test_dataset(self):
exp = Experiment(title='test exp1',
institution_name='monash',
created_by=self.user)
exp.save()
exp2 = Experiment(title='test exp2',
institution_name='monash',
created_by=self.user)
exp2.save()
group = Group(name="Test Manager Group")
group.save()
group.user_set.add(self.user)
facility = Facility(name="Test Facility", manager_group=group)
facility.save()
instrument = Instrument(name="Test Instrument", facility=facility)
instrument.save()
dataset = Dataset(description='test dataset1')
dataset.instrument = instrument
dataset.save()
dataset.experiments.set([exp, exp2])
dataset.save()
dataset_id = dataset.id
del dataset
dataset = Dataset.objects.get(pk=dataset_id)
self.assertEqual(dataset.description, 'test dataset1')
self.assertEqual(dataset.experiments.count(), 2)
self.assertIn(exp, list(dataset.experiments.iterator()))
self.assertIn(exp2, list(dataset.experiments.iterator()))
self.assertEqual(instrument, dataset.instrument)
target_id = Dataset.objects.first().id
self.assertEqual(
dataset.get_absolute_url(), '/dataset/%d' % target_id,
dataset.get_absolute_url() + ' != /dataset/%d' % target_id)
def setUp(self):
# Create test owner without enough details
username, email, password = ('testuser',
'*****@*****.**',
'password')
user = User.objects.create_user(username, email, password)
profile = UserProfile(user=user, isDjangoAccount=True)
profile.save()
# Create test experiment and make user the owner of it
experiment = Experiment(title='Text Experiment',
institution_name='Test Uni',
created_by=user)
experiment.save()
acl = ExperimentACL(
pluginId='django_user',
entityId=str(user.id),
experiment=experiment,
canRead=True,
isOwner=True,
aclOwnershipType=ExperimentACL.OWNER_OWNED,
)
acl.save()
dataset = Dataset(description='dataset description...')
dataset.save()
dataset.experiments.add(experiment)
dataset.save()
def create_datafile(index):
testfile = path.join(path.dirname(__file__), 'fixtures',
'jeol_sem_test%d.txt' % index)
size, sha512sum = get_size_and_sha512sum(testfile)
datafile = Dataset_File(dataset=dataset,
filename=path.basename(testfile),
url='file://'+path.abspath(testfile),
protocol='file',
size=size,
sha512sum=sha512sum)
datafile.verify()
datafile.save()
return datafile
self.dataset = dataset
self.datafiles = [create_datafile(i) for i in (1,2)]
def test_contextual_view(self):
"""
Given schema on dataset, check that image file created
"""
user = _create_test_user()
license = _create_license()
exp = _create_test_experiment(user, license)
ds = Dataset(description='happy snaps of plumage')
ds.save()
ds = _create_test_dataset(
ds, exp.id, {
"output.dat": 'test data\n',
"grexp.dat": '1 2\n2 3\n3 7\n',
"grfinal21.dat": '1 2\n 2 4\n4 9\n'
})
sch = Schema(namespace=self.HRMCSCHEMA,
name="hrmc_views",
type=Schema.DATASET)
sch.save()
param = ParameterName(schema=sch,
name="plot",
full_name="scatterplot",
units="image",
data_type=ParameterName.FILENAME)
param.save()
dps = DatasetParameterSet(schema=sch, dataset=ds)
dps.save()
ds.experiments.add(exp)
ds.save()
client = Client()
response = client.get('/dataset/%s' % ds.id)
self.assertEqual(response.status_code, 200)
param_sets = get_param_sets(ds)
self.assertTrue(param_sets)
dp = DatasetParameter.objects.get(parameterset=param_sets[0],
name=param)
self.assertTrue(dp)
self.assertNotEquals(dp.string_value, "") # ie, it has a filename
def setUp(self):
"""
setting up essential objects, copied from tests above
"""
Location.force_initialize()
self.location = Location.get_location('local')
user = '******'
pwd = 'secret'
email = ''
self.user = User.objects.create_user(user, email, pwd)
self.userProfile = UserProfile(user=self.user).save()
self.exp = Experiment(title='test exp1',
institution_name='monash', created_by=self.user)
self.exp.save()
self.acl = ObjectACL(
pluginId=django_user,
entityId=str(self.user.id),
content_object=self.exp,
canRead=True,
isOwner=True,
aclOwnershipType=ObjectACL.OWNER_OWNED,
)
self.acl.save()
self.dataset = Dataset(description='dataset description...')
self.dataset.save()
self.dataset.experiments.add(self.exp)
self.dataset.save()
self.dataset_file = Dataset_File(dataset=self.dataset,
size=42, filename="foo",
md5sum="junk")
self.dataset_file.save()
self.replica = Replica(datafile=self.dataset_file,
url="http://foo",
location=self.location,
verified=False)
self.replica.save()
def test_hrmc_filter(self):
"""
Make an experiment, lood up grexp file and check
dataset schema missing, then loadup grfinal and check dataset schema
created
"""
user = _create_test_user()
license = _create_license()
exp = _create_test_experiment(user, license)
ds = Dataset(description='happy snaps of plumage')
ds.save()
_create_test_dataset(ds, exp.id, {
"output.dat": 'hello',
"grexp.dat": '2 5\n6 15\n'
})
ds.experiments.add(exp)
ds.save()
sch = Schema(namespace=self.HRMCSCHEMA,
name="hrmc_views",
type=Schema.DATASET)
sch.save()
param = ParameterName(schema=sch,
name="plot",
full_name="scatterplot",
units="image",
data_type=ParameterName.FILENAME)
param.save()
param_sets = get_param_sets(ds)
self.assertEquals(list(param_sets), [])
_create_test_dataset(ds, exp.id, {'grfinal21.dat': "1 3\n5 14\n"})
df2 = Dataset_File(dataset=ds, url='path/grfinal21.dat')
df2.save()
h = hrmc.HRMCOutput('HRMC', self.HRMCSCHEMA)
h(sender=Dataset_File, instance=df2)
param_sets = get_param_sets(ds)
self.assertEquals([x.schema.namespace for x in param_sets],
[self.HRMCSCHEMA])
def add_dataset(request, experiment_id):
if not has_experiment_write(request, experiment_id):
return HttpResponseForbidden()
# Process form or prepopulate it
if request.method == 'POST':
form = DatasetForm(request.POST)
if form.is_valid():
dataset = Dataset()
dataset.description = form.cleaned_data['description']
dataset.save()
experiment = Experiment.objects.get(id=experiment_id)
dataset.experiments.add(experiment)
dataset.save()
return _redirect_303('tardis_portal.view_dataset', dataset.id)
else:
form = DatasetForm()
c = {'form': form}
return HttpResponse(
render_response_index(request,
'tardis_portal/add_or_edit_dataset.html', c))
def _create_datafile():
user = User.objects.create_user('testuser', '*****@*****.**', 'pwd')
user.save()
full_access = Experiment.PUBLIC_ACCESS_FULL
experiment = Experiment.objects.create(title="IIIF Test",
created_by=user,
public_access=full_access)
experiment.save()
ObjectACL(content_object=experiment,
pluginId='django_user',
entityId=str(user.id),
isOwner=True,
canRead=True,
canWrite=True,
canDelete=True,
aclOwnershipType=ObjectACL.OWNER_OWNED).save()
dataset = Dataset()
dataset.save()
dataset.experiments.add(experiment)
dataset.save()
# Create new Datafile
tempfile = TemporaryUploadedFile('iiif_stored_file', None, None, None)
with Image(filename='magick:rose') as img:
img.format = 'tiff'
img.save(file=tempfile.file)
tempfile.file.flush()
datafile = DataFile(dataset=dataset,
size=os.path.getsize(tempfile.file.name),
filename='iiif_named_file',
mimetype='image/tiff')
compute_md5 = getattr(settings, 'COMPUTE_MD5', True)
compute_sha512 = getattr(settings, 'COMPUTE_SHA512', True)
checksums = compute_checksums(open(tempfile.file.name, 'r'),
compute_md5=compute_md5,
compute_sha512=compute_sha512)
if compute_md5:
datafile.md5sum = checksums['md5sum']
if compute_sha512:
datafile.sha512sum = checksums['sha512sum']
datafile.save()
datafile.file_object = tempfile
return datafile
def get_or_create_dataset(self, name, top=None):
'''
returns existing or created dataset given a name
returns False if the dataset is not unique by name
top is the directory
'''
ds = Dataset.objects.filter(description=name,
experiments=self.experiment)
if len(ds) == 1:
return ds[0]
elif len(ds) > 1:
return False
ds = Dataset(description=name)
if top is not None:
ds.directory = top
ds.save()
self.tag_user(ds, top)
else:
ds.save()
ds.experiments.add(self.experiment)
return ds
def _create_datafile():
user = User.objects.create_user('testuser', '*****@*****.**', 'pwd')
user.save()
UserProfile(user=user).save()
Location.force_initialize()
full_access = Experiment.PUBLIC_ACCESS_FULL
experiment = Experiment.objects.create(title="IIIF Test",
created_by=user,
public_access=full_access)
experiment.save()
ObjectACL(content_object=experiment,
pluginId='django_user',
entityId=str(user.id),
isOwner=True,
canRead=True,
canWrite=True,
canDelete=True,
aclOwnershipType=ObjectACL.OWNER_OWNED).save()
dataset = Dataset()
dataset.save()
dataset.experiments.add(experiment)
dataset.save()
# Create new Datafile
tempfile = TemporaryUploadedFile('iiif_stored_file', None, None, None)
with Image(filename='magick:rose') as img:
img.format = 'tiff'
img.save(file=tempfile.file)
tempfile.file.flush()
datafile = Dataset_File(dataset=dataset,
size=os.path.getsize(tempfile.file.name),
filename='iiif_named_file')
replica = Replica(datafile=datafile,
url=write_uploaded_file_to_dataset(dataset, tempfile),
location=Location.get_default_location())
replica.verify(allowEmptyChecksums=True)
datafile.save()
replica.datafile = datafile
replica.save()
return datafile
def add_dataset(request, experiment_id):
if not has_experiment_write(request, experiment_id):
return HttpResponseForbidden()
# Process form or prepopulate it
if request.method == 'POST':
form = DatasetForm(request.POST)
if form.is_valid():
dataset = Dataset()
dataset.description = form.cleaned_data['description']
dataset.save()
experiment = Experiment.objects.get(id=experiment_id)
dataset.experiments.add(experiment)
dataset.save()
return _redirect_303('tardis_portal.view_dataset',
dataset.id)
else:
form = DatasetForm()
c = {'form': form}
return HttpResponse(render_response_index(request,
'tardis_portal/add_or_edit_dataset.html', c))
