def test_instrument(self):
group = Group(name="Test Manager Group")
group.save()
facility = Facility(name="Test Facility", manager_group=group)
facility.save()
self.assertEqual(str(facility), "Test Facility")
instrument = Instrument(name="Test Instrument", facility=facility)
instrument.save()
self.assertEqual(str(instrument), "Test Instrument")
self.assertEqual(len(instrument.getParameterSets()), 0)
schema = Schema(namespace='test instrument schema namespace',
type=Schema.INSTRUMENT)
schema.save()
parname = ParameterName(schema=schema,
name='name',
full_name='full_name')
parname.save()
pset = InstrumentParameterSet.objects.create(instrument=instrument,
schema=schema)
pset.save()
self.assertEqual(len(instrument.getParameterSets()), 1)
def save_rif_cs_profile(self, experiment, profile):
"""
Save selected profile choice as experiment parameter
"""
namespace = "http://monash.edu.au/rif-cs/profile/"
schema = None
try:
schema = Schema.objects.get(namespace__exact=namespace)
except Schema.DoesNotExist:
logger.debug('Schema ' + namespace + ' does not exist. Creating.')
schema = Schema(namespace=namespace)
schema.save()
try:
parametername = ParameterName.objects.get(
schema__namespace__exact=schema.namespace, name="profile")
except ParameterName.DoesNotExist:
logger.debug("profile does not exist. Creating.")
parametername = ParameterName(name="profile", schema=schema)
parametername.save()
parameterset = None
try:
parameterset = \
ExperimentParameterSet.objects.get(\
schema=schema,
experiment=experiment)
except ExperimentParameterSet.DoesNotExist, e:
parameterset = ExperimentParameterSet(\
schema=schema,
experiment=experiment)
parameterset.save()
def _make_param(self,schema,name,paramtype):
try:
param = ParameterName.objects.get(schema=schema,
name=name,data_type=paramtype
)
except ParameterName.DoesNotExist:
logger.debug("%s does not exist. Creating." % name)
param = ParameterName(name=name,schema=schema,data_type=paramtype)
param.save()
return param
def getParamaters(self, schema, metadata):
"""Return a list of the paramaters that will be saved.
"""
param_objects = ParameterName.objects.filter(schema=schema)
parameters = []
for p in metadata:
if self.tagsToFind and not p in self.tagsToFind:
continue
if p in self.tagsToExclude:
continue
parameter = filter(lambda x: x.name == p, param_objects)
if parameter:
parameters.append(parameter[0])
continue
# detect type of parameter
datatype = ParameterName.STRING
# integer data type test
try:
int(metadata[p][0])
except (ValueError, TypeError):
pass
else:
datatype = ParameterName.NUMERIC
# float data type test
try:
float(metadata[p][0])
except (ValueError, TypeError):
pass
else:
datatype = ParameterName.NUMERIC
# fraction data type test
if isinstance(metadata[p][0], Fraction):
datatype = ParameterName.NUMERIC
unit = ""
if metadata[p][1]:
unit = metadata[p][1]
new_param = ParameterName(schema=schema,
name=p,
full_name=p,
data_type=datatype,
units=unit)
new_param.save()
parameters.append(new_param)
return parameters
def _make_param(self, schema, name, paramtype):
try:
param = ParameterName.objects.get(schema=schema,
name=name,
data_type=paramtype)
except ParameterName.DoesNotExist:
logger.debug("%s does not exist. Creating." % name)
param = ParameterName(name=name,
schema=schema,
data_type=paramtype)
param.save()
return param
def getParamaters(self, schema, metadata):
"""Return a list of the paramaters that will be saved.
"""
param_objects = ParameterName.objects.filter(schema=schema)
parameters = []
for p in metadata:
if self.tagsToFind and not p in self.tagsToFind:
continue
if p in self.tagsToExclude:
continue
parameter = filter(lambda x: x.name == p, param_objects)
if parameter:
parameters.append(parameter[0])
continue
# detect type of parameter
datatype = ParameterName.STRING
# Int test
try:
int(metadata[p])
except ValueError:
pass
except TypeError:
pass
else:
datatype = ParameterName.NUMERIC
# Fraction test
if isinstance(metadata[p], Fraction):
datatype = ParameterName.NUMERIC
# Float test
try:
float(metadata[p])
except ValueError:
pass
except TypeError:
pass
else:
datatype = ParameterName.NUMERIC
new_param = ParameterName(schema=schema,
name=p,
full_name=p,
data_type=datatype)
new_param.save()
parameters.append(new_param)
return parameters
def setUp(self):
# Load schemas for test
from django.core.management import call_command
call_command('loaddata', 'as_schemas')
self.client = Client()
self.experiments = []
try:
user = User.objects.get(username='******')
except User.DoesNotExist:
user = User.objects.create_user('test', '', 'test')
user.save()
self.userprofile = user.userprofile
# base_path = path.abspath(path.dirname(__file__))
experiment = Experiment(title='SAXS Test',
created_by=user)
experiment.save()
acl = ObjectACL(pluginId=django_user,
entityId=str(user.id),
content_object=experiment,
canRead=True,
canWrite=True,
canDelete=True,
isOwner=True)
acl.save()
self.experiments += [experiment]
schema = Schema.objects.get(type=Schema.DATAFILE, subtype='saxs')
parameter = ParameterName.objects.get(schema=schema, name='io')
parameter.is_searchable = True
parameter.save()
self.io_param_name = parameter.getUniqueShortName()
schema = Schema.objects.get(type=Schema.DATASET, subtype='saxs')
parameter = ParameterName.objects.get(schema=schema, name='frqimn')
parameter.is_searchable = True
parameter.save()
self.frqimn_param_name = parameter.getUniqueShortName()
new_schema = Schema()
new_schema.namespace = 'testschemawithduplicatename'
new_schema.save()
new_param = ParameterName(
schema=new_schema,
name='title',
full_name='Duplicate title parametername',
is_searchable=True)
new_param.save()
def _get_parameters(schema,metadata):
""" Returns set of parameters from schema matched to elements in
metadata, or creates them based on the metadata values. Types
are based on seen values, favouring numerics over strings.
"""
param_objects = ParameterName.objects.filter(schema=schema)
logger.debug("param_objects=%s\n" % param_objects)
parameters = []
for p in metadata.keys():
logger.debug("pp=%s\n" % p)
logger.debug("metadata[pp][0] = %s" % metadata[p][0])
logger.debug("metadata[pp][1] = %s" % metadata[p][1])
parameter = filter(lambda x: x.name == p,param_objects)
logger.debug("pparameter=%s\n" % parameter)
if parameter:
parameters.append(parameter[0])
continue
# work out the best type from the value
datatype = ParameterName.LONGSTRING
units = None
try:
int(metadata[p][0])
except (ValueError,TypeError):
pass
else:
datatype = ParameterName.NUMERIC
units = str(metadata[p][1])
try:
float(metadata[p][0])
except (ValueError,TypeError):
pass
else:
datatype = ParameterName.NUMERIC
units = str(metadata[p][1])
logger.debug("units=%s" % str(metadata[p][1]))
if units:
new_param = ParameterName(schema=schema,
name=p,
full_name=p,
data_type=datatype,
units=units)
else:
new_param = ParameterName(schema=schema,
name=p,
full_name=p,
data_type=datatype)
new_param.save()
logger.debug("new_param=%s" % new_param)
logger.debug("datatype=%s" % datatype)
parameters.append(new_param)
return parameters
def getOrCreateParameterNames(self, schema, paramnames):
""" Takes a list of paramnames (defined in the __init__ method) to get or create new parameter names objects
"""
pn_objects = []
for paramname in paramnames:
param_objects = ParameterName.objects.filter(schema=schema, name=paramname['name'])
if len(param_objects) == 0:
pn = ParameterName(schema=schema, name=paramname['name'], full_name=paramname['full_name'],
data_type=paramname['data_type'])
pn.save()
else:
pn = param_objects[0]
pn_objects.append(pn)
return pn_objects
def get_parameters(schema, metadata):
param_objects = ParameterName.objects.filter(schema=schema)
logger.debug("param_objects=%s\n" % param_objects)
parameters = []
for p in metadata.keys():
logger.debug("pp=%s\n" % p)
logger.debug("metadata[pp][0] = %s" % metadata[p][0])
logger.debug("metadata[pp][1] = %s" % metadata[p][1])
parameter = filter(lambda x: x.name == p, param_objects)
logger.debug("pparameter=%s\n" % parameter)
if parameter:
parameters.append(parameter[0])
continue
datatype = ParameterName.STRING
units = None
try:
int(metadata[p][0])
except (ValueError, TypeError):
pass
else:
datatype = ParameterName.NUMERIC
units = str(metadata[p][1])
try:
float(metadata[p][0])
except (ValueError, TypeError):
pass
else:
datatype = ParameterName.NUMERIC
units = str(metadata[p][1])
logger.debug("units=%s" % str(metadata[p][1]))
if units:
new_param = ParameterName(schema=schema, name=p, full_name=p, data_type=datatype, units=units)
else:
new_param = ParameterName(schema=schema, name=p, full_name=p, data_type=datatype)
new_param.save()
logger.debug("new_param=%s" % new_param)
logger.debug("datatype=%s" % datatype)
parameters.append(new_param)
return parameters
def test_contextual_view(self):
"""
Given schema on dataset, check that image file created
"""
user = _create_test_user()
license = _create_license()
exp = _create_test_experiment(user, license)
ds = Dataset(description='happy snaps of plumage')
ds.save()
ds = _create_test_dataset(
ds, exp.id, {
"output.dat": 'test data\n',
"grexp.dat": '1 2\n2 3\n3 7\n',
"grfinal21.dat": '1 2\n 2 4\n4 9\n'
})
sch = Schema(namespace=self.HRMCSCHEMA,
name="hrmc_views",
type=Schema.DATASET)
sch.save()
param = ParameterName(schema=sch,
name="plot",
full_name="scatterplot",
units="image",
data_type=ParameterName.FILENAME)
param.save()
dps = DatasetParameterSet(schema=sch, dataset=ds)
dps.save()
ds.experiments.add(exp)
ds.save()
client = Client()
response = client.get('/dataset/%s' % ds.id)
self.assertEqual(response.status_code, 200)
param_sets = get_param_sets(ds)
self.assertTrue(param_sets)
dp = DatasetParameter.objects.get(parameterset=param_sets[0],
name=param)
self.assertTrue(dp)
self.assertNotEquals(dp.string_value, "") # ie, it has a filename
def test_hrmc_filter(self):
"""
Make an experiment, lood up grexp file and check
dataset schema missing, then loadup grfinal and check dataset schema
created
"""
user = _create_test_user()
license = _create_license()
exp = _create_test_experiment(user, license)
ds = Dataset(description='happy snaps of plumage')
ds.save()
_create_test_dataset(ds, exp.id, {
"output.dat": 'hello',
"grexp.dat": '2 5\n6 15\n'
})
ds.experiments.add(exp)
ds.save()
sch = Schema(namespace=self.HRMCSCHEMA,
name="hrmc_views",
type=Schema.DATASET)
sch.save()
param = ParameterName(schema=sch,
name="plot",
full_name="scatterplot",
units="image",
data_type=ParameterName.FILENAME)
param.save()
param_sets = get_param_sets(ds)
self.assertEquals(list(param_sets), [])
_create_test_dataset(ds, exp.id, {'grfinal21.dat': "1 3\n5 14\n"})
df2 = Dataset_File(dataset=ds, url='path/grfinal21.dat')
df2.save()
h = hrmc.HRMCOutput('HRMC', self.HRMCSCHEMA)
h(sender=Dataset_File, instance=df2)
param_sets = get_param_sets(ds)
self.assertEquals([x.schema.namespace for x in param_sets],
[self.HRMCSCHEMA])
def test_contextual_view(self):
"""
Given schema on dataset, check that image file created
"""
user = _create_test_user()
license = _create_license()
exp = _create_test_experiment(user, license)
ds = Dataset(description='happy snaps of plumage')
ds.save()
ds = _create_test_dataset(ds, exp.id, {
"output.dat": 'test data\n',
"grexp.dat": '1 2\n2 3\n3 7\n',
"grfinal21.dat": '1 2\n 2 4\n4 9\n'})
sch = Schema(namespace=self.HRMCSCHEMA,
name="hrmc_views", type=Schema.DATASET)
sch.save()
param = ParameterName(schema=sch, name="plot",
full_name="scatterplot", units="image",
data_type=ParameterName.FILENAME
)
param.save()
dps = DatasetParameterSet(schema=sch, dataset=ds)
dps.save()
ds.experiments.add(exp)
ds.save()
client = Client()
response = client.get('/dataset/%s' % ds.id)
self.assertEqual(response.status_code, 200)
param_sets = get_param_sets(ds)
self.assertTrue(param_sets)
dp = DatasetParameter.objects.get(parameterset=param_sets[0],
name=param)
self.assertTrue(dp)
self.assertNotEquals(dp.string_value, "") # ie, it has a filename
def getOrCreateParameterNames(self, schema, paramnames):
""" Takes a list of paramnames (defined in the __init__ method) to get or create new parameter names objects
"""
pn_objects = []
for paramname in paramnames:
param_objects = ParameterName.objects.filter(
schema=schema, name=paramname['name'])
if len(param_objects) == 0:
pn = ParameterName(schema=schema,
name=paramname['name'],
full_name=paramname['full_name'],
data_type=paramname['data_type'])
pn.save()
else:
pn = param_objects[0]
pn_objects.append(pn)
return pn_objects
def test_hrmc_filter(self):
"""
Make an experiment, lood up grexp file and check
dataset schema missing, then loadup grfinal and check dataset schema
created
"""
user = _create_test_user()
license = _create_license()
exp = _create_test_experiment(user, license)
ds = Dataset(description='happy snaps of plumage')
ds.save()
_create_test_dataset(ds, exp.id,
{"output.dat": 'hello', "grexp.dat": '2 5\n6 15\n'})
ds.experiments.add(exp)
ds.save()
sch = Schema(namespace=self.HRMCSCHEMA,
name="hrmc_views", type=Schema.DATASET)
sch.save()
param = ParameterName(schema=sch, name="plot",
full_name="scatterplot", units="image",
data_type=ParameterName.FILENAME
)
param.save()
param_sets = get_param_sets(ds)
self.assertEquals(list(param_sets), [])
_create_test_dataset(ds, exp.id, {'grfinal21.dat': "1 3\n5 14\n"})
df2 = Dataset_File(dataset=ds, url='path/grfinal21.dat')
df2.save()
h = hrmc.HRMCOutput('HRMC', self.HRMCSCHEMA)
h(sender=Dataset_File, instance=df2)
param_sets = get_param_sets(ds)
self.assertEquals([x.schema.namespace for x in param_sets],
[self.HRMCSCHEMA])
def save_rif_cs_profile(self, experiment, profile):
"""
Save selected profile choice as experiment parameter
"""
namespace = "http://monash.edu.au/rif-cs/profile/"
schema = None
try:
schema = Schema.objects.get(
namespace__exact=namespace)
except Schema.DoesNotExist:
logger.debug('Schema ' + namespace +
' does not exist. Creating.')
schema = Schema(namespace=namespace)
schema.save()
try:
parametername = ParameterName.objects.get(
schema__namespace__exact=schema.namespace,
name="profile")
except ParameterName.DoesNotExist:
logger.debug("profile does not exist. Creating.")
parametername = ParameterName(name="profile",schema=schema)
parametername.save()
parameterset = None
try:
parameterset = \
ExperimentParameterSet.objects.get(\
schema=schema,
experiment=experiment)
except ExperimentParameterSet.DoesNotExist, e:
parameterset = ExperimentParameterSet(\
schema=schema,
experiment=experiment)
parameterset.save()
def _get_parameters(schema, metadata):
""" Returns set of parameters from schema matched to elements in
metadata, or creates them based on the metadata values. Types
are based on seen values, favouring numerics over strings.
"""
param_objects = ParameterName.objects.filter(schema=schema)
logger.debug("param_objects=%s\n" % param_objects)
parameters = []
for p in metadata.keys():
logger.debug("pp=%s\n" % p)
logger.debug("metadata[pp][0] = %s" % metadata[p][0])
logger.debug("metadata[pp][1] = %s" % metadata[p][1])
parameter = filter(lambda x: x.name == p, param_objects)
logger.debug("pparameter=%s\n" % parameter)
if parameter:
parameters.append(parameter[0])
continue
# work out the best type from the value
datatype = ParameterName.LONGSTRING
units = None
try:
int(metadata[p][0])
except (ValueError, TypeError):
pass
else:
datatype = ParameterName.NUMERIC
units = str(metadata[p][1])
try:
float(metadata[p][0])
except (ValueError, TypeError):
pass
else:
datatype = ParameterName.NUMERIC
units = str(metadata[p][1])
logger.debug("units=%s" % str(metadata[p][1]))
if units:
new_param = ParameterName(schema=schema,
name=p,
full_name=p,
data_type=datatype,
units=units)
else:
new_param = ParameterName(schema=schema,
name=p,
full_name=p,
data_type=datatype)
new_param.save()
logger.debug("new_param=%s" % new_param)
logger.debug("datatype=%s" % datatype)
parameters.append(new_param)
return parameters
class ParameterSetManagerTestCase(TestCase):
def setUp(self):
from django.contrib.auth.models import User
from tempfile import mkdtemp
user = '******'
pwd = 'secret'
email = ''
self.user = User.objects.create_user(user, email, pwd)
self.test_dir = mkdtemp()
self.exp = Experiment(title='test exp1',
institution_name='monash',
created_by=self.user)
self.exp.save()
self.dataset = Dataset(description="dataset description...")
self.dataset.save()
self.dataset.experiments.add(self.exp)
self.dataset.save()
self.datafile = DataFile(dataset=self.dataset,
filename="testfile.txt",
size="42",
md5sum='bogus')
self.datafile.save()
self.dfo = DataFileObject(
datafile=self.datafile,
storage_box=self.datafile.get_default_storage_box(),
uri="1/testfile.txt")
self.dfo.save()
self.schema = Schema(namespace="http://localhost/psmtest/df/",
name="Parameter Set Manager",
type=3)
self.schema.save()
self.parametername1 = ParameterName(schema=self.schema,
name="parameter1",
full_name="Parameter 1")
self.parametername1.save()
self.parametername2 = ParameterName(schema=self.schema,
name="parameter2",
full_name="Parameter 2",
data_type=ParameterName.NUMERIC)
self.parametername2.save()
self.parametername3 = ParameterName(schema=self.schema,
name="parameter3",
full_name="Parameter 3",
data_type=ParameterName.DATETIME)
self.parametername3.save()
self.datafileparameterset = DatafileParameterSet(
schema=self.schema, datafile=self.datafile)
self.datafileparameterset.save()
self.datafileparameter1 = DatafileParameter(
parameterset=self.datafileparameterset,
name=self.parametername1,
string_value="test1")
self.datafileparameter1.save()
self.datafileparameter2 = DatafileParameter(
parameterset=self.datafileparameterset,
name=self.parametername2,
numerical_value=2)
self.datafileparameter2.save()
# Create a ParameterName and Parameter of type LINK to an experiment
self.parametername_exp_link = ParameterName(
schema=self.schema,
name="exp_link",
full_name="This parameter is a experiment LINK",
data_type=ParameterName.LINK)
self.parametername_exp_link.save()
self.exp_link_param = DatafileParameter(
parameterset=self.datafileparameterset,
name=self.parametername_exp_link)
exp_url = self.exp.get_absolute_url() # /experiment/view/1/
self.exp_link_param.set_value(exp_url)
self.exp_link_param.save()
# Create a ParameterName and Parameter of type LINK to a dataset
self.parametername_dataset_link = ParameterName(
schema=self.schema,
name="dataset_link",
full_name="This parameter is a dataset LINK",
data_type=ParameterName.LINK)
self.parametername_dataset_link.save()
self.dataset_link_param = DatafileParameter(
parameterset=self.datafileparameterset,
name=self.parametername_dataset_link)
dataset_url = self.dataset.get_absolute_url() # /dataset/1/
self.dataset_link_param.set_value(dataset_url)
self.dataset_link_param.save()
# Create a ParameterName type LINK to an unresolvable (non-URL)
# free-text value
self.parametername_unresolvable_link = ParameterName(
schema=self.schema,
name="freetext_link",
full_name="This parameter is a non-URL LINK",
data_type=ParameterName.LINK)
self.parametername_unresolvable_link.save()
def tearDown(self):
self.exp.delete()
self.user.delete()
self.parametername1.delete()
self.parametername2.delete()
self.parametername3.delete()
self.parametername_exp_link.delete()
self.parametername_dataset_link.delete()
self.parametername_unresolvable_link.delete()
self.schema.delete()
def test_existing_parameterset(self):
psm = ParameterSetManager(parameterset=self.datafileparameterset)
self.assertTrue(
psm.get_schema().namespace == "http://localhost/psmtest/df/")
self.assertTrue(psm.get_param("parameter1").string_value == "test1")
self.assertTrue(psm.get_param("parameter2", True) == 2)
def test_new_parameterset(self):
psm = ParameterSetManager(parentObject=self.datafile,
schema="http://localhost/psmtest/df2/")
self.assertTrue(
psm.get_schema().namespace == "http://localhost/psmtest/df2/")
psm.set_param("newparam1", "test3", "New Parameter 1")
self.assertTrue(psm.get_param("newparam1").string_value == "test3")
self.assertTrue(
psm.get_param("newparam1").name.full_name == "New Parameter 1")
psm.new_param("newparam1", "test4")
self.assertTrue(len(psm.get_params("newparam1", True)) == 2)
psm.set_param_list("newparam2", ("a", "b", "c", "d"))
self.assertTrue(len(psm.get_params("newparam2")) == 4)
psm.set_params_from_dict({"newparam2": "test5", "newparam3": 3})
self.assertTrue(psm.get_param("newparam2", True) == "test5")
# the newparam3 gets created and '3' is set to a string_value
# since once cannot assume that an initial numeric value
# will imply continuing numeric type for this new param
self.assertTrue(psm.get_param("newparam3").string_value == '3')
psm.delete_params("newparam1")
self.assertTrue(len(psm.get_params("newparam1", True)) == 0)
def test_link_parameter_type(self):
"""
Test that Parameter.link_gfk (GenericForeignKey) is correctly
assigned after using Parameter.set_value(some_url) for a LINK Parameter.
"""
psm = ParameterSetManager(parameterset=self.datafileparameterset)
# Check link to experiment
exp_url = self.exp.get_absolute_url() # /experiment/view/1/
self.assertTrue(psm.get_param("exp_link").string_value == exp_url)
self.assertTrue(psm.get_param("exp_link").link_id == self.exp.id)
exp_ct = ContentType.objects.get(model__iexact="experiment")
self.assertTrue(psm.get_param("exp_link").link_ct == exp_ct)
self.assertTrue(psm.get_param("exp_link").link_gfk == self.exp)
# Check link to dataset
dataset_url = self.dataset.get_absolute_url() # /dataset/1/
self.assertTrue(
psm.get_param("dataset_link").string_value == dataset_url)
self.assertTrue(
psm.get_param("dataset_link").link_id == self.dataset.id)
dataset_ct = ContentType.objects.get(model__iexact="dataset")
self.assertTrue(psm.get_param("dataset_link").link_ct == dataset_ct)
self.assertTrue(psm.get_param("dataset_link").link_gfk == self.dataset)
def test_link_parameter_type_extra(self):
# make a second ParameterSet for testing some variations
# in URL values
self.datafileparameterset2 = DatafileParameterSet(
schema=self.schema, datafile=self.datafile)
self.datafileparameterset2.save()
psm = ParameterSetManager(parameterset=self.datafileparameterset2)
self.dataset_link_param2 = DatafileParameter(
parameterset=self.datafileparameterset2,
name=self.parametername_dataset_link)
# /dataset/1 - no trailing slash
dataset_url = self.dataset.get_absolute_url()
self.dataset_link_param2.set_value(dataset_url)
self.dataset_link_param2.save()
# Check link_id/link_ct/link_gfk to dataset
self.assertTrue(
psm.get_param("dataset_link").link_id == self.dataset.id)
dataset_ct = ContentType.objects.get(model__iexact="dataset")
self.assertTrue(psm.get_param("dataset_link").link_ct == dataset_ct)
self.assertTrue(psm.get_param("dataset_link").link_gfk == self.dataset)
# Test links of the form /api/v1/experiment/<experiment_id>/
self.exp_link_param2 = DatafileParameter(
parameterset=self.datafileparameterset2,
name=self.parametername_exp_link)
exp_url = '/api/v1/experiment/%s/' % self.exp.id
self.exp_link_param2.set_value(exp_url)
self.exp_link_param2.save()
# Check link_id/link_ct/link_gfk to experiment
self.assertTrue(psm.get_param("exp_link").link_id == self.exp.id)
exp_ct = ContentType.objects.get(model__iexact="experiment")
self.assertTrue(psm.get_param("exp_link").link_ct == exp_ct)
self.assertTrue(psm.get_param("exp_link").link_gfk == self.exp)
def test_unresolvable_link_parameter(self):
"""
Test that LINK Parameters that can't be resolved to a model (including
non-URL values) still work.
"""
self.datafileparameterset3 = DatafileParameterSet(
schema=self.schema, datafile=self.datafile)
self.datafileparameterset3.save()
psm = ParameterSetManager(parameterset=self.datafileparameterset3)
# Create a Parameter of type LINK to an unresolvable (non-URL)
# free-text value
self.freetext_link_param = DatafileParameter(
parameterset=self.datafileparameterset3,
name=self.parametername_unresolvable_link)
self.assertRaises(
SuspiciousOperation,
lambda: self.freetext_link_param.set_value("FREETEXT_ID_123"))
def test_tz_naive_date_handling(self):
"""
Ensure that dates are handling in a timezone-aware way.
"""
psm = ParameterSetManager(parameterset=self.datafileparameterset)
psm.new_param("parameter3", str(datetime(1970, 01, 01, 10, 0, 0)))
expect(psm.get_param("parameter3", True))\
.to_equal(datetime(1970, 01, 01, 0, 0, 0, tzinfo=pytz.utc))
def test_tz_aware_date_handling(self):
"""
Ensure that dates are handling in a timezone-aware way.
"""
psm = ParameterSetManager(parameterset=self.datafileparameterset)
psm.new_param("parameter3", '1970-01-01T08:00:00+08:00')
expect(psm.get_param("parameter3", True))\
.to_equal(datetime(1970, 01, 01, 0, 0, 0, tzinfo=pytz.utc))
def setUp(self):
from django.contrib.auth.models import User
from tempfile import mkdtemp
user = '******'
pwd = 'secret'
email = ''
self.user = User.objects.create_user(user, email, pwd)
self.test_dir = mkdtemp()
self.exp = Experiment(title='test exp1',
institution_name='monash',
created_by=self.user)
self.exp.save()
self.dataset = Dataset(description="dataset description...")
self.dataset.save()
self.dataset.experiments.add(self.exp)
self.dataset.save()
self.datafile = DataFile(dataset=self.dataset,
filename="testfile.txt",
size="42", md5sum='bogus')
self.datafile.save()
self.dfo = DataFileObject(
datafile=self.datafile,
storage_box=self.datafile.get_default_storage_box(),
uri="1/testfile.txt")
self.dfo.save()
self.schema = Schema(
namespace="http://localhost/psmtest/df/",
name="Parameter Set Manager", type=3)
self.schema.save()
self.parametername1 = ParameterName(
schema=self.schema, name="parameter1",
full_name="Parameter 1")
self.parametername1.save()
self.parametername2 = ParameterName(
schema=self.schema, name="parameter2",
full_name="Parameter 2",
data_type=ParameterName.NUMERIC)
self.parametername2.save()
self.parametername3 = ParameterName(
schema=self.schema, name="parameter3",
full_name="Parameter 3",
data_type=ParameterName.DATETIME)
self.parametername3.save()
self.datafileparameterset = DatafileParameterSet(
schema=self.schema, datafile=self.datafile)
self.datafileparameterset.save()
self.datafileparameter1 = DatafileParameter(
parameterset=self.datafileparameterset,
name=self.parametername1, string_value="test1")
self.datafileparameter1.save()
self.datafileparameter2 = DatafileParameter(
parameterset=self.datafileparameterset,
name=self.parametername2, numerical_value=2)
self.datafileparameter2.save()
# Create a ParameterName and Parameter of type LINK to an experiment
self.parametername_exp_link = ParameterName(
schema=self.schema, name="exp_link",
full_name="This parameter is a experiment LINK",
data_type=ParameterName.LINK)
self.parametername_exp_link.save()
self.exp_link_param = DatafileParameter(
parameterset=self.datafileparameterset,
name=self.parametername_exp_link)
exp_url = self.exp.get_absolute_url() # /experiment/view/1/
self.exp_link_param.set_value(exp_url)
self.exp_link_param.save()
# Create a ParameterName and Parameter of type LINK to a dataset
self.parametername_dataset_link = ParameterName(
schema=self.schema, name="dataset_link",
full_name="This parameter is a dataset LINK",
data_type=ParameterName.LINK)
self.parametername_dataset_link.save()
self.dataset_link_param = DatafileParameter(
parameterset=self.datafileparameterset,
name=self.parametername_dataset_link)
dataset_url = self.dataset.get_absolute_url() # /dataset/1/
self.dataset_link_param.set_value(dataset_url)
self.dataset_link_param.save()
# Create a ParameterName type LINK to an unresolvable (non-URL)
# free-text value
self.parametername_unresolvable_link = ParameterName(
schema=self.schema, name="freetext_link",
full_name="This parameter is a non-URL LINK",
data_type=ParameterName.LINK)
self.parametername_unresolvable_link.save()
class ParameterSetManagerTestCase(TestCase):
def setUp(self):
from django.contrib.auth.models import User
from tempfile import mkdtemp
user = '******'
pwd = 'secret'
email = ''
self.user = User.objects.create_user(user, email, pwd)
self.test_dir = mkdtemp()
self.exp = Experiment(title='test exp1',
institution_name='monash',
created_by=self.user)
self.exp.save()
self.dataset = Dataset(description="dataset description...")
self.dataset.save()
self.dataset.experiments.add(self.exp)
self.dataset.save()
self.datafile = DataFile(dataset=self.dataset,
filename="testfile.txt",
size="42", md5sum='bogus')
self.datafile.save()
self.dfo = DataFileObject(
datafile=self.datafile,
storage_box=self.datafile.get_default_storage_box(),
uri="1/testfile.txt")
self.dfo.save()
self.schema = Schema(
namespace="http://localhost/psmtest/df/",
name="Parameter Set Manager", type=3)
self.schema.save()
self.parametername1 = ParameterName(
schema=self.schema, name="parameter1",
full_name="Parameter 1")
self.parametername1.save()
self.parametername2 = ParameterName(
schema=self.schema, name="parameter2",
full_name="Parameter 2",
data_type=ParameterName.NUMERIC)
self.parametername2.save()
self.parametername3 = ParameterName(
schema=self.schema, name="parameter3",
full_name="Parameter 3",
data_type=ParameterName.DATETIME)
self.parametername3.save()
self.datafileparameterset = DatafileParameterSet(
schema=self.schema, datafile=self.datafile)
self.datafileparameterset.save()
self.datafileparameter1 = DatafileParameter(
parameterset=self.datafileparameterset,
name=self.parametername1, string_value="test1")
self.datafileparameter1.save()
self.datafileparameter2 = DatafileParameter(
parameterset=self.datafileparameterset,
name=self.parametername2, numerical_value=2)
self.datafileparameter2.save()
# Create a ParameterName and Parameter of type LINK to an experiment
self.parametername_exp_link = ParameterName(
schema=self.schema, name="exp_link",
full_name="This parameter is a experiment LINK",
data_type=ParameterName.LINK)
self.parametername_exp_link.save()
self.exp_link_param = DatafileParameter(
parameterset=self.datafileparameterset,
name=self.parametername_exp_link)
exp_url = self.exp.get_absolute_url() # /experiment/view/1/
self.exp_link_param.set_value(exp_url)
self.exp_link_param.save()
# Create a ParameterName and Parameter of type LINK to a dataset
self.parametername_dataset_link = ParameterName(
schema=self.schema, name="dataset_link",
full_name="This parameter is a dataset LINK",
data_type=ParameterName.LINK)
self.parametername_dataset_link.save()
self.dataset_link_param = DatafileParameter(
parameterset=self.datafileparameterset,
name=self.parametername_dataset_link)
dataset_url = self.dataset.get_absolute_url() # /dataset/1/
self.dataset_link_param.set_value(dataset_url)
self.dataset_link_param.save()
# Create a ParameterName type LINK to an unresolvable (non-URL)
# free-text value
self.parametername_unresolvable_link = ParameterName(
schema=self.schema, name="freetext_link",
full_name="This parameter is a non-URL LINK",
data_type=ParameterName.LINK)
self.parametername_unresolvable_link.save()
def tearDown(self):
self.exp.delete()
self.user.delete()
self.parametername1.delete()
self.parametername2.delete()
self.parametername3.delete()
self.parametername_exp_link.delete()
self.parametername_dataset_link.delete()
self.parametername_unresolvable_link.delete()
self.schema.delete()
def test_existing_parameterset(self):
psm = ParameterSetManager(parameterset=self.datafileparameterset)
self.assertTrue(psm.get_schema().namespace ==
"http://localhost/psmtest/df/")
self.assertTrue(psm.get_param("parameter1").string_value == "test1")
self.assertTrue(psm.get_param("parameter2", True) == 2)
def test_new_parameterset(self):
psm = ParameterSetManager(parentObject=self.datafile,
schema="http://localhost/psmtest/df2/")
self.assertTrue(psm.get_schema().namespace ==
"http://localhost/psmtest/df2/")
psm.set_param("newparam1", "test3", "New Parameter 1")
self.assertTrue(psm.get_param("newparam1").string_value ==
"test3")
self.assertTrue(psm.get_param("newparam1").name.full_name ==
"New Parameter 1")
psm.new_param("newparam1", "test4")
self.assertTrue(len(psm.get_params("newparam1", True)) == 2)
psm.set_param_list("newparam2", ("a", "b", "c", "d"))
self.assertTrue(len(psm.get_params("newparam2")) == 4)
psm.set_params_from_dict(
{"newparam2": "test5", "newparam3": 3})
self.assertTrue(psm.get_param("newparam2", True) == "test5")
# the newparam3 gets created and '3' is set to a string_value
# since once cannot assume that an initial numeric value
# will imply continuing numeric type for this new param
self.assertTrue(psm.get_param("newparam3").string_value == '3')
psm.delete_params("newparam1")
self.assertTrue(len(psm.get_params("newparam1", True)) == 0)
def test_link_parameter_type(self):
"""
Test that Parameter.link_gfk (GenericForeignKey) is correctly
assigned after using Parameter.set_value(some_url) for a LINK Parameter.
"""
psm = ParameterSetManager(parameterset=self.datafileparameterset)
# Check link to experiment
exp_url = self.exp.get_absolute_url() # /experiment/view/1/
self.assertTrue(psm.get_param("exp_link").string_value ==
exp_url)
self.assertTrue(psm.get_param("exp_link").link_id ==
self.exp.id)
exp_ct = ContentType.objects.get(model__iexact="experiment")
self.assertTrue(psm.get_param("exp_link").link_ct == exp_ct)
self.assertTrue(psm.get_param("exp_link").link_gfk == self.exp)
# Check link to dataset
dataset_url = self.dataset.get_absolute_url() # /dataset/1/
self.assertTrue(psm.get_param("dataset_link").string_value ==
dataset_url)
self.assertTrue(psm.get_param("dataset_link").link_id ==
self.dataset.id)
dataset_ct = ContentType.objects.get(model__iexact="dataset")
self.assertTrue(psm.get_param("dataset_link").link_ct == dataset_ct)
self.assertTrue(psm.get_param("dataset_link").link_gfk == self.dataset)
def test_link_parameter_type_extra(self):
# make a second ParameterSet for testing some variations
# in URL values
self.datafileparameterset2 = DatafileParameterSet(
schema=self.schema, datafile=self.datafile)
self.datafileparameterset2.save()
psm = ParameterSetManager(parameterset=self.datafileparameterset2)
self.dataset_link_param2 = DatafileParameter(
parameterset=self.datafileparameterset2,
name=self.parametername_dataset_link)
# /dataset/1 - no trailing slash
dataset_url = self.dataset.get_absolute_url()
self.dataset_link_param2.set_value(dataset_url)
self.dataset_link_param2.save()
# Check link_id/link_ct/link_gfk to dataset
self.assertTrue(psm.get_param("dataset_link").link_id ==
self.dataset.id)
dataset_ct = ContentType.objects.get(model__iexact="dataset")
self.assertTrue(psm.get_param("dataset_link").link_ct == dataset_ct)
self.assertTrue(psm.get_param("dataset_link").link_gfk == self.dataset)
# Test links of the form /api/v1/experiment/<experiment_id>/
self.exp_link_param2 = DatafileParameter(
parameterset=self.datafileparameterset2,
name=self.parametername_exp_link)
exp_url = '/api/v1/experiment/%s/' % self.exp.id
self.exp_link_param2.set_value(exp_url)
self.exp_link_param2.save()
# Check link_id/link_ct/link_gfk to experiment
self.assertTrue(psm.get_param("exp_link").link_id ==
self.exp.id)
exp_ct = ContentType.objects.get(model__iexact="experiment")
self.assertTrue(psm.get_param("exp_link").link_ct == exp_ct)
self.assertTrue(psm.get_param("exp_link").link_gfk == self.exp)
def test_unresolvable_link_parameter(self):
"""
Test that LINK Parameters that can't be resolved to a model (including
non-URL values) still work.
"""
self.datafileparameterset3 = DatafileParameterSet(
schema=self.schema, datafile=self.datafile)
self.datafileparameterset3.save()
psm = ParameterSetManager(parameterset=self.datafileparameterset3)
# Create a Parameter of type LINK to an unresolvable (non-URL)
# free-text value
self.freetext_link_param = DatafileParameter(
parameterset=self.datafileparameterset3,
name=self.parametername_unresolvable_link)
self.assertRaises(SuspiciousOperation,
lambda: self.freetext_link_param.set_value(
"FREETEXT_ID_123"))
def test_tz_naive_date_handling(self):
"""
Ensure that dates are handling in a timezone-aware way.
"""
psm = ParameterSetManager(parameterset=self.datafileparameterset)
psm.new_param("parameter3", str(datetime(1970, 01, 01, 10, 0, 0)))
expect(psm.get_param("parameter3", True))\
.to_equal(datetime(1970, 01, 01, 0, 0, 0, tzinfo=pytz.utc))
def test_tz_aware_date_handling(self):
"""
Ensure that dates are handling in a timezone-aware way.
"""
psm = ParameterSetManager(parameterset=self.datafileparameterset)
psm.new_param("parameter3",
'1970-01-01T08:00:00+08:00')
expect(psm.get_param("parameter3", True))\
.to_equal(datetime(1970, 01, 01, 0, 0, 0, tzinfo=pytz.utc))
def _get_create_parname(self, parname,
fullparname=None, example_value=None):
try:
paramName = ParameterName.objects.get(name=parname,
schema__id=self.get_schema().id)
except ObjectDoesNotExist:
paramName = ParameterName()
paramName.schema = self.get_schema()
paramName.name = parname
if fullparname:
paramName.full_name = fullparname
else:
paramName.full_name = parname
if example_value:
try:
float(example_value)
paramName.data_type = ParameterName.NUMERIC
except (TypeError, ValueError):
paramName.data_type = ParameterName.STRING
else:
paramName.data_type = ParameterName.STRING
paramName.is_searchable = True
paramName.save()
return paramName
def test_parameter(self):
exp = Experiment(
title='test exp1',
institution_name='Australian Synchrotron',
approved=True,
created_by=self.user,
public_access=Experiment.PUBLIC_ACCESS_NONE,
)
exp.save()
dataset = Dataset(description="dataset description")
dataset.save()
dataset.experiments.add(exp)
dataset.save()
df_file = DataFile(dataset=dataset,
filename='file.txt',
size=42,
md5sum='bogus')
df_file.save()
df_schema = Schema(namespace='http://www.cern.ch/felzmann/schema1.xml',
type=Schema.DATAFILE)
df_schema.save()
ds_schema = Schema(namespace='http://www.cern.ch/felzmann/schema2.xml',
type=Schema.DATASET)
ds_schema.save()
exp_schema = Schema(
namespace='http://www.cern.ch/felzmann/schema3.xml',
type=Schema.EXPERIMENT)
exp_schema.save()
df_parname = ParameterName(schema=df_schema,
name='name',
full_name='full_name',
units='image/jpg',
data_type=ParameterName.FILENAME)
df_parname.save()
ds_parname = ParameterName(schema=ds_schema,
name='name',
full_name='full_name',
units='image/jpg',
data_type=ParameterName.FILENAME)
ds_parname.save()
exp_parname = ParameterName(schema=exp_schema,
name='name',
full_name='full_name',
units='image/jpg',
data_type=ParameterName.FILENAME)
exp_parname.save()
df_parset = DatafileParameterSet(schema=df_schema, datafile=df_file)
df_parset.save()
ds_parset = DatasetParameterSet(schema=ds_schema, dataset=dataset)
ds_parset.save()
exp_parset = ExperimentParameterSet(schema=exp_schema, experiment=exp)
exp_parset.save()
with self.settings(METADATA_STORE_PATH=os.path.dirname(__file__)):
filename = 'test.jpg'
df_parameter = DatafileParameter(name=df_parname,
parameterset=df_parset,
string_value=filename)
df_parameter.save()
ds_parameter = DatasetParameter(name=ds_parname,
parameterset=ds_parset,
string_value=filename)
ds_parameter.save()
exp_parameter = ExperimentParameter(name=exp_parname,
parameterset=exp_parset,
string_value=filename)
exp_parameter.save()
self.assertEqual(
"<a href='/display/DatafileImage/load/%i/' target='_blank'><img style='width: 300px;' src='/display/DatafileImage/load/%i/' /></a>"
% # noqa
(df_parameter.id, df_parameter.id),
df_parameter.get())
self.assertEqual(
"<a href='/display/DatasetImage/load/%i/' target='_blank'><img style='width: 300px;' src='/display/DatasetImage/load/%i/' /></a>"
% # noqa
(ds_parameter.id, ds_parameter.id),
ds_parameter.get())
self.assertEqual(
"<a href='/display/ExperimentImage/load/%i/' target='_blank'><img style='width: 300px;' src='/display/ExperimentImage/load/%i/' /></a>"
% # noqa
(exp_parameter.id, exp_parameter.id),
exp_parameter.get())
def setUp(self):
from django.contrib.auth.models import User
from tempfile import mkdtemp
user = '******'
pwd = 'secret'
email = ''
self.user = User.objects.create_user(user, email, pwd)
self.test_dir = mkdtemp()
self.exp = Experiment(title='test exp1',
institution_name='monash',
created_by=self.user)
self.exp.save()
self.dataset = Dataset(description="dataset description...")
self.dataset.save()
self.dataset.experiments.add(self.exp)
self.dataset.save()
self.datafile = DataFile(dataset=self.dataset,
filename="testfile.txt",
size="42",
md5sum='bogus')
self.datafile.save()
self.dfo = DataFileObject(
datafile=self.datafile,
storage_box=self.datafile.get_default_storage_box(),
uri="1/testfile.txt")
self.dfo.save()
self.schema = Schema(namespace="http://localhost/psmtest/df/",
name="Parameter Set Manager",
type=3)
self.schema.save()
self.parametername1 = ParameterName(schema=self.schema,
name="parameter1",
full_name="Parameter 1")
self.parametername1.save()
self.parametername2 = ParameterName(schema=self.schema,
name="parameter2",
full_name="Parameter 2",
data_type=ParameterName.NUMERIC)
self.parametername2.save()
self.parametername3 = ParameterName(schema=self.schema,
name="parameter3",
full_name="Parameter 3",
data_type=ParameterName.DATETIME)
self.parametername3.save()
self.datafileparameterset = DatafileParameterSet(
schema=self.schema, datafile=self.datafile)
self.datafileparameterset.save()
self.datafileparameter1 = DatafileParameter(
parameterset=self.datafileparameterset,
name=self.parametername1,
string_value="test1")
self.datafileparameter1.save()
self.datafileparameter2 = DatafileParameter(
parameterset=self.datafileparameterset,
name=self.parametername2,
numerical_value=2)
self.datafileparameter2.save()
# Create a ParameterName and Parameter of type LINK to an experiment
self.parametername_exp_link = ParameterName(
schema=self.schema,
name="exp_link",
full_name="This parameter is a experiment LINK",
data_type=ParameterName.LINK)
self.parametername_exp_link.save()
self.exp_link_param = DatafileParameter(
parameterset=self.datafileparameterset,
name=self.parametername_exp_link)
exp_url = self.exp.get_absolute_url() # /experiment/view/1/
self.exp_link_param.set_value(exp_url)
self.exp_link_param.save()
# Create a ParameterName and Parameter of type LINK to a dataset
self.parametername_dataset_link = ParameterName(
schema=self.schema,
name="dataset_link",
full_name="This parameter is a dataset LINK",
data_type=ParameterName.LINK)
self.parametername_dataset_link.save()
self.dataset_link_param = DatafileParameter(
parameterset=self.datafileparameterset,
name=self.parametername_dataset_link)
dataset_url = self.dataset.get_absolute_url() # /dataset/1/
self.dataset_link_param.set_value(dataset_url)
self.dataset_link_param.save()
# Create a ParameterName type LINK to an unresolvable (non-URL)
# free-text value
self.parametername_unresolvable_link = ParameterName(
schema=self.schema,
name="freetext_link",
full_name="This parameter is a non-URL LINK",
data_type=ParameterName.LINK)
self.parametername_unresolvable_link.save()
