def _get_parameters(schema, metadata):
""" Returns set of parameters from schema matched to elements in
metadata, or creates them based on the metadata values. Types
are based on seen values, favouring numerics over strings.
"""
param_objects = ParameterName.objects.filter(schema=schema)
logger.debug("param_objects=%s\n" % param_objects)
parameters = []
for p in metadata.keys():
logger.debug("pp=%s\n" % p)
logger.debug("metadata[pp][0] = %s" % metadata[p][0])
logger.debug("metadata[pp][1] = %s" % metadata[p][1])
parameter = filter(lambda x: x.name == p, param_objects)
logger.debug("pparameter=%s\n" % parameter)
if parameter:
parameters.append(parameter[0])
continue
# work out the best type from the value
datatype = ParameterName.LONGSTRING
units = None
try:
int(metadata[p][0])
except (ValueError, TypeError):
pass
else:
datatype = ParameterName.NUMERIC
units = str(metadata[p][1])
try:
float(metadata[p][0])
except (ValueError, TypeError):
pass
else:
datatype = ParameterName.NUMERIC
units = str(metadata[p][1])
logger.debug("units=%s" % str(metadata[p][1]))
if units:
new_param = ParameterName(schema=schema,
name=p,
full_name=p,
data_type=datatype,
units=units)
else:
new_param = ParameterName(schema=schema,
name=p,
full_name=p,
data_type=datatype)
new_param.save()
logger.debug("new_param=%s" % new_param)
logger.debug("datatype=%s" % datatype)
parameters.append(new_param)
return parameters
def test_instrument(self):
group = Group(name="Test Manager Group")
group.save()
facility = Facility(name="Test Facility", manager_group=group)
facility.save()
self.assertEqual(str(facility), "Test Facility")
instrument = Instrument(name="Test Instrument", facility=facility)
instrument.save()
self.assertEqual(str(instrument), "Test Instrument")
self.assertEqual(len(instrument.getParameterSets()), 0)
schema = Schema(namespace='test instrument schema namespace',
type=Schema.INSTRUMENT)
schema.save()
parname = ParameterName(schema=schema,
name='name',
full_name='full_name')
parname.save()
pset = InstrumentParameterSet.objects.create(instrument=instrument,
schema=schema)
pset.save()
self.assertEqual(len(instrument.getParameterSets()), 1)
def _get_create_parname(self,
parname,
fullparname=None,
example_value=None):
try:
paramName = ParameterName.objects.get(
name=parname, schema__id=self.get_schema().id)
except ObjectDoesNotExist:
paramName = ParameterName()
paramName.schema = self.get_schema()
paramName.name = parname
if fullparname:
paramName.full_name = fullparname
else:
paramName.full_name = parname
if example_value:
try:
float(example_value)
paramName.data_type = ParameterName.NUMERIC
except (TypeError, ValueError):
paramName.data_type = ParameterName.STRING
else:
paramName.data_type = ParameterName.STRING
paramName.is_searchable = True
paramName.save()
return paramName
def save_rif_cs_profile(self, experiment, profile):
"""
Save selected profile choice as experiment parameter
"""
namespace = "http://monash.edu.au/rif-cs/profile/"
schema = None
try:
schema = Schema.objects.get(namespace__exact=namespace)
except Schema.DoesNotExist:
logger.debug('Schema ' + namespace + ' does not exist. Creating.')
schema = Schema(namespace=namespace)
schema.save()
try:
parametername = ParameterName.objects.get(
schema__namespace__exact=schema.namespace, name="profile")
except ParameterName.DoesNotExist:
logger.debug("profile does not exist. Creating.")
parametername = ParameterName(name="profile", schema=schema)
parametername.save()
parameterset = None
try:
parameterset = \
ExperimentParameterSet.objects.get(\
schema=schema,
experiment=experiment)
except ExperimentParameterSet.DoesNotExist, e:
parameterset = ExperimentParameterSet(\
schema=schema,
experiment=experiment)
parameterset.save()
def getParamaters(self, schema, metadata):
"""Return a list of the paramaters that will be saved.
"""
param_objects = ParameterName.objects.filter(schema=schema)
parameters = []
for p in metadata:
if self.tagsToFind and not p in self.tagsToFind:
continue
if p in self.tagsToExclude:
continue
parameter = filter(lambda x: x.name == p, param_objects)
if parameter:
parameters.append(parameter[0])
continue
# detect type of parameter
datatype = ParameterName.STRING
# Int test
try:
int(metadata[p])
except ValueError:
pass
except TypeError:
pass
else:
datatype = ParameterName.NUMERIC
# Fraction test
if isinstance(metadata[p], Fraction):
datatype = ParameterName.NUMERIC
# Float test
try:
float(metadata[p])
except ValueError:
pass
except TypeError:
pass
else:
datatype = ParameterName.NUMERIC
new_param = ParameterName(schema=schema,
name=p,
full_name=p,
data_type=datatype)
new_param.save()
parameters.append(new_param)
return parameters
def _make_param(self, schema, name, paramtype):
try:
param = ParameterName.objects.get(schema=schema,
name=name,
data_type=paramtype)
except ParameterName.DoesNotExist:
logger.debug("%s does not exist. Creating." % name)
param = ParameterName(name=name,
schema=schema,
data_type=paramtype)
param.save()
return param
def test_contextual_view(self):
"""
Given schema on dataset, check that image file created
"""
user = _create_test_user()
license = _create_license()
exp = _create_test_experiment(user, license)
ds = Dataset(description='happy snaps of plumage')
ds.save()
ds = _create_test_dataset(
ds, exp.id, {
"output.dat": 'test data\n',
"grexp.dat": '1 2\n2 3\n3 7\n',
"grfinal21.dat": '1 2\n 2 4\n4 9\n'
})
sch = Schema(namespace=self.HRMCSCHEMA,
name="hrmc_views",
type=Schema.DATASET)
sch.save()
param = ParameterName(schema=sch,
name="plot",
full_name="scatterplot",
units="image",
data_type=ParameterName.FILENAME)
param.save()
dps = DatasetParameterSet(schema=sch, dataset=ds)
dps.save()
ds.experiments.add(exp)
ds.save()
client = Client()
response = client.get('/dataset/%s' % ds.id)
self.assertEqual(response.status_code, 200)
param_sets = get_param_sets(ds)
self.assertTrue(param_sets)
dp = DatasetParameter.objects.get(parameterset=param_sets[0],
name=param)
self.assertTrue(dp)
self.assertNotEquals(dp.string_value, "") # ie, it has a filename
def test_hrmc_filter(self):
"""
Make an experiment, lood up grexp file and check
dataset schema missing, then loadup grfinal and check dataset schema
created
"""
user = _create_test_user()
license = _create_license()
exp = _create_test_experiment(user, license)
ds = Dataset(description='happy snaps of plumage')
ds.save()
_create_test_dataset(ds, exp.id, {
"output.dat": 'hello',
"grexp.dat": '2 5\n6 15\n'
})
ds.experiments.add(exp)
ds.save()
sch = Schema(namespace=self.HRMCSCHEMA,
name="hrmc_views",
type=Schema.DATASET)
sch.save()
param = ParameterName(schema=sch,
name="plot",
full_name="scatterplot",
units="image",
data_type=ParameterName.FILENAME)
param.save()
param_sets = get_param_sets(ds)
self.assertEquals(list(param_sets), [])
_create_test_dataset(ds, exp.id, {'grfinal21.dat': "1 3\n5 14\n"})
df2 = Dataset_File(dataset=ds, url='path/grfinal21.dat')
df2.save()
h = hrmc.HRMCOutput('HRMC', self.HRMCSCHEMA)
h(sender=Dataset_File, instance=df2)
param_sets = get_param_sets(ds)
self.assertEquals([x.schema.namespace for x in param_sets],
[self.HRMCSCHEMA])
def getOrCreateParameterNames(self, schema, paramnames):
""" Takes a list of paramnames (defined in the __init__ method) to get or create new parameter names objects
"""
pn_objects = []
for paramname in paramnames:
param_objects = ParameterName.objects.filter(
schema=schema, name=paramname['name'])
if len(param_objects) == 0:
pn = ParameterName(schema=schema,
name=paramname['name'],
full_name=paramname['full_name'],
data_type=paramname['data_type'])
pn.save()
else:
pn = param_objects[0]
pn_objects.append(pn)
return pn_objects
def test_parameter(self):
exp = Experiment(
title='test exp1',
institution_name='Australian Synchrotron',
approved=True,
created_by=self.user,
public_access=Experiment.PUBLIC_ACCESS_NONE,
)
exp.save()
dataset = Dataset(description="dataset description")
dataset.save()
dataset.experiments.add(exp)
dataset.save()
df_file = DataFile(dataset=dataset,
filename='file.txt',
size=42,
md5sum='bogus')
df_file.save()
df_schema = Schema(namespace='http://www.cern.ch/felzmann/schema1.xml',
type=Schema.DATAFILE)
df_schema.save()
ds_schema = Schema(namespace='http://www.cern.ch/felzmann/schema2.xml',
type=Schema.DATASET)
ds_schema.save()
exp_schema = Schema(
namespace='http://www.cern.ch/felzmann/schema3.xml',
type=Schema.EXPERIMENT)
exp_schema.save()
df_parname = ParameterName(schema=df_schema,
name='name',
full_name='full_name',
units='image/jpg',
data_type=ParameterName.FILENAME)
df_parname.save()
ds_parname = ParameterName(schema=ds_schema,
name='name',
full_name='full_name',
units='image/jpg',
data_type=ParameterName.FILENAME)
ds_parname.save()
exp_parname = ParameterName(schema=exp_schema,
name='name',
full_name='full_name',
units='image/jpg',
data_type=ParameterName.FILENAME)
exp_parname.save()
df_parset = DatafileParameterSet(schema=df_schema, datafile=df_file)
df_parset.save()
ds_parset = DatasetParameterSet(schema=ds_schema, dataset=dataset)
ds_parset.save()
exp_parset = ExperimentParameterSet(schema=exp_schema, experiment=exp)
exp_parset.save()
with self.settings(METADATA_STORE_PATH=os.path.dirname(__file__)):
filename = 'test.jpg'
df_parameter = DatafileParameter(name=df_parname,
parameterset=df_parset,
string_value=filename)
df_parameter.save()
ds_parameter = DatasetParameter(name=ds_parname,
parameterset=ds_parset,
string_value=filename)
ds_parameter.save()
exp_parameter = ExperimentParameter(name=exp_parname,
parameterset=exp_parset,
string_value=filename)
exp_parameter.save()
self.assertEqual(
"<a href='/display/DatafileImage/load/%i/' target='_blank'><img style='width: 300px;' src='/display/DatafileImage/load/%i/' /></a>"
% # noqa
(df_parameter.id, df_parameter.id),
df_parameter.get())
self.assertEqual(
"<a href='/display/DatasetImage/load/%i/' target='_blank'><img style='width: 300px;' src='/display/DatasetImage/load/%i/' /></a>"
% # noqa
(ds_parameter.id, ds_parameter.id),
ds_parameter.get())
self.assertEqual(
"<a href='/display/ExperimentImage/load/%i/' target='_blank'><img style='width: 300px;' src='/display/ExperimentImage/load/%i/' /></a>"
% # noqa
(exp_parameter.id, exp_parameter.id),
exp_parameter.get())
def setUp(self):
from django.contrib.auth.models import User
from tempfile import mkdtemp
user = '******'
pwd = 'secret'
email = ''
self.user = User.objects.create_user(user, email, pwd)
self.test_dir = mkdtemp()
self.exp = Experiment(title='test exp1',
institution_name='monash',
created_by=self.user)
self.exp.save()
self.dataset = Dataset(description="dataset description...")
self.dataset.save()
self.dataset.experiments.add(self.exp)
self.dataset.save()
self.datafile = DataFile(dataset=self.dataset,
filename="testfile.txt",
size="42",
md5sum='bogus')
self.datafile.save()
self.dfo = DataFileObject(
datafile=self.datafile,
storage_box=self.datafile.get_default_storage_box(),
uri="1/testfile.txt")
self.dfo.save()
self.schema = Schema(namespace="http://localhost/psmtest/df/",
name="Parameter Set Manager",
type=3)
self.schema.save()
self.parametername1 = ParameterName(schema=self.schema,
name="parameter1",
full_name="Parameter 1")
self.parametername1.save()
self.parametername2 = ParameterName(schema=self.schema,
name="parameter2",
full_name="Parameter 2",
data_type=ParameterName.NUMERIC)
self.parametername2.save()
self.parametername3 = ParameterName(schema=self.schema,
name="parameter3",
full_name="Parameter 3",
data_type=ParameterName.DATETIME)
self.parametername3.save()
self.datafileparameterset = DatafileParameterSet(
schema=self.schema, datafile=self.datafile)
self.datafileparameterset.save()
self.datafileparameter1 = DatafileParameter(
parameterset=self.datafileparameterset,
name=self.parametername1,
string_value="test1")
self.datafileparameter1.save()
self.datafileparameter2 = DatafileParameter(
parameterset=self.datafileparameterset,
name=self.parametername2,
numerical_value=2)
self.datafileparameter2.save()
# Create a ParameterName and Parameter of type LINK to an experiment
self.parametername_exp_link = ParameterName(
schema=self.schema,
name="exp_link",
full_name="This parameter is a experiment LINK",
data_type=ParameterName.LINK)
self.parametername_exp_link.save()
self.exp_link_param = DatafileParameter(
parameterset=self.datafileparameterset,
name=self.parametername_exp_link)
exp_url = self.exp.get_absolute_url() # /experiment/view/1/
self.exp_link_param.set_value(exp_url)
self.exp_link_param.save()
# Create a ParameterName and Parameter of type LINK to a dataset
self.parametername_dataset_link = ParameterName(
schema=self.schema,
name="dataset_link",
full_name="This parameter is a dataset LINK",
data_type=ParameterName.LINK)
self.parametername_dataset_link.save()
self.dataset_link_param = DatafileParameter(
parameterset=self.datafileparameterset,
name=self.parametername_dataset_link)
dataset_url = self.dataset.get_absolute_url() # /dataset/1/
self.dataset_link_param.set_value(dataset_url)
self.dataset_link_param.save()
# Create a ParameterName type LINK to an unresolvable (non-URL)
# free-text value
self.parametername_unresolvable_link = ParameterName(
schema=self.schema,
name="freetext_link",
full_name="This parameter is a non-URL LINK",
data_type=ParameterName.LINK)
self.parametername_unresolvable_link.save()
