def save_metadata(instance, schema, metadata):
parameters = get_parameters(schema, metadata)
logger.debug("parameters=%s" % parameters)
if not parameters:
return None
try:
ps = DatasetParameterSet.objects.get(schema=schema, dataset=instance)
# return ps
except DatasetParameterSet.DoesNotExist:
ps = DatasetParameterSet(schema=schema, dataset=instance)
ps.save()
logger.debug("ps=%s\n" % ps)
logger.debug("metadata2=%s\n" % metadata)
for p in parameters:
logger.debug("p=%s\n" % p)
if p.name in metadata:
logger.debug("found p =%s %s\n" % (p.name, p.units))
try:
dfp = DatasetParameter.objects.get(parameterset=ps, name=p)
except DatasetParameter.DoesNotExist:
dfp = DatasetParameter(parameterset=ps, name=p)
# TODO: handle bad type
if p.isNumeric():
dfp.numerical_value = metadata[p.name][0]
logger.debug("numeric")
else:
dfp.string_value = metadata[p.name][0]
logger.debug("dfp=%(dfp)s" % locals())
dfp.save()
def __init__(self, dataset=None, dataset_id=None,
description="", experiment_id=None):
"""
instantiate new task or existing task
:param dataset: optional parameter to instanciate task from
metadata, will be tested for completeness and copied into
new task if complete
:type dataset: Dataset
"""
if dataset:
self.dataset = dataset
elif dataset_id:
self.dataset = Dataset.objects.get(pk=dataset_id)
else:
if description == "":
raise TypeError("No description given")
if not experiment_id:
raise TypeError("No experiment id given")
self.dataset = Dataset()
self.dataset.experiment_id = experiment_id
self.dataset.description = description
self.dataset.save()
try:
thisparameterset = DatasetParameterSet.objects.get(
schema=self.get_schema(),
dataset=self.dataset)
except ObjectDoesNotExist:
thisparameterset = DatasetParameterSet(
schema=self.get_schema(),
dataset=self.dataset)
thisparameterset.save()
super(Task, self).__init__(parameterset=thisparameterset)
def _save_metadata(instance,schema,metadataset):
""" Creates schema from the metadataset and associates it
with the instance. If metadata value is empty, then
existing value is unchanged.
"""
parameters = _get_parameters(schema, metadataset)
logger.debug("parameters=%s" % parameters)
if not parameters:
return None
try:
ps = DatasetParameterSet.objects.get(schema=schema,
dataset=instance)
except DatasetParameterSet.DoesNotExist:
ps = DatasetParameterSet(schema=schema,dataset=instance)
ps.save()
logger.debug("ps=%s\n" % ps)
logger.debug("metadata2=%s\n" % metadataset)
for p in parameters:
logger.debug("p=%s\n" % p)
if p.name in metadataset:
logger.debug("found p =%s %s\n" % (p.name,p.units))
if p.isNumeric():
val = metadataset[p.name][0]
if val:
dfp = DatasetParameter.objects.filter(parameterset=ps,
name=p)
if not dfp:
dfp = DatasetParameter(parameterset=ps,
name=p)
dfp.numerical_value = val
logger.debug("new numeric")
dfp.save()
else:
for dp in dfp:
dp.numerical_value = val
dp.save()
logger.debug("numeric")
else:
val = metadataset[p.name][0]
logger.debug("val=%s" % val)
if val:
dfp = DatasetParameter.objects.filter(parameterset=ps,
name=p)
if not dfp:
dfp = DatasetParameter(parameterset=ps,
name=p)
dfp.string_value = metadataset[p.name][0]
dfp.save()
logger.debug("new string")
else:
for dp in dfp:
dp.string_value = metadataset[p.name][0]
dp.save()
logger.debug("string")
logger.debug("done")
def _save_metadata(instance, schema, metadataset):
""" Creates schema from the metadataset and associates it
with the instance. If metadata value is empty, then
existing value is unchanged.
"""
parameters = _get_parameters(schema, metadataset)
logger.debug("parameters=%s" % parameters)
if not parameters:
return None
try:
ps = DatasetParameterSet.objects.get(schema=schema, dataset=instance)
except DatasetParameterSet.DoesNotExist:
ps = DatasetParameterSet(schema=schema, dataset=instance)
ps.save()
logger.debug("ps=%s\n" % ps)
logger.debug("metadata2=%s\n" % metadataset)
for p in parameters:
logger.debug("p=%s\n" % p)
if p.name in metadataset:
logger.debug("found p =%s %s\n" % (p.name, p.units))
if p.isNumeric():
val = metadataset[p.name][0]
if val:
dfp = DatasetParameter.objects.filter(parameterset=ps,
name=p)
if not dfp:
dfp = DatasetParameter(parameterset=ps, name=p)
dfp.numerical_value = val
logger.debug("new numeric")
dfp.save()
else:
for dp in dfp:
dp.numerical_value = val
dp.save()
logger.debug("numeric")
else:
val = metadataset[p.name][0]
logger.debug("val=%s" % val)
if val:
dfp = DatasetParameter.objects.filter(parameterset=ps,
name=p)
if not dfp:
dfp = DatasetParameter(parameterset=ps, name=p)
dfp.string_value = metadataset[p.name][0]
dfp.save()
logger.debug("new string")
else:
for dp in dfp:
dp.string_value = metadataset[p.name][0]
dp.save()
logger.debug("string")
logger.debug("done")
def test_contextual_view(self):
"""
Given schema on dataset, check that image file created
"""
user = _create_test_user()
license = _create_license()
exp = _create_test_experiment(user, license)
ds = Dataset(description='happy snaps of plumage')
ds.save()
ds = _create_test_dataset(
ds, exp.id, {
"output.dat": 'test data\n',
"grexp.dat": '1 2\n2 3\n3 7\n',
"grfinal21.dat": '1 2\n 2 4\n4 9\n'
})
sch = Schema(namespace=self.HRMCSCHEMA,
name="hrmc_views",
type=Schema.DATASET)
sch.save()
param = ParameterName(schema=sch,
name="plot",
full_name="scatterplot",
units="image",
data_type=ParameterName.FILENAME)
param.save()
dps = DatasetParameterSet(schema=sch, dataset=ds)
dps.save()
ds.experiments.add(exp)
ds.save()
client = Client()
response = client.get('/dataset/%s' % ds.id)
self.assertEqual(response.status_code, 200)
param_sets = get_param_sets(ds)
self.assertTrue(param_sets)
dp = DatasetParameter.objects.get(parameterset=param_sets[0],
name=param)
self.assertTrue(dp)
self.assertNotEquals(dp.string_value, "") # ie, it has a filename
def test_contextual_view(self):
"""
Given schema on dataset, check that image file created
"""
user = _create_test_user()
license = _create_license()
exp = _create_test_experiment(user, license)
ds = Dataset(description='happy snaps of plumage')
ds.save()
ds = _create_test_dataset(ds, exp.id, {
"output.dat": 'test data\n',
"grexp.dat": '1 2\n2 3\n3 7\n',
"grfinal21.dat": '1 2\n 2 4\n4 9\n'})
sch = Schema(namespace=self.HRMCSCHEMA,
name="hrmc_views", type=Schema.DATASET)
sch.save()
param = ParameterName(schema=sch, name="plot",
full_name="scatterplot", units="image",
data_type=ParameterName.FILENAME
)
param.save()
dps = DatasetParameterSet(schema=sch, dataset=ds)
dps.save()
ds.experiments.add(exp)
ds.save()
client = Client()
response = client.get('/dataset/%s' % ds.id)
self.assertEqual(response.status_code, 200)
param_sets = get_param_sets(ds)
self.assertTrue(param_sets)
dp = DatasetParameter.objects.get(parameterset=param_sets[0],
name=param)
self.assertTrue(dp)
self.assertNotEquals(dp.string_value, "") # ie, it has a filename
def __init__(self, parameterset=None, parentObject=None,
schema=None):
"""
instantiate new task or existing ParameterSet
:param dataset: optional parameter to instanciate task from
metadata, will be tested for completeness and copied into
new task if complete
:type dataset: Dataset
:param schema: Schema namespace
:type schema: string
"""
if parameterset:
self.parameterset = parameterset
self.schema = self.parameterset.schema
self.namespace = self.schema.namespace
if isinstance(self.parameterset, DatafileParameterSet):
self.parameters = DatafileParameter.objects.filter(\
parameterset=self.parameterset).order_by('name__full_name')
self.blank_param = DatafileParameter
elif isinstance(self.parameterset, DatasetParameterSet):
self.parameters = DatasetParameter.objects.filter(\
parameterset=self.parameterset).order_by('name__full_name')
self.blank_param = DatasetParameter
elif isinstance(self.parameterset, ExperimentParameterSet):
self.parameters = ExperimentParameter.objects.filter(\
parameterset=self.parameterset).order_by('name__full_name')
self.blank_param = ExperimentParameter
else:
raise TypeError("Invalid parameterset object given.")
elif parentObject and schema:
self.namespace = schema
if isinstance(parentObject, Dataset_File):
self.parameterset = DatafileParameterSet(
schema=self.get_schema(), dataset_file=parentObject)
self.parameterset.save()
self.parameters = DatafileParameter.objects.filter(
parameterset=self.parameterset)
self.blank_param = DatafileParameter
elif isinstance(parentObject, Dataset):
self.parameterset = DatasetParameterSet(
schema=self.get_schema(), dataset=parentObject)
self.parameterset.save()
self.parameters = DatasetParameter.objects.filter(
parameterset=self.parameterset)
self.blank_param = DatasetParameter
elif isinstance(parentObject, Experiment):
self.parameterset = ExperimentParameterSet(
schema=self.get_schema(), experiment=parentObject)
self.parameterset.save()
self.parameters = ExperimentParameter.objects.filter(
parameterset=self.parameterset)
self.blank_param = ExperimentParameter
else:
raise TypeError("Invalid parent object." +
"Must be an experiment/dataset/datafile not " + str(type(parentObject)))
else:
raise TypeError("Missing arguments")
def __call__(self, sender, **kwargs):
"""post save callback entry point.
:param sender: The model class.
:param instance: The actual instance being saved.
:param created: A boolean; True if a new record was created.
:type created: bool
"""
logger.debug("hrmc __call__")
datafile_instance = kwargs.get("instance")
dataset_instance = datafile_instance.dataset
try:
sch = Schema.objects.get(namespace=self.schema)
except Schema.DoesNotExist:
logger.debug("no hrmc schema")
return None
except MultipleObjectsReturned:
logger.error("multiple hrmc schemas returned")
return None
logger.debug("sch=%s" % sch)
try:
ps = DatasetParameterSet.objects.get(schema=sch, dataset=dataset_instance)
except DatasetParameterSet.DoesNotExist:
pass
except MultipleObjectsReturned:
logger.warn("Multiple dataset paramter sets for %s returned" % self.name)
# NB: If multiple filters run in dataset could get race condition
# which causes multiple DatasetParameterSets to be created.
# However, we know that all data will be the same for this schema
# so can safely delete any extras we find.
pslist = [x.id for x in DatasetParameterSet.objects.filter(schema=sch, dataset=dataset_instance)]
logger.debug("pslist=%s" % pslist)
DatasetParameterSet.objects.filter(id__in=pslist[1:]).delete()
return None
else:
logger.debug("parameterset already exists")
return None
logger.debug("dataset_instance=%s" % dataset_instance)
filepath = datafile_instance.get_absolute_filepath()
logger.debug("filepath=%s" % filepath)
grexp_file = None
grfinal_file = None
for df in Dataset_File.objects.filter(dataset=dataset_instance):
logger.debug("df=%s" % df.filename)
if "grexp.dat" in df.filename:
grexp_file = df
if df.filename.startswith("grfinal"):
grfinal_file = df
if grexp_file and grfinal_file:
logger.debug("found all files")
try:
ps = DatasetParameterSet.objects.get(schema=sch, dataset=dataset_instance)
except DatasetParameterSet.DoesNotExist:
ps = DatasetParameterSet(schema=sch, dataset=dataset_instance)
ps.save()
logger.debug("created new dataset")
return None
except MultipleObjectsReturned:
logger.error("Multiple dataset paramter sets for %s returned" % self.name)
return None
else:
logger.debug("parameterset already exists")
return None
else:
logger.debug("one or more files missing")
def __call__(self, sender, **kwargs):
"""post save callback entry point.
:param sender: The model class.
:param instance: The actual instance being saved.
:param created: A boolean; True if a new record was created.
:type created: bool
"""
logger.debug("hrmc __call__")
datafile_instance = kwargs.get('instance')
dataset_instance = datafile_instance.dataset
try:
sch = Schema.objects.get(namespace=self.schema)
except Schema.DoesNotExist:
logger.debug("no hrmc schema")
return None
except MultipleObjectsReturned:
logger.error("multiple hrmc schemas returned")
return None
logger.debug("sch=%s" % sch)
try:
ps = DatasetParameterSet.objects.get(schema=sch,
dataset=dataset_instance)
except DatasetParameterSet.DoesNotExist:
pass
except MultipleObjectsReturned:
logger.warn("Multiple dataset paramter sets for %s returned" %
self.name)
# NB: If multiple filters run in dataset could get race condition
# which causes multiple DatasetParameterSets to be created.
# However, we know that all data will be the same for this schema
# so can safely delete any extras we find.
pslist = [
x.id for x in DatasetParameterSet.objects.filter(
schema=sch, dataset=dataset_instance)
]
logger.debug("pslist=%s" % pslist)
DatasetParameterSet.objects.filter(id__in=pslist[1:]).delete()
return None
else:
logger.debug("parameterset already exists")
return None
logger.debug("dataset_instance=%s" % dataset_instance)
filepath = datafile_instance.get_absolute_filepath()
logger.debug("filepath=%s" % filepath)
grexp_file = None
grfinal_file = None
for df in Dataset_File.objects.filter(dataset=dataset_instance):
logger.debug("df=%s" % df.filename)
if "grexp.dat" in df.filename:
grexp_file = df
if df.filename.startswith("grfinal"):
grfinal_file = df
if grexp_file and grfinal_file:
logger.debug("found all files")
try:
ps = DatasetParameterSet.objects.get(schema=sch,
dataset=dataset_instance)
except DatasetParameterSet.DoesNotExist:
ps = DatasetParameterSet(schema=sch, dataset=dataset_instance)
ps.save()
logger.debug("created new dataset")
return None
except MultipleObjectsReturned:
logger.error("Multiple dataset paramter sets for %s returned" %
self.name)
return None
else:
logger.debug("parameterset already exists")
return None
else:
logger.debug("one or more files missing")
def test_parameter(self):
exp = Experiment(
title='test exp1',
institution_name='Australian Synchrotron',
approved=True,
created_by=self.user,
public_access=Experiment.PUBLIC_ACCESS_NONE,
)
exp.save()
dataset = Dataset(description="dataset description")
dataset.save()
dataset.experiments.add(exp)
dataset.save()
df_file = DataFile(dataset=dataset,
filename='file.txt',
size=42,
md5sum='bogus')
df_file.save()
df_schema = Schema(namespace='http://www.cern.ch/felzmann/schema1.xml',
type=Schema.DATAFILE)
df_schema.save()
ds_schema = Schema(namespace='http://www.cern.ch/felzmann/schema2.xml',
type=Schema.DATASET)
ds_schema.save()
exp_schema = Schema(
namespace='http://www.cern.ch/felzmann/schema3.xml',
type=Schema.EXPERIMENT)
exp_schema.save()
df_parname = ParameterName(schema=df_schema,
name='name',
full_name='full_name',
units='image/jpg',
data_type=ParameterName.FILENAME)
df_parname.save()
ds_parname = ParameterName(schema=ds_schema,
name='name',
full_name='full_name',
units='image/jpg',
data_type=ParameterName.FILENAME)
ds_parname.save()
exp_parname = ParameterName(schema=exp_schema,
name='name',
full_name='full_name',
units='image/jpg',
data_type=ParameterName.FILENAME)
exp_parname.save()
df_parset = DatafileParameterSet(schema=df_schema, datafile=df_file)
df_parset.save()
ds_parset = DatasetParameterSet(schema=ds_schema, dataset=dataset)
ds_parset.save()
exp_parset = ExperimentParameterSet(schema=exp_schema, experiment=exp)
exp_parset.save()
with self.settings(METADATA_STORE_PATH=os.path.dirname(__file__)):
filename = 'test.jpg'
df_parameter = DatafileParameter(name=df_parname,
parameterset=df_parset,
string_value=filename)
df_parameter.save()
ds_parameter = DatasetParameter(name=ds_parname,
parameterset=ds_parset,
string_value=filename)
ds_parameter.save()
exp_parameter = ExperimentParameter(name=exp_parname,
parameterset=exp_parset,
string_value=filename)
exp_parameter.save()
self.assertEqual(
"<a href='/display/DatafileImage/load/%i/' target='_blank'><img style='width: 300px;' src='/display/DatafileImage/load/%i/' /></a>"
% # noqa
(df_parameter.id, df_parameter.id),
df_parameter.get())
self.assertEqual(
"<a href='/display/DatasetImage/load/%i/' target='_blank'><img style='width: 300px;' src='/display/DatasetImage/load/%i/' /></a>"
% # noqa
(ds_parameter.id, ds_parameter.id),
ds_parameter.get())
self.assertEqual(
"<a href='/display/ExperimentImage/load/%i/' target='_blank'><img style='width: 300px;' src='/display/ExperimentImage/load/%i/' /></a>"
% # noqa
(exp_parameter.id, exp_parameter.id),
exp_parameter.get())