def save_metadata(instance, schema, metadata):
parameters = get_parameters(schema, metadata)
logger.debug("parameters=%s" % parameters)
if not parameters:
return None
try:
ps = DatasetParameterSet.objects.get(schema=schema, dataset=instance)
# return ps
except DatasetParameterSet.DoesNotExist:
ps = DatasetParameterSet(schema=schema, dataset=instance)
ps.save()
logger.debug("ps=%s\n" % ps)
logger.debug("metadata2=%s\n" % metadata)
for p in parameters:
logger.debug("p=%s\n" % p)
if p.name in metadata:
logger.debug("found p =%s %s\n" % (p.name, p.units))
try:
dfp = DatasetParameter.objects.get(parameterset=ps, name=p)
except DatasetParameter.DoesNotExist:
dfp = DatasetParameter(parameterset=ps, name=p)
# TODO: handle bad type
if p.isNumeric():
dfp.numerical_value = metadata[p.name][0]
logger.debug("numeric")
else:
dfp.string_value = metadata[p.name][0]
logger.debug("dfp=%(dfp)s" % locals())
dfp.save()
def _save_metadata(instance, schema, metadataset):
""" Creates schema from the metadataset and associates it
with the instance. If metadata value is empty, then
existing value is unchanged.
"""
parameters = _get_parameters(schema, metadataset)
logger.debug("parameters=%s" % parameters)
if not parameters:
return None
try:
ps = DatasetParameterSet.objects.get(schema=schema, dataset=instance)
except DatasetParameterSet.DoesNotExist:
ps = DatasetParameterSet(schema=schema, dataset=instance)
ps.save()
logger.debug("ps=%s\n" % ps)
logger.debug("metadata2=%s\n" % metadataset)
for p in parameters:
logger.debug("p=%s\n" % p)
if p.name in metadataset:
logger.debug("found p =%s %s\n" % (p.name, p.units))
if p.isNumeric():
val = metadataset[p.name][0]
if val:
dfp = DatasetParameter.objects.filter(parameterset=ps,
name=p)
if not dfp:
dfp = DatasetParameter(parameterset=ps, name=p)
dfp.numerical_value = val
logger.debug("new numeric")
dfp.save()
else:
for dp in dfp:
dp.numerical_value = val
dp.save()
logger.debug("numeric")
else:
val = metadataset[p.name][0]
logger.debug("val=%s" % val)
if val:
dfp = DatasetParameter.objects.filter(parameterset=ps,
name=p)
if not dfp:
dfp = DatasetParameter(parameterset=ps, name=p)
dfp.string_value = metadataset[p.name][0]
dfp.save()
logger.debug("new string")
else:
for dp in dfp:
dp.string_value = metadataset[p.name][0]
dp.save()
logger.debug("string")
logger.debug("done")
def _save_metadata(instance,schema,metadataset):
""" Creates schema from the metadataset and associates it
with the instance. If metadata value is empty, then
existing value is unchanged.
"""
parameters = _get_parameters(schema, metadataset)
logger.debug("parameters=%s" % parameters)
if not parameters:
return None
try:
ps = DatasetParameterSet.objects.get(schema=schema,
dataset=instance)
except DatasetParameterSet.DoesNotExist:
ps = DatasetParameterSet(schema=schema,dataset=instance)
ps.save()
logger.debug("ps=%s\n" % ps)
logger.debug("metadata2=%s\n" % metadataset)
for p in parameters:
logger.debug("p=%s\n" % p)
if p.name in metadataset:
logger.debug("found p =%s %s\n" % (p.name,p.units))
if p.isNumeric():
val = metadataset[p.name][0]
if val:
dfp = DatasetParameter.objects.filter(parameterset=ps,
name=p)
if not dfp:
dfp = DatasetParameter(parameterset=ps,
name=p)
dfp.numerical_value = val
logger.debug("new numeric")
dfp.save()
else:
for dp in dfp:
dp.numerical_value = val
dp.save()
logger.debug("numeric")
else:
val = metadataset[p.name][0]
logger.debug("val=%s" % val)
if val:
dfp = DatasetParameter.objects.filter(parameterset=ps,
name=p)
if not dfp:
dfp = DatasetParameter(parameterset=ps,
name=p)
dfp.string_value = metadataset[p.name][0]
dfp.save()
logger.debug("new string")
else:
for dp in dfp:
dp.string_value = metadataset[p.name][0]
dp.save()
logger.debug("string")
logger.debug("done")
def get_image_to_show(dataset):
try:
sch = Schema.objects.get(namespace__exact=HRMC_DATASET_SCHEMA)
except Schema.DoesNotExist:
logger.debug("no hrmc schema")
return None
except MultipleObjectsReturned:
logger.error("multiple hrmc schemas returned")
return None
#FIXME: possible that more than once dataset can appear, so pick only one.
try:
ps = DatasetParameterSet.objects.get(schema=sch, dataset=dataset)
except DatasetParameterSet.DoesNotExist:
logger.debug("datset parameterset not found")
return None
except MultipleObjectsReturned:
logger.error("multiple dataset paramter sets returned")
# NB: If admin tool added additional param set,
# we know that all data will be the same for this schema
# so can safely delete any extras we find.
pslist = [x.id for x in DatasetParameterSet.objects.filter(schema=sch,
dataset=dataset)]
logger.debug("pslist=%s" % pslist)
DatasetParameterSet.objects.filter(id__in=pslist[1:]).delete()
ps = DatasetParameterSet.objects.get(id=pslist[0])
logger.debug("found ps=%s" % ps)
for param in DatasetParameter.objects.filter(parameterset=ps):
logger.debug("param=%s" % param)
logger.debug("param.name=%s" % param.name)
if "plot" in param.name.name:
logger.debug("found existing image")
return param
logger.debug("building plots")
display_image = None
grfinal_file = None
grexp_file = None
for df in Dataset_File.objects.filter(dataset=dataset):
logger.debug("testing %s" % df.filename)
if "grexp.dat" in df.filename:
grexp_file = df
if df.filename.startswith("grfinal"):
grfinal_file = df
if grexp_file and grfinal_file and is_matplotlib_imported:
logger.debug("found both")
fp = grexp_file.get_absolute_filepath()
grexp_buff = []
with open(fp) as f:
for d in f.read():
grexp_buff.append(d)
fp = grfinal_file.get_absolute_filepath()
grfinal_buff = []
with open(fp) as f:
for d in f.read():
grfinal_buff.append(d)
mat = re.compile("grfinal(\d+)\.dat").match(grfinal_file.filename)
if mat:
grlabel = "Calculation %s" % mat.group(1)
else:
grlabel = grfinal_file.filename
xs = []
ys = []
for l in ''.join(grfinal_buff).split("\n"):
#logger.debug("l=%s" % l)
if l:
x, y = l.split()
xs.append(float(x))
ys.append(float(y))
matplotlib.pyplot.plot(xs, ys, color="blue", markeredgecolor = 'blue', marker="D", label=str(grlabel))
xs = []
ys = []
for l in ''.join(grexp_buff).split("\n"):
#logger.debug("l=%s" % l)
if l:
x, y = l.split()
xs.append(float(x))
ys.append(float(y))
matplotlib.pyplot.plot(xs, ys, color="red", markeredgecolor = 'red', marker="o", label="Experiment")
import tempfile
pfile = tempfile.mktemp()
logger.debug("pfile=%s" % pfile)
pyplot.xlabel("r (Angstroms)")
pyplot.ylabel("g(r)")
pyplot.grid(True)
#legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
legend()
pyplot.xlim(xmin=0)
fig = matplotlib.pyplot.gcf()
fig.set_size_inches(15.5, 13.5)
matplotlib.pyplot.savefig("%s.png" % pfile, dpi=100)
with open("%s.png" % pfile) as pf:
read = pf.read()
encoded = base64.b64encode(read)
matplotlib.pyplot.close()
try:
pn = ParameterName.objects.get(schema=sch, name="plot")
except DatasetParameterSet.DoesNotExist:
logger.error("schema is missing plot parameter")
return None
except MultipleObjectsReturned:
logger.error("schema is multiple plot parameters")
return None
logger.debug("ready to save")
dfp = DatasetParameter(parameterset=ps,
name=pn)
dfp.string_value = encoded
dfp.save()
display_image = dfp
else:
logger.debug("one or more files unavailable")
return None
logger.debug("made display_image %s" % display_image)
return display_image
def test_parameter(self):
exp = Experiment(
title='test exp1',
institution_name='Australian Synchrotron',
approved=True,
created_by=self.user,
public_access=Experiment.PUBLIC_ACCESS_NONE,
)
exp.save()
dataset = Dataset(description="dataset description")
dataset.save()
dataset.experiments.add(exp)
dataset.save()
df_file = DataFile(dataset=dataset,
filename='file.txt',
size=42,
md5sum='bogus')
df_file.save()
df_schema = Schema(namespace='http://www.cern.ch/felzmann/schema1.xml',
type=Schema.DATAFILE)
df_schema.save()
ds_schema = Schema(namespace='http://www.cern.ch/felzmann/schema2.xml',
type=Schema.DATASET)
ds_schema.save()
exp_schema = Schema(
namespace='http://www.cern.ch/felzmann/schema3.xml',
type=Schema.EXPERIMENT)
exp_schema.save()
df_parname = ParameterName(schema=df_schema,
name='name',
full_name='full_name',
units='image/jpg',
data_type=ParameterName.FILENAME)
df_parname.save()
ds_parname = ParameterName(schema=ds_schema,
name='name',
full_name='full_name',
units='image/jpg',
data_type=ParameterName.FILENAME)
ds_parname.save()
exp_parname = ParameterName(schema=exp_schema,
name='name',
full_name='full_name',
units='image/jpg',
data_type=ParameterName.FILENAME)
exp_parname.save()
df_parset = DatafileParameterSet(schema=df_schema, datafile=df_file)
df_parset.save()
ds_parset = DatasetParameterSet(schema=ds_schema, dataset=dataset)
ds_parset.save()
exp_parset = ExperimentParameterSet(schema=exp_schema, experiment=exp)
exp_parset.save()
with self.settings(METADATA_STORE_PATH=os.path.dirname(__file__)):
filename = 'test.jpg'
df_parameter = DatafileParameter(name=df_parname,
parameterset=df_parset,
string_value=filename)
df_parameter.save()
ds_parameter = DatasetParameter(name=ds_parname,
parameterset=ds_parset,
string_value=filename)
ds_parameter.save()
exp_parameter = ExperimentParameter(name=exp_parname,
parameterset=exp_parset,
string_value=filename)
exp_parameter.save()
self.assertEqual(
"<a href='/display/DatafileImage/load/%i/' target='_blank'><img style='width: 300px;' src='/display/DatafileImage/load/%i/' /></a>"
% # noqa
(df_parameter.id, df_parameter.id),
df_parameter.get())
self.assertEqual(
"<a href='/display/DatasetImage/load/%i/' target='_blank'><img style='width: 300px;' src='/display/DatasetImage/load/%i/' /></a>"
% # noqa
(ds_parameter.id, ds_parameter.id),
ds_parameter.get())
self.assertEqual(
"<a href='/display/ExperimentImage/load/%i/' target='_blank'><img style='width: 300px;' src='/display/ExperimentImage/load/%i/' /></a>"
% # noqa
(exp_parameter.id, exp_parameter.id),
exp_parameter.get())
def get_image_to_show(dataset):
try:
sch = Schema.objects.get(namespace__exact=HRMC_DATASET_SCHEMA)
except Schema.DoesNotExist:
logger.debug("no hrmc schema")
return None
except MultipleObjectsReturned:
logger.error("multiple hrmc schemas returned")
return None
#FIXME: possible that more than once dataset can appear, so pick only one.
try:
ps = DatasetParameterSet.objects.get(schema=sch, dataset=dataset)
except DatasetParameterSet.DoesNotExist:
logger.debug("datset parameterset not found")
return None
except MultipleObjectsReturned:
logger.error("multiple dataset paramter sets returned")
# NB: If admin tool added additional param set,
# we know that all data will be the same for this schema
# so can safely delete any extras we find.
pslist = [
x.id for x in DatasetParameterSet.objects.filter(schema=sch,
dataset=dataset)
]
logger.debug("pslist=%s" % pslist)
DatasetParameterSet.objects.filter(id__in=pslist[1:]).delete()
ps = DatasetParameterSet.objects.get(id=pslist[0])
logger.debug("found ps=%s" % ps)
for param in DatasetParameter.objects.filter(parameterset=ps):
logger.debug("param=%s" % param)
logger.debug("param.name=%s" % param.name)
if "plot" in param.name.name:
logger.debug("found existing image")
return param
logger.debug("building plots")
display_image = None
grfinal_file = None
grexp_file = None
for df in Dataset_File.objects.filter(dataset=dataset):
logger.debug("testing %s" % df.filename)
if "grexp.dat" in df.filename:
grexp_file = df
if df.filename.startswith("grfinal"):
grfinal_file = df
if grexp_file and grfinal_file and is_matplotlib_imported:
logger.debug("found both")
fp = grexp_file.get_absolute_filepath()
grexp_buff = []
with open(fp) as f:
for d in f.read():
grexp_buff.append(d)
fp = grfinal_file.get_absolute_filepath()
grfinal_buff = []
with open(fp) as f:
for d in f.read():
grfinal_buff.append(d)
mat = re.compile("grfinal(\d+)\.dat").match(grfinal_file.filename)
if mat:
grlabel = "Calculation %s" % mat.group(1)
else:
grlabel = grfinal_file.filename
xs = []
ys = []
for l in ''.join(grfinal_buff).split("\n"):
#logger.debug("l=%s" % l)
if l:
x, y = l.split()
xs.append(float(x))
ys.append(float(y))
matplotlib.pyplot.plot(xs,
ys,
color="blue",
markeredgecolor='blue',
marker="D",
label=str(grlabel))
xs = []
ys = []
for l in ''.join(grexp_buff).split("\n"):
#logger.debug("l=%s" % l)
if l:
x, y = l.split()
xs.append(float(x))
ys.append(float(y))
matplotlib.pyplot.plot(xs,
ys,
color="red",
markeredgecolor='red',
marker="o",
label="Experiment")
import tempfile
pfile = tempfile.mktemp()
logger.debug("pfile=%s" % pfile)
pyplot.xlabel("r (Angstroms)")
pyplot.ylabel("g(r)")
pyplot.grid(True)
#legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
legend()
pyplot.xlim(xmin=0)
fig = matplotlib.pyplot.gcf()
fig.set_size_inches(15.5, 13.5)
matplotlib.pyplot.savefig("%s.png" % pfile, dpi=100)
with open("%s.png" % pfile) as pf:
read = pf.read()
encoded = base64.b64encode(read)
matplotlib.pyplot.close()
try:
pn = ParameterName.objects.get(schema=sch, name="plot")
except DatasetParameterSet.DoesNotExist:
logger.error("schema is missing plot parameter")
return None
except MultipleObjectsReturned:
logger.error("schema is multiple plot parameters")
return None
logger.debug("ready to save")
dfp = DatasetParameter(parameterset=ps, name=pn)
dfp.string_value = encoded
dfp.save()
display_image = dfp
else:
logger.debug("one or more files unavailable")
return None
logger.debug("made display_image %s" % display_image)
return display_image