def replay_file(filename):
# type: (str) -> None
"""
Replay the template used to generate *filename*.
If the replayed reduction differs from the original, the differences are
displayed, and the new version saved into /tmp/filename.
Raises *RuntimeError* if the files differ.
"""
# Here's how the export file is constructed by the reductus client.
# This snippet is from reflweb/static/editor.js in the function
# webreduce.editor.export_data
"""
var header = {
template_data: {
template: params.template,
node: params.node,
terminal: params.terminal,
server_git_hash: result.server_git_hash,
server_mtime: new Date((result.server_mtime || 0.0) * 1000).toISOString()
}
};
webreduce.download('# ' + JSON.stringify(header).slice(1,-1) + '\n' + result.values.join('\n\n'), filename);
"""
# Load the template and the target output
with open(filename, 'r') as fid:
first_line = fid.readline()
template_data = json.loads(TEMPLATE.sub('{', first_line))
old_content = fid.read()
# Show the template
#print(json.dumps(template_data['template'], indent=2))
# Make sure instrument is available
template_module = template_data['template']['modules'][0]['module']
instrument_id = template_module.split('.')[1]
load_instrument(instrument_id)
# run the template
template = Template(**template_data['template'])
# extract module 'refl' from ncnr.refl.module into module id
target = template_data['node'], template_data['terminal']
template_config = {}
retval = process_template(template, template_config, target=target)
export = retval.get_export()
new_content = '\n\n'.join(v['export_string'] for v in export['values'])
has_diff = show_diff(old_content, new_content)
if has_diff:
# Save the new output into athe temp directory so we can easily update
# the regression tests
new_path = os.path.join('/tmp', os.path.basename(filename))
with open(new_path, 'wb') as fid:
fid.write(encode(first_line))
fid.write(encode(new_content))
raise RuntimeError(
"File replay for %r differs; new file stored in %r" %
(filename, new_path))
def find_calculated(template_def, config):
"""
Returns a vector of true/false for each node in the template indicating
whether that node value has been calculated yet.
"""
template = Template(**template_def)
retval = dataflow.calc.find_calculated(template, config)
return retval
def LoadMAGIKPSDMany(fileinfo=None, collapse=True, collapse_axis='y', auto_PolState=False, PolState='', flip=True, transpose=True):
"""
loads a data file into a MetaArray and returns that.
Checks to see if data being loaded is 2D; if not, quits
Need to rebin and regrid if the detector is moving...
**Inputs**
fileinfo (fileinfo[]): Files to open.
collapse {Collapse along one of the axes} (bool): sum over axis of detector
collapse_axis {number index of axis to collapse along} (opt:x|y): axis to sum over
auto_PolState {Automatic polarization identify} (bool): automatically determine the polarization state from entry name
PolState (str): polarization state if not automatically detected
flip (bool): flip the data up and down
transpose (bool): transpose the data
**Returns**
output (ospec2d[]): all the entries loaded.
2016-04-01 Brian Maranville
"""
outputs = []
kwconfig = {
"collapse": collapse,
"collapse_axis": collapse_axis,
"auto_PolState": auto_PolState,
"PolState": PolState,
"flip": flip,
"transpose": transpose
}
for fi in fileinfo:
template_def = {
"name": "loader_template",
"description": "Offspecular remote loader",
"modules": [
{"module": "ncnr.ospec.LoadMAGIKPSD", "version": "0.1", "config": {}}
],
"wires": [],
"instrument": "ncnr.magik",
"version": "0.0"
}
template = Template(**template_def)
config = {"0": {"fileinfo": {"path": fi['path'], "source": fi['source'], "mtime": fi['mtime']}}}
config["0"].update(kwconfig)
nodenum = 0
terminal_id = "output"
retval = process_template(template, config, target=(nodenum, terminal_id))
outputs.extend(retval.values)
return outputs
def refl_load(file_descriptors):
"""
file_descriptors will be a list of dicts like
[{"path": "ncnrdata/cgd/201511/21066/data/HMDSO_17nm_dry14.nxz.cgd", "mtime": 1447353278}, ...]
"""
modules = [{"module": "ncnr.refl.load", "version": "0.1", "config": {}}]
template = Template("test",
"test template",
modules, [],
"ncnr.magik",
version='0.0')
retval = process_template(template, {0: {
"files": file_descriptors
}},
target=(0, "output"))
return retval.todict()
def play_file(filename):
with open(filename) as fid:
template_json = json.loads(fid.read())
# Make sure instrument is available
template_module = template_json['modules'][0]['module']
instrument_id = template_module.split('.')[1]
load_instrument(instrument_id)
node = max(find_leaves(template_json))
node_module = lookup_module(template_json['modules'][node]['module'])
terminal = node_module.outputs[0]['id']
#print(json.dumps(template_json, indent=2))
template = Template(**template_json)
template_config = {}
target = node, terminal
retval = process_template(template, template_config, target=target)
export = retval.get_export()
if export['values']:
basename = export['values'][0].get('name', 'replay')
ext = export['values'][0].get('file_suffix', '.refl')
filename = basename + ext
else:
filename = 'replay.dat'
template_data = {
'template': template_json,
'node': node,
'terminal': terminal,
'server_git_hash': None,
'server_mtime': None,
#server_git_hash: result.server_git_hash,
#server_mtime: new Date((result.server_mtime || 0.0) * 1000).toISOString()
}
first_line = '# ' + json.dumps({'template_data': template_data})[1:-1]
new_content = '\n\n'.join(v['export_string'] for v in export['values'])
with open(filename, 'wb') as file:
print("writing", filename)
file.write(encode(first_line))
file.write(b'\n')
file.write(encode(new_content))
def LoadVSANSHe3Parallel(filelist=None, check_timestamps=True):
"""
loads a data file into a VSansData obj and returns that.
**Inputs**
filelist (fileinfo[]): Files to open.
check_timestamps (bool): verify that timestamps on file match request
**Returns**
output (raw[]): all the entries loaded.
| 2018-04-29 Brian Maranville
| 2019-11-20 Brian Maranville changed metadata list
"""
from dataflow.calc import process_template
from dataflow.core import Template
template_def = {
"name": "loader_template",
"description": "VSANS remote loader",
"modules": [
{"module": "ncnr.vsans.LoadVSANSHe3", "version": "0.1", "config": {}}
],
"wires": [],
"instrument": "ncnr.vsans",
"version": "0.0"
}
template = Template(**template_def)
output = []
for fi in filelist:
#config = {"0": {"filelist": [{"path": fi["path"], "source": fi["source"], "mtime": fi["mtime"]}]}}
config = {"0": {"filelist": [fi]}}
nodenum = 0
terminal_id = "output"
retval = process_template(template, config, target=(nodenum, terminal_id))
output.extend(retval.values)
return output
def calc_template(template_def, config):
""" json-rpc wrapper for process_template """
template = Template(**template_def)
#print "template_def:", template_def, "config:", config
try:
retvals = process_template(template, config, target=(None, None))
except Exception:
print("==== template ====")
pprint(template_def)
print("==== config ====")
pprint(config)
#traceback.print_exc()
raise
output = {}
for rkey, rv in retvals.items():
module_id, terminal_id = rkey
module_key = str(module_id)
output.setdefault(module_key, {})
output[module_key][terminal_id] = rv.todict()
return output
def LoadVSANS(filelist=None, check_timestamps=True):
"""
loads a data file into a VSansData obj and returns that. (uses cached values)
**Inputs**
filelist (fileinfo[]): Files to open.
check_timestamps (bool): verify that timestamps on file match request
**Returns**
output (raw[]): all the entries loaded.
2018-10-30 Brian Maranville
"""
from dataflow.calc import process_template
from dataflow.core import Template
template_def = {
"name": "loader_template",
"description": "VSANS remote loader",
"modules": [
{"module": "ncnr.vsans._LoadVSANS", "version": "0.1", "config": {}}
],
"wires": [],
"instrument": "ncnr.vsans",
"version": "0.0"
}
template = Template(**template_def)
output = []
for fi in filelist:
config = {"0": {"filelist": [fi], "check_timestamps": check_timestamps}}
nodenum = 0
terminal_id = "output"
retval = process_template(template, config, target=(nodenum, terminal_id))
output.extend(retval.values)
return output
dict(source=[12, 'ABS'], target=[13, 'ABS']),
dict(source=[13, 'OneD'], target=[8, 'input']),
#dict(source =[9,'COR'],target = [8,'input']),
#dict(source=[2, 'output'], target=[3, 'input']),
#dict(source=[3, 'output'], target=[4, 'input']),
#dict(source=[4, 'output'], target=[5, 'input']),
#dict(source=[5, 'output'], target=[1, 'input']),
#dict(source =[11,'DIV'],target = [8,'input']),
]
config = dict((n, d['config']) for (n, d) in enumerate(modules))
template = Template(
name='test sans',
description='example sans data',
modules=modules,
wires=wires,
instrument=SANS_INS.id,
)
#f = open("/home/elakian/tem.txt","w")
#f.write("Template: ")
#f.write( simplejson.dumps(wireit.template_to_wireit_diagram(template)))
#f.write("\n")
#f.write("Lang: ")
#f.write(simplejson.dumps(wireit.instrument_to_wireit_language(SANS_INS)))
#f.close()
print 'TEMPLATE', simplejson.dumps(
wireit.template_to_wireit_diagram(template))
#print 'RAW_INSTRUMENT: ', wireit.instrument_to_wireit_language(SANS_INS)
print 'LANGUAGE', simplejson.dumps(
wireit.instrument_to_wireit_language(SANS_INS))
def compare(plot_diff=False):
Environment("sans")
DATA_PATH = dirname(abspath(__file__))
def find_file(filename):
return joinpath(DATA_PATH, filename)
with open(find_file('absolute_scaling_lowQ_SANS.json'), 'rt') as fid:
template_def_low = json.loads(fid.read())
with open(find_file('absolute_scaling_highQ_SANS.json'), 'rt') as fid:
template_def_high = json.loads(fid.read())
template_low = Template(**template_def_low)
template_high = Template(**template_def_high)
output_low = process_template(template_low, {}, target=(13, 'output'))
output_high = process_template(template_high, {}, target=(13, 'output'))
# compare reductus, IGOR:
d298 = np.loadtxt(find_file("AUG17298.ABS"), skiprows=14)
d299 = np.loadtxt(find_file("AUG17299.ABS"), skiprows=14)
q_IGOR_low = d298[:, 0]
dq_IGOR_low = d298[:, 3]
meanQ_IGOR_low = d298[:, 4]
shadow_IGOR_low = d298[:, 5]
q_reductus_low = output_low.values[0].Q
dq_reductus_low = output_low.values[0].dQ
shadow_reductus_low = output_low.values[0].ShadowFactor
q_IGOR_high = d299[:, 0]
dq_IGOR_high = d299[:, 3]
q_reductus_high = output_high.values[0].Q
dq_reductus_high = output_high.values[0].dQ
I_IGOR_low = d298[:, 1]
dI_IGOR_low = d298[:, 2]
I_IGOR_high = d299[:, 1]
dI_IGOR_high = d299[:, 2]
I_reductus_low = output_low.values[0].I
dI_reductus_low = output_low.values[0].dI
I_reductus_high = output_high.values[0].I
dI_reductus_high = output_high.values[0].dI
if plot_diff:
from matplotlib import pyplot as plt
plt.plot(output_low.values[0].Q,
output_low.values[0].dQ,
'bo',
label="dQ: reductus")
plt.plot(output_high.values[0].Q,
output_high.values[0].dQ,
'bo',
label="dQ: reductus")
plt.plot(q_IGOR_low, dq_IGOR_low, label="dQ: IGOR")
plt.plot(q_IGOR_high, dq_IGOR_high, label="dQ: IGOR")
plt.legend()
plt.figure()
plt.plot(q_IGOR_low[:10],
shadow_IGOR_low[:10],
label="Shadow factor: IGOR")
plt.plot(q_reductus_low[:10],
shadow_reductus_low[:10],
label="Shadow factor: reductus")
plt.legend()
plt.figure()
plt.plot(q_IGOR_low, dq_IGOR_low / q_IGOR_low, label="dQ: IGOR")
plt.plot(q_reductus_low,
dq_reductus_low / q_reductus_low,
label="dQ: reductus")
plt.yscale('log')
plt.figure()
plt.errorbar(q_IGOR_low, I_IGOR_low, yerr=dI_IGOR_low, label="IGOR")
plt.errorbar(q_IGOR_high, I_IGOR_high, yerr=dI_IGOR_high, label="IGOR")
plt.errorbar(q_reductus_low,
I_reductus_low,
yerr=dI_reductus_low,
label="reductus")
plt.errorbar(q_reductus_high,
I_reductus_high,
yerr=dI_reductus_high,
label="reductus")
plt.yscale('log')
plt.legend()
plt.show()
def calc_terminal(template_def,
config,
nodenum,
terminal_id,
return_type='full',
export_type="column",
concatenate=True):
""" json-rpc wrapper for calc_single
template_def =
{"name": "template_name",
"description": "template description!",
"modules": ["list of modules"],
"wires": ["list of wires"],
"instrument": "facility.instrument_name",
"version": "2.7.3"
}
where modules in list of modules above have structure:
module =
{"module": "facility.instrument_name.module_name",
"version": "0.3.2"
}
and wires have structure:
[["wire_start_module_id:wire_start_terminal_id", "wire_end_module_id:wire_end_terminal_id"],
["1:output", "2:input"],
["0:xslice", "3:input"]
]
config =
[{"param": "value"}, ...]
nodenum is the module number from the template for which you wish to get the calculated value
terminal_id is the id of the terminal for that module, that you want to get the value from
(output terminals only).
"""
template = Template(**template_def)
#print "template_def:", template_def, "config:", config, "target:",nodenum,terminal_id
#print "modules","\n".join(m for m in df._module_registry.keys())
try:
retval = process_template(template,
config,
target=(nodenum, terminal_id))
except Exception:
print("==== template ====")
pprint(template_def)
print("==== config ====")
pprint(config)
traceback.print_exc()
raise
if return_type == 'full':
return retval.todict()
elif return_type == 'plottable':
return retval.get_plottable()
elif return_type == 'metadata':
return retval.get_metadata()
elif return_type == 'export':
# inject git version hash into export data:
rev_id = revision_info()
template_data = {
"template_data": {
"template": template_def,
"config": config,
"node": nodenum,
"terminal": terminal_id,
"server_git_hash": rev_id,
"export_type": export_type,
#"datasources": fetch.DATA_SOURCES, # Is this needed?
}
}
to_export = retval.get_export(export_type=export_type,
template_data=template_data,
concatenate=concatenate)
return to_export
raise KeyError(
return_type +
" not a valid return_type (should be one of ['full', 'plottable', 'metadata', 'export'])"
)
]
wires = [
dict(source=[0, 'output'], target=[1, 'input']),
dict(source=[1, 'output'], target=[2, 'input']),
]
# I'm unsure why config is needed currently if nothing needs to be supplied
# However, it does need to be the same length as the modules list
config = [
{},
{},
{},
]
template = Template(
name='test rowan',
description='example ROWAN diagram',
modules=modules,
wires=wires,
instrument=ROWAN26.id,
)
# the actual call to perform the reduction
result = run_template(template, config)
pprint(result)
# (testing, andy)
# ========= Convert the instrument definition to WireIt language =========
#from dataflow.wireit import instrument_to_wireit_language as wlang
#from dataflow.wireit import template_to_wireit_diagram as wdiag
#print "LANGUAGE:"
#print wlang(ROWAN26)
#print
def run_template(template_data, concatenate=True):
"""
Run a template defined by *template_data*.
Returns *bundle* and *exports*.
*bundle* is a :class:`dataflow.core.Bundle` object with a *values*
attribute containing the list of items in the bundle, and a *datatype*
attribute giving the data type for each value.
*exports* is a list of *[{'filename': str, 'value': valu*}, ...]* where
value depends on the export type requested in *template_data*.
Output from "column" export will contain a string with the file content,
with the first line containing the template data structure.
Output from "hdf" export will contain a byte sequence defining the
HDF file, with template data stored in the attribute NXroot@template_def.
Output from "json" export will contain a JSON string with hierarchical
structure *{template_data: json, outputs: [json, json, ...]}*.
If *concatenate* then all datasets will be combined into a single value.
Example::
from dataflow.rev import revision_info
revision = revision_info()
template_data = {
"template": json.loads(template_str),
"config": {}, # optional?
"node": node_id,
"terminal": terminal_id,
"export_type": "column",
"server_git_hash": revision,
"datasources": [
# ignored...
{'url': '...', 'start_path': '...', 'name': '...'},
],
}
bundle, export = run_template(template_data, concatenate=False)
for entry in export['values']:
with open(entry['filename'], 'w') as fd:
fd.write(entry['value'])
"""
#print("template_data", template_data['datasources'])
template_def = template_data['template']
template_config = template_data.get('config', {})
target = template_data['node'], template_data['terminal']
# CRUFT: use template_data["export_type"] when regression files are updated
template = Template(**template_def)
#template.show() # for debugging, show the template structure
# run the template
# TODO: use datasources given in template? It may be a security risk...
#datasources = template_data.get('datasources', [])
#if datasources:
# original = fetch.DATA_SOURCES
# fetch.DATA_SOURCES = datasources
# try:
# retval = process_template(template, template_config, target=target)
# finally:
# fetch.DATA_SOURCES = original
#else:
bundle = process_template(template, template_config, target=target)
# Smoke test on get_plottable(); not checking that it is correct yet.
bundle.get_plottable()
# Uncomment the following to save plottable during debugging.
#with open("plottable.json", "w") as fid:
# fid.write(json.dumps(bundle.get_plottable(), indent=2))
# TODO: default to column, json, hdf, ...
export_type = template_data.get("export_type", "column")
if export_type in bundle.datatype.export_types:
export = bundle.get_export(
template_data=template_data,
concatenate=concatenate,
export_type=export_type,
)
else:
export = None
return bundle, export
dict(source=[0, 'output'], target=[1, 'input']),
dict(source=[1, 'output'], target=[2, 'input']),
dict(source=[2, 'output'], target=[3, 'input']),
]
config = [
{
'files': ['f1.bt7', 'f2.bt7']
},
{},
{},
{},
]
template = Template(
name='test tas',
description='example TAS diagram',
modules=modules,
wires=wires,
instrument=BT7.id,
)
# verify wire json objects
print "== instrument\n", ppjson(json.dumps(instrument_to_wireit_language(BT7)))
print "== menu\n", ppjson(json.dumps(instrument_to_wireit_menu(BT7)))
tjson = json.dumps(template_to_wireit_diagram(template))
print "== diagram\n", ppjson(tjson)
t = wireit_diagram_to_template(json.loads(tjson), instrument=BT7)
print "== template\n", t.name, ":", t.description
pprint(t.modules)
pprint(t.wires)
# run the reductions