def validate(self, data_spec):
"""Ensure the given data spec is compatible with this port.
:param dict data_spec: Data specification
:returns: bool
>>> spec = {'name': 'a', 'type': 'number', 'format': 'number'}
>>> port = Port(spec)
>>> port.validate({'format': 'number', 'data': 1.5})
True
>>> port.validate({'format': 'json', 'data': '1.5'})
True
>>> port.validate({'format': 'number', 'data': '1.5'})
False
>>> port.validate({'format': 'unknown format', 'data': '...'})
False
"""
try:
return isvalid(self.type, data_spec)
except Exception: # catchall validation error
return False
def run(task, inputs, outputs, task_inputs, task_outputs, validate, # noqa
auto_convert, **kwargs):
# Make map of steps
steps = {step["name"]: step for step in task["steps"]}
# Make map of input bindings
bindings = {step["name"]: {} for step in task["steps"]}
# Create dependency graph and downstream pointers
dependencies = {step["name"]: set() for step in task["steps"]}
downstream = {}
for conn in task["connections"]:
# Add dependency graph link for internal links
if "input_step" in conn and "output_step" in conn:
dependencies[conn["input_step"]].add(conn["output_step"])
# Add downstream links for links with output
if "output_step" in conn:
ds = downstream.setdefault(conn["output_step"], {})
ds_list = ds.setdefault(conn["output"], [])
ds_list.append(conn)
# Set initial bindings for inputs
if "input_step" in conn and "output_step" not in conn:
name = conn["name"]
bindings[conn["input_step"]][conn["input"]] = {
"format": task_inputs[name]["format"],
"data": inputs[name]["script_data"]
}
# Traverse analyses in topological order
for step_set in toposort(dependencies):
for step in step_set:
# Visualizations cannot be executed
if ("visualization" in steps[step] and
steps[step]["visualization"]):
continue
# Run step
print "--- beginning: %s ---" % steps[step]["name"]
out = girder_worker.run(steps[step]["task"], bindings[step])
print "--- finished: %s ---" % steps[step]["name"]
# Update bindings of downstream analyses
if step in downstream:
for name, conn_list in downstream[step].iteritems():
for conn in conn_list:
if "input_step" in conn:
# This is a connection to a downstream step
b = bindings[conn["input_step"]]
b[conn["input"]] = out[name]
else:
# This is a connection to a final output
o = outputs[conn["name"]]
o["script_data"] = out[name]["data"]
# Output visualization paramaters
outputs["_visualizations"] = []
for step in task["steps"]:
if "visualization" not in step or not step["visualization"]:
continue
vis_bindings = {}
for b, value in bindings[step["name"]].iteritems():
script_output = value
vis_input = None
for step_input in step["task"]["inputs"]:
if step_input["name"] == b:
vis_input = step_input
if not vis_input:
raise Exception(
"Could not find visualization input named " + b + "."
)
# Validate the output
if (validate and not
girder_worker.isvalid(vis_input["type"], script_output)):
raise Exception(
"Output %s (%s) is not in the expected type (%s) and "
"format (%s)." % (name, type(script_output["data"]),
vis_input["type"], script_output["format"]))
if auto_convert:
vis_bindings[b] = girder_worker.convert(
vis_input["type"],
script_output,
{"format": vis_input["format"]}
)
elif script_output["format"] == vis_input["format"]:
data = script_output["data"]
if "mode" in script_output:
girder_worker.io.push(data, script_output)
else:
vis_bindings[b] = {
"type": vis_input["type"],
"format": vis_input["format"],
"data": data
}
else:
raise Exception(
"Expected exact format match but '" +
script_output["format"] +
"' != '" + vis_input["format"] + "'."
)
if "script_data" in vis_bindings[b]:
del vis_bindings[b]["script_data"]
outputs["_visualizations"].append({
"mode": "preset",
"type": step["name"],
"inputs": vis_bindings
})
def run(task, inputs, outputs, task_inputs, task_outputs, validate, # noqa
auto_convert, **kwargs):
# Make map of steps
steps = {step['name']: step for step in task['steps']}
# Make map of input bindings
bindings = {step['name']: {} for step in task['steps']}
# Create dependency graph and downstream pointers
dependencies = {step['name']: set() for step in task['steps']}
downstream = {}
for conn in task['connections']:
# Add dependency graph link for internal links
if 'input_step' in conn and 'output_step' in conn:
dependencies[conn['input_step']].add(conn['output_step'])
# Add downstream links for links with output
if 'output_step' in conn:
ds = downstream.setdefault(conn['output_step'], {})
ds_list = ds.setdefault(conn['output'], [])
ds_list.append(conn)
# Set initial bindings for inputs
if 'input_step' in conn and 'output_step' not in conn:
name = conn['name']
bindings[conn['input_step']][conn['input']] = {
'format': task_inputs[name]['format'],
'data': inputs[name]['script_data']
}
# Traverse analyses in topological order
for step_set in toposort(dependencies):
for step in step_set:
# Visualizations cannot be executed
if ('visualization' in steps[step] and
steps[step]['visualization']):
continue
# Run step
print '--- beginning: %s ---' % steps[step]['name']
out = girder_worker.run(steps[step]['task'], bindings[step])
print '--- finished: %s ---' % steps[step]['name']
# Update bindings of downstream analyses
if step in downstream:
for name, conn_list in downstream[step].iteritems():
for conn in conn_list:
if 'input_step' in conn:
# This is a connection to a downstream step
b = bindings[conn['input_step']]
b[conn['input']] = out[name]
else:
# This is a connection to a final output
o = outputs[conn['name']]
o['script_data'] = out[name]['data']
# Output visualization parameters
outputs['_visualizations'] = []
for step in task['steps']:
if 'visualization' not in step or not step['visualization']:
continue
vis_bindings = {}
for b, value in bindings[step['name']].iteritems():
script_output = value
vis_input = None
for step_input in step['task']['inputs']:
if step_input['name'] == b:
vis_input = step_input
if not vis_input:
raise Exception(
'Could not find visualization input named ' + b + '.'
)
# Validate the output
if (validate and not
girder_worker.isvalid(vis_input['type'], script_output)):
raise Exception(
'Output %s (%s) is not in the expected type (%s) and '
'format (%s).' % (
name, type(script_output['data']),
vis_input['type'], script_output['format']))
if auto_convert:
vis_bindings[b] = girder_worker.convert(
vis_input['type'],
script_output,
{'format': vis_input['format']}
)
elif script_output['format'] == vis_input['format']:
data = script_output['data']
if 'mode' in script_output:
girder_worker.io.push(data, script_output)
else:
vis_bindings[b] = {
'type': vis_input['type'],
'format': vis_input['format'],
'data': data
}
else:
raise Exception(
'Expected exact format match but "' +
script_output['format'] +
'" != "' + vis_input['format'] + '".'
)
if 'script_data' in vis_bindings[b]:
del vis_bindings[b]['script_data']
outputs['_visualizations'].append({
'mode': 'preset',
'type': step['name'],
'inputs': vis_bindings
})
