def get_outputs(self, job_dir, job):
result, outs = {}, self.app.outputs
eval = ValueResolver(job)
for out in outs:
out_binding = out.output_binding
ret = os.getcwd()
os.chdir(job_dir)
pattern = eval.resolve(out_binding.get('glob')) or ""
patterns = chain(*[self.glob_or(p) for p in wrap_in_list(pattern)])
files = chain(*[glob.glob(p) for p in patterns])
res = [
File({
'class': 'File',
'path': os.path.abspath(p),
'size': os.stat(p).st_size,
'checksum': 'sha1$' + checksum(os.path.abspath(p)),
'metadata': meta(p, job.inputs, eval, out_binding),
'secondaryFiles': secondary_files(p, out_binding, eval)
}) for p in files
]
if out_binding.get('outputEval'):
self.app.load_file_content(out_binding, res)
result[out.id] = eval.resolve(out_binding.get('outputEval'),
[r.to_dict() for r in res])
else:
result[out.id] = res
os.chdir(ret)
if out.depth == 0:
res = result[out.id]
result[out.id] = res[0] if res and isinstance(res,
list) else res
return result
def get_outputs(self, job_dir, job):
result, outs = {}, self.app.outputs
eval = ValueResolver(job)
for out in outs:
out_binding = out.output_binding
ret = os.getcwd()
os.chdir(job_dir)
pattern = eval.resolve(out_binding.get('glob')) or ""
patterns = chain(*[self.glob_or(p) for p in wrap_in_list(pattern)])
files = chain(*[glob.glob(p) for p in patterns])
res = [File({
'class': 'File',
'path': os.path.abspath(p),
'size': os.stat(p).st_size,
'checksum': 'sha1$' + checksum(os.path.abspath(p)),
'metadata': meta(p, job.inputs, eval, out_binding),
'secondaryFiles': secondary_files(p, out_binding, eval)
}) for p in files]
if out_binding.get('outputEval'):
self.app.load_file_content(out_binding, res)
result[out.id] = eval.resolve(
out_binding.get('outputEval'),
[r.to_dict() for r in res]
)
else:
result[out.id] = res
os.chdir(ret)
if out.depth == 0:
res = result[out.id]
result[out.id] = res[0] if res and isinstance(res, list) else res
return result
def secondary_files(p, outputBinding, evaluator):
secondaryFiles = []
secFiles = wrap_in_list(evaluator.resolve(outputBinding.get('secondaryFiles', [])))
for s in secFiles:
path = sec_files_naming_conv(p, s)
secondaryFiles.append({'path': to_abspath(path)})
return secondaryFiles
def make_avro(schema, named_defs):
names = Names()
make_avsc_object(FILE_SCHEMA, names)
for d in named_defs:
make_avsc_object(d, names)
avsc = make_avsc_object(fix_file_type(wrap_in_list(schema)), names)
return avsc
def secondary_files(p, outputBinding, evaluator):
secondaryFiles = []
secFiles = wrap_in_list(
evaluator.resolve(outputBinding.get('secondaryFiles', [])))
for s in secFiles:
path = sec_files_naming_conv(p, s)
secondaryFiles.append({'path': to_abspath(path)})
return secondaryFiles
def make_avro(schema, named_defs):
names = Names()
make_avsc_object(FILE_SCHEMA, names)
for d in named_defs:
make_avsc_object(d, names)
avsc = make_avsc_object(fix_file_type(wrap_in_list(schema)), names)
return avsc
def __init__(self, id, validator=None, required=False, label=None,
description=None, depth=0, input_binding=None, connect=None,
value=None):
super(WorkflowStepInput, self).__init__(
id, validator, required, label, description, depth, input_binding
)
self.connect = wrap_in_list(connect) if connect is not None else []
self.value = value
def from_dict(cls, context, d):
id = d.get('id')
ec = d.get('engineConfig')
engine_config = wrap_in_list(ec) if ec else None
docker_image = None
for r in d.get('requirements', []):
if r.get('class') == 'DockerRequirement':
docker_image = r.get('dockerImageId', r.get('dockerPull'))
return cls(id, docker_image, engine_config)
def from_dict(cls, context, d):
id = d.get('id')
ec = d.get('engineConfig')
engine_config = wrap_in_list(ec) if ec else None
docker_image = None
for r in d.get('requirements', []):
if r.get('class') == 'DockerRequirement':
docker_image = r.get('dockerImageId', r.get('dockerPull'))
return cls(id, docker_image, engine_config)
def __init__(self,
id,
validator=None,
required=False,
label=None,
description=None,
depth=0,
output_binding=None,
connect=None):
super(WorkflowOutput,
self).__init__(id, validator, required, label, description,
depth, output_binding)
self.connect = wrap_in_list(connect) if connect is not None else []
def __init__(self,
id,
validator=None,
required=False,
label=None,
description=None,
depth=0,
output_binding=None,
source=None):
super(WorkflowOutput,
self).__init__(id, validator, required, label, description,
depth, output_binding)
self.source = wrap_in_list(source) if source is not None else []
def add_edge_or_input(self, step, input_name, input_val):
node_id = step.id
if isinstance(input_val, dict) and '$from' in input_val:
frm = wrap_in_list(input_val['$from'])
for inp in frm:
if '.' in inp:
node, outp = inp.split('.')
self.graph.add_edge(node, node_id, Relation(outp, input_name))
else:
# TODO: merge input schemas if one input goes to different apps
input = step.app.get_input(input_name)
if inp not in self.graph.nodes:
self.add_node(inp, input)
self.graph.add_edge(
inp, node_id, InputRelation(input_name)
)
wf_input = copy.deepcopy(input)
wf_input.id = inp
self.inputs.append(wf_input)
else:
self.graph.node_data(node_id).inputs[input_name] = input_val
def from_dict(cls, context, d):
instance = super(OutputParameter, cls).from_dict(context, d)
connect = d.get('connect')
instance.connect = wrap_in_list(connect) if connect is not None else []
return instance
def from_dict(cls, context, d):
instance = super(WorkflowStepInput, cls).from_dict(context, d)
source = d.get('source')
instance.source = wrap_in_list(source) if source is not None else []
instance.value = d.get('default')
return instance
def __init__(self, app_id, steps, context,
inputs=None, outputs=None, to=None,
app_description=None,
annotations=None,
platform_features=None):
self.graph = Graph()
self.inputs = inputs or []
self.outputs = outputs or []
self.executor = context.executor
self.steps = steps
self.to = to or {}
self.context = context
for step in steps:
self.add_node(step.id, AppNode(step.app, {}))
for step in steps:
# inputs
for input_port, input_val in six.iteritems(step.inputs):
inp = wrap_in_list(input_val)
for item in inp:
self.add_edge_or_input(step, input_port, item)
# outputs
if step.outputs:
for output_port, output_val in six.iteritems(step.outputs):
self.to[output_val['$to']] = output_port
if isinstance(step.app, WorkflowApp):
output_node = step.app.get_output(step.app.to.get(output_port))
else:
output_node = step.app.get_output(output_port)
output_id = output_val['$to']
self.add_node(output_id, output_node)
self.graph.add_edge(
step.id, output_id, OutputRelation(output_port)
)
# output_node.id = output_val['$to']
self.outputs.append(output_node)
if not self.graph.connected():
pass
# raise ValidationError('Graph is not connected')
schema = {
"@type": "JsonSchema",
"type": "object",
"properties": {},
"required": []
}
for inp in self.inputs:
schema['properties'][inp.id] = inp.validator.schema
if inp.required:
schema['required'].append(inp.id)
super(WorkflowApp, self).__init__(
app_id, JsonSchema(context, schema), self.outputs,
app_description=app_description,
annotations=annotations,
platform_features=platform_features
)
def from_dict(cls, context, d):
instance = super(WorkflowStepInput, cls).from_dict(context, d)
connect = d.get('connect')
instance.connect = wrap_in_list(connect) if connect is not None else []
instance.value = d.get('default')
return instance
def from_dict(cls, context, d):
instance = super(OutputParameter, cls).from_dict(context, d)
source = d.get('source')
instance.source = wrap_in_list(source) if source is not None else []
return instance
def from_dict(cls, context, d):
instance = super(OutputParameter, cls).from_dict(context, d)
source = d.get('source')
instance.source = wrap_in_list(source) if source is not None else []
return instance
def from_dict(cls, context, d):
instance = super(OutputParameter, cls).from_dict(context, d)
connect = d.get('connect')
instance.connect = wrap_in_list(connect) if connect is not None else []
return instance
def from_dict(cls, context, d):
instance = super(WorkflowStepInput, cls).from_dict(context, d)
connect = d.get('connect')
instance.connect = wrap_in_list(connect) if connect is not None else []
instance.value = d.get('default')
return instance
def from_dict(cls, context, d):
instance = super(WorkflowStepInput, cls).from_dict(context, d)
source = d.get('source')
instance.source = wrap_in_list(source) if source is not None else []
instance.value = d.get('default')
return instance
def __init__(self, id, validator=None, required=False, label=None,
description=None, depth=0, output_binding=None, source=None):
super(WorkflowOutput, self).__init__(
id, validator, required, label, description, depth, output_binding
)
self.source = wrap_in_list(source) if source is not None else []
