def get_outputs(self, job_dir, job): result, outs = {}, self.app.outputs eval = ValueResolver(job) for out in outs: out_binding = out.output_binding ret = os.getcwd() os.chdir(job_dir) pattern = eval.resolve(out_binding.get('glob')) or "" patterns = chain(*[self.glob_or(p) for p in wrap_in_list(pattern)]) files = chain(*[glob.glob(p) for p in patterns]) res = [ File({ 'class': 'File', 'path': os.path.abspath(p), 'size': os.stat(p).st_size, 'checksum': 'sha1$' + checksum(os.path.abspath(p)), 'metadata': meta(p, job.inputs, eval, out_binding), 'secondaryFiles': secondary_files(p, out_binding, eval) }) for p in files ] if out_binding.get('outputEval'): self.app.load_file_content(out_binding, res) result[out.id] = eval.resolve(out_binding.get('outputEval'), [r.to_dict() for r in res]) else: result[out.id] = res os.chdir(ret) if out.depth == 0: res = result[out.id] result[out.id] = res[0] if res and isinstance(res, list) else res return result
def get_outputs(self, job_dir, job): result, outs = {}, self.app.outputs eval = ValueResolver(job) for out in outs: out_binding = out.output_binding ret = os.getcwd() os.chdir(job_dir) pattern = eval.resolve(out_binding.get('glob')) or "" patterns = chain(*[self.glob_or(p) for p in wrap_in_list(pattern)]) files = chain(*[glob.glob(p) for p in patterns]) res = [File({ 'class': 'File', 'path': os.path.abspath(p), 'size': os.stat(p).st_size, 'checksum': 'sha1$' + checksum(os.path.abspath(p)), 'metadata': meta(p, job.inputs, eval, out_binding), 'secondaryFiles': secondary_files(p, out_binding, eval) }) for p in files] if out_binding.get('outputEval'): self.app.load_file_content(out_binding, res) result[out.id] = eval.resolve( out_binding.get('outputEval'), [r.to_dict() for r in res] ) else: result[out.id] = res os.chdir(ret) if out.depth == 0: res = result[out.id] result[out.id] = res[0] if res and isinstance(res, list) else res return result
def secondary_files(p, outputBinding, evaluator): secondaryFiles = [] secFiles = wrap_in_list(evaluator.resolve(outputBinding.get('secondaryFiles', []))) for s in secFiles: path = sec_files_naming_conv(p, s) secondaryFiles.append({'path': to_abspath(path)}) return secondaryFiles
def make_avro(schema, named_defs): names = Names() make_avsc_object(FILE_SCHEMA, names) for d in named_defs: make_avsc_object(d, names) avsc = make_avsc_object(fix_file_type(wrap_in_list(schema)), names) return avsc
def secondary_files(p, outputBinding, evaluator): secondaryFiles = [] secFiles = wrap_in_list( evaluator.resolve(outputBinding.get('secondaryFiles', []))) for s in secFiles: path = sec_files_naming_conv(p, s) secondaryFiles.append({'path': to_abspath(path)}) return secondaryFiles
def __init__(self, id, validator=None, required=False, label=None, description=None, depth=0, input_binding=None, connect=None, value=None): super(WorkflowStepInput, self).__init__( id, validator, required, label, description, depth, input_binding ) self.connect = wrap_in_list(connect) if connect is not None else [] self.value = value
def from_dict(cls, context, d): id = d.get('id') ec = d.get('engineConfig') engine_config = wrap_in_list(ec) if ec else None docker_image = None for r in d.get('requirements', []): if r.get('class') == 'DockerRequirement': docker_image = r.get('dockerImageId', r.get('dockerPull')) return cls(id, docker_image, engine_config)
def __init__(self, id, validator=None, required=False, label=None, description=None, depth=0, output_binding=None, connect=None): super(WorkflowOutput, self).__init__(id, validator, required, label, description, depth, output_binding) self.connect = wrap_in_list(connect) if connect is not None else []
def __init__(self, id, validator=None, required=False, label=None, description=None, depth=0, output_binding=None, source=None): super(WorkflowOutput, self).__init__(id, validator, required, label, description, depth, output_binding) self.source = wrap_in_list(source) if source is not None else []
def add_edge_or_input(self, step, input_name, input_val): node_id = step.id if isinstance(input_val, dict) and '$from' in input_val: frm = wrap_in_list(input_val['$from']) for inp in frm: if '.' in inp: node, outp = inp.split('.') self.graph.add_edge(node, node_id, Relation(outp, input_name)) else: # TODO: merge input schemas if one input goes to different apps input = step.app.get_input(input_name) if inp not in self.graph.nodes: self.add_node(inp, input) self.graph.add_edge( inp, node_id, InputRelation(input_name) ) wf_input = copy.deepcopy(input) wf_input.id = inp self.inputs.append(wf_input) else: self.graph.node_data(node_id).inputs[input_name] = input_val
def from_dict(cls, context, d): instance = super(OutputParameter, cls).from_dict(context, d) connect = d.get('connect') instance.connect = wrap_in_list(connect) if connect is not None else [] return instance
def from_dict(cls, context, d): instance = super(WorkflowStepInput, cls).from_dict(context, d) source = d.get('source') instance.source = wrap_in_list(source) if source is not None else [] instance.value = d.get('default') return instance
def __init__(self, app_id, steps, context, inputs=None, outputs=None, to=None, app_description=None, annotations=None, platform_features=None): self.graph = Graph() self.inputs = inputs or [] self.outputs = outputs or [] self.executor = context.executor self.steps = steps self.to = to or {} self.context = context for step in steps: self.add_node(step.id, AppNode(step.app, {})) for step in steps: # inputs for input_port, input_val in six.iteritems(step.inputs): inp = wrap_in_list(input_val) for item in inp: self.add_edge_or_input(step, input_port, item) # outputs if step.outputs: for output_port, output_val in six.iteritems(step.outputs): self.to[output_val['$to']] = output_port if isinstance(step.app, WorkflowApp): output_node = step.app.get_output(step.app.to.get(output_port)) else: output_node = step.app.get_output(output_port) output_id = output_val['$to'] self.add_node(output_id, output_node) self.graph.add_edge( step.id, output_id, OutputRelation(output_port) ) # output_node.id = output_val['$to'] self.outputs.append(output_node) if not self.graph.connected(): pass # raise ValidationError('Graph is not connected') schema = { "@type": "JsonSchema", "type": "object", "properties": {}, "required": [] } for inp in self.inputs: schema['properties'][inp.id] = inp.validator.schema if inp.required: schema['required'].append(inp.id) super(WorkflowApp, self).__init__( app_id, JsonSchema(context, schema), self.outputs, app_description=app_description, annotations=annotations, platform_features=platform_features )
def from_dict(cls, context, d): instance = super(WorkflowStepInput, cls).from_dict(context, d) connect = d.get('connect') instance.connect = wrap_in_list(connect) if connect is not None else [] instance.value = d.get('default') return instance
def from_dict(cls, context, d): instance = super(OutputParameter, cls).from_dict(context, d) source = d.get('source') instance.source = wrap_in_list(source) if source is not None else [] return instance
def __init__(self, id, validator=None, required=False, label=None, description=None, depth=0, output_binding=None, source=None): super(WorkflowOutput, self).__init__( id, validator, required, label, description, depth, output_binding ) self.source = wrap_in_list(source) if source is not None else []