def _gen_pykwargs(context, pipe, module_id, steps=None): module = pipe['modules'][module_id] yield ('conf', module['conf']) describe = context.describe_input or context.describe_dependencies if not (describe and steps): wires = pipe['wires'] module_type = module['type'] # find the default input of this module for key, pipe_wire in wires.items(): moduleid = utils.pythonise(pipe_wire['src']['moduleid']) # todo? this equates the outputs is_default_out_only = ( utils.pythonise(pipe_wire['tgt']['moduleid']) == module_id and pipe_wire['tgt']['id'] != '_INPUT' and pipe_wire['src']['id'].startswith('_OUTPUT') ) # if the wire is to this module and it's *NOT* the default input # but it *is* the default output if is_default_out_only: # set the extra inputs of this module as pykwargs of this module pipe_id = utils.pythonise(pipe_wire['tgt']['id']) yield (pipe_id, steps[moduleid] if steps else Id(moduleid)) # set the embedded module in the pykwargs if this is loop module if module_type == 'loop': value = module['conf']['embed']['value'] pipe_id = utils.pythonise(value['id']) updated = steps[pipe_id] if steps else Id('pipe_%s' % pipe_id) yield ('embed', updated) # set splits in the pykwargs if this is split module def filter_func(x): module_id == utils.pythonise(x[1]['src']['moduleid']) if module_type == 'split': filtered = filter(filter_func, pipe['wires'].items()) count = len(filtered) updated = count if steps else Id(count) yield ('splits', updated)
def _gen_pykwargs(context, pipe, module_id, steps=None): module = pipe['modules'][module_id] yield ('conf', module['conf']) describe = context.describe_input or context.describe_dependencies if not (describe and steps): wires = pipe['wires'] module_type = module['type'] # find the default input of this module for key, pipe_wire in wires.items(): moduleid = utils.pythonise(pipe_wire['src']['moduleid']) # todo? this equates the outputs is_default_out_only = ( utils.pythonise(pipe_wire['tgt']['moduleid']) == module_id and pipe_wire['tgt']['id'] != '_INPUT' and pipe_wire['src']['id'].startswith('_OUTPUT')) # if the wire is to this module and it's *NOT* the default input # but it *is* the default output if is_default_out_only: # set the extra inputs of this module as pykwargs of this module pipe_id = utils.pythonise(pipe_wire['tgt']['id']) yield (pipe_id, steps[moduleid] if steps else Id(moduleid)) # set the embedded module in the pykwargs if this is loop module if module_type == 'loop': value = module['conf']['embed']['value'] pipe_id = utils.pythonise(value['id']) updated = steps[pipe_id] if steps else Id('pipe_%s' % pipe_id) yield ('embed', updated) # set splits in the pykwargs if this is split module def filter_func(x): module_id == utils.pythonise(x[1]['src']['moduleid']) if module_type == 'split': filtered = filter(filter_func, pipe['wires'].items()) count = len(filtered) updated = count if steps else Id(count) yield ('splits', updated)
def _get_input_module(pipe, module_id, steps): input_module = steps['forever'] if steps else 'forever' if module_id in pipe['embed']: input_module = '_INPUT' else: for key, pipe_wire in pipe['wires'].items(): moduleid = utils.pythonise(pipe_wire['src']['moduleid']) # todo? this equates the outputs is_default_in_and_out = ( utils.pythonise(pipe_wire['tgt']['moduleid']) == module_id and pipe_wire['tgt']['id'] == '_INPUT' and pipe_wire['src']['id'].startswith('_OUTPUT')) # if the wire is to this module and it's the default input and it's # the default output: if is_default_in_and_out: input_module = steps[moduleid] if steps else moduleid break return input_module
def _get_input_module(pipe, module_id, steps): input_module = steps['forever'] if steps else 'forever' if module_id in pipe['embed']: input_module = '_INPUT' else: for key, pipe_wire in pipe['wires'].items(): moduleid = utils.pythonise(pipe_wire['src']['moduleid']) # todo? this equates the outputs is_default_in_and_out = ( utils.pythonise(pipe_wire['tgt']['moduleid']) == module_id and pipe_wire['tgt']['id'] == '_INPUT' and pipe_wire['src']['id'].startswith('_OUTPUT') ) # if the wire is to this module and it's the default input and it's # the default output: if is_default_in_and_out: input_module = steps[moduleid] if steps else moduleid break return input_module
def parse_pipe_def(pipe_def, pipe_name='anonymous'): """Parse pipe JSON into internal structures Parameters ---------- pipe_def -- JSON representation of the pipe pipe_name -- a name for the pipe (used for linking pipes) Returns: pipe -- an internal representation of a pipe """ graph = defaultdict(list, utils.gen_graph1(pipe_def)) [graph[k].append(v) for k, v in utils.gen_graph2(pipe_def)] modules = dict(utils.gen_modules(pipe_def)) embed = dict(utils.gen_embedded_modules(pipe_def)) modules.update(embed) return { 'name': utils.pythonise(pipe_name), 'modules': modules, 'embed': embed, 'graph': dict(utils.gen_graph3(graph)), 'wires': dict(utils.gen_wires(pipe_def)), }
def filter_func(x): module_id == utils.pythonise(x[1]['src']['moduleid'])