def _gen_pykwargs(context, pipe, module_id, steps=None):
module = pipe['modules'][module_id]
yield ('conf', module['conf'])
describe = context.describe_input or context.describe_dependencies
if not (describe and steps):
wires = pipe['wires']
module_type = module['type']
# find the default input of this module
for key, pipe_wire in wires.items():
moduleid = utils.pythonise(pipe_wire['src']['moduleid'])
# todo? this equates the outputs
is_default_out_only = (
utils.pythonise(pipe_wire['tgt']['moduleid']) == module_id
and pipe_wire['tgt']['id'] != '_INPUT'
and pipe_wire['src']['id'].startswith('_OUTPUT')
)
# if the wire is to this module and it's *NOT* the default input
# but it *is* the default output
if is_default_out_only:
# set the extra inputs of this module as pykwargs of this module
pipe_id = utils.pythonise(pipe_wire['tgt']['id'])
yield (pipe_id, steps[moduleid] if steps else Id(moduleid))
# set the embedded module in the pykwargs if this is loop module
if module_type == 'loop':
value = module['conf']['embed']['value']
pipe_id = utils.pythonise(value['id'])
updated = steps[pipe_id] if steps else Id('pipe_%s' % pipe_id)
yield ('embed', updated)
# set splits in the pykwargs if this is split module
def filter_func(x):
module_id == utils.pythonise(x[1]['src']['moduleid'])
if module_type == 'split':
filtered = filter(filter_func, pipe['wires'].items())
count = len(filtered)
updated = count if steps else Id(count)
yield ('splits', updated)
def _gen_pykwargs(context, pipe, module_id, steps=None):
module = pipe['modules'][module_id]
yield ('conf', module['conf'])
describe = context.describe_input or context.describe_dependencies
if not (describe and steps):
wires = pipe['wires']
module_type = module['type']
# find the default input of this module
for key, pipe_wire in wires.items():
moduleid = utils.pythonise(pipe_wire['src']['moduleid'])
# todo? this equates the outputs
is_default_out_only = (
utils.pythonise(pipe_wire['tgt']['moduleid']) == module_id
and pipe_wire['tgt']['id'] != '_INPUT'
and pipe_wire['src']['id'].startswith('_OUTPUT'))
# if the wire is to this module and it's *NOT* the default input
# but it *is* the default output
if is_default_out_only:
# set the extra inputs of this module as pykwargs of this module
pipe_id = utils.pythonise(pipe_wire['tgt']['id'])
yield (pipe_id, steps[moduleid] if steps else Id(moduleid))
# set the embedded module in the pykwargs if this is loop module
if module_type == 'loop':
value = module['conf']['embed']['value']
pipe_id = utils.pythonise(value['id'])
updated = steps[pipe_id] if steps else Id('pipe_%s' % pipe_id)
yield ('embed', updated)
# set splits in the pykwargs if this is split module
def filter_func(x):
module_id == utils.pythonise(x[1]['src']['moduleid'])
if module_type == 'split':
filtered = filter(filter_func, pipe['wires'].items())
count = len(filtered)
updated = count if steps else Id(count)
yield ('splits', updated)
def _get_input_module(pipe, module_id, steps):
input_module = steps['forever'] if steps else 'forever'
if module_id in pipe['embed']:
input_module = '_INPUT'
else:
for key, pipe_wire in pipe['wires'].items():
moduleid = utils.pythonise(pipe_wire['src']['moduleid'])
# todo? this equates the outputs
is_default_in_and_out = (
utils.pythonise(pipe_wire['tgt']['moduleid']) == module_id
and pipe_wire['tgt']['id'] == '_INPUT'
and pipe_wire['src']['id'].startswith('_OUTPUT'))
# if the wire is to this module and it's the default input and it's
# the default output:
if is_default_in_and_out:
input_module = steps[moduleid] if steps else moduleid
break
return input_module
def _get_input_module(pipe, module_id, steps):
input_module = steps['forever'] if steps else 'forever'
if module_id in pipe['embed']:
input_module = '_INPUT'
else:
for key, pipe_wire in pipe['wires'].items():
moduleid = utils.pythonise(pipe_wire['src']['moduleid'])
# todo? this equates the outputs
is_default_in_and_out = (
utils.pythonise(pipe_wire['tgt']['moduleid']) == module_id
and pipe_wire['tgt']['id'] == '_INPUT'
and pipe_wire['src']['id'].startswith('_OUTPUT')
)
# if the wire is to this module and it's the default input and it's
# the default output:
if is_default_in_and_out:
input_module = steps[moduleid] if steps else moduleid
break
return input_module
def parse_pipe_def(pipe_def, pipe_name='anonymous'):
"""Parse pipe JSON into internal structures
Parameters
----------
pipe_def -- JSON representation of the pipe
pipe_name -- a name for the pipe (used for linking pipes)
Returns:
pipe -- an internal representation of a pipe
"""
graph = defaultdict(list, utils.gen_graph1(pipe_def))
[graph[k].append(v) for k, v in utils.gen_graph2(pipe_def)]
modules = dict(utils.gen_modules(pipe_def))
embed = dict(utils.gen_embedded_modules(pipe_def))
modules.update(embed)
return {
'name': utils.pythonise(pipe_name),
'modules': modules,
'embed': embed,
'graph': dict(utils.gen_graph3(graph)),
'wires': dict(utils.gen_wires(pipe_def)),
}
def parse_pipe_def(pipe_def, pipe_name='anonymous'):
"""Parse pipe JSON into internal structures
Parameters
----------
pipe_def -- JSON representation of the pipe
pipe_name -- a name for the pipe (used for linking pipes)
Returns:
pipe -- an internal representation of a pipe
"""
graph = defaultdict(list, utils.gen_graph1(pipe_def))
[graph[k].append(v) for k, v in utils.gen_graph2(pipe_def)]
modules = dict(utils.gen_modules(pipe_def))
embed = dict(utils.gen_embedded_modules(pipe_def))
modules.update(embed)
return {
'name': utils.pythonise(pipe_name),
'modules': modules,
'embed': embed,
'graph': dict(utils.gen_graph3(graph)),
'wires': dict(utils.gen_wires(pipe_def)),
}
def filter_func(x):
module_id == utils.pythonise(x[1]['src']['moduleid'])
def filter_func(x):
module_id == utils.pythonise(x[1]['src']['moduleid'])
