def _parse_pipe(json_pipe, pipe_name="anonymous"):
"""Parse pipe JSON into internal structures
Keyword arguments:
json_pipe -- JSON representation of the pipe
pipe_name -- a name for the pipe (used for linking pipes)
Returns:
pipe -- an internal representation of a pipe
"""
pipe = {"name": util.pythonise(pipe_name)}
pipe["modules"] = {}
pipe["embed"] = {}
pipe["graph"] = {}
pipe["wires"] = {}
modules = json_pipe["modules"]
if not isinstance(modules, list):
modules = [modules]
for module in modules:
pipe["modules"][util.pythonise(module["id"])] = module
pipe["graph"][util.pythonise(module["id"])] = []
if module["type"] == "loop":
embed = module["conf"]["embed"]["value"]
pipe["modules"][util.pythonise(embed["id"])] = embed
pipe["graph"][util.pythonise(embed["id"])] = []
pipe["embed"][util.pythonise(embed["id"])] = embed
# make the loop dependent on its embedded module
pipe["graph"][util.pythonise(embed["id"])].append(util.pythonise(module["id"]))
wires = json_pipe["wires"]
if not isinstance(wires, list):
wires = [wires]
for wire in wires:
pipe["graph"][util.pythonise(wire["src"]["moduleid"])].append(util.pythonise(wire["tgt"]["moduleid"]))
# Remove any orphan nodes
for node in pipe["graph"].keys():
targetted = [node in pipe["graph"][k] for k in pipe["graph"]]
if not pipe["graph"][node] and not any(targetted):
del pipe["graph"][node]
for wire in wires:
pipe["wires"][util.pythonise(wire["id"])] = wire
return pipe
def build_pipe(context, pipe):
"""Convert a pipe into an executable Python pipeline
If context.describe_input then just return the input requirements instead of the pipeline
Note: any subpipes must be available to import as .py files
current namespace can become polluted by submodule wrapper definitions
"""
pyinput = []
module_sequence = topological_sort(pipe["graph"])
# First pass to find and import any required subpipelines and user inputs
# Note: assumes they have already been compiled to accessible .py files
for module_id in module_sequence:
module = pipe["modules"][module_id]
if module["type"].startswith("pipe:"):
__import__(util.pythonise(module["type"]))
if module["conf"] and "prompt" in module["conf"] and context.describe_input:
pyinput.append(
(
module["conf"]["position"]["value"],
module["conf"]["name"]["value"],
module["conf"]["prompt"]["value"],
module["conf"]["default"]["type"],
module["conf"]["default"]["value"],
)
)
# Note: there seems to be no need to recursively collate inputs from subpipelines
if context.describe_input:
return sorted(pyinput)
steps = {}
steps["forever"] = pipeforever.pipe_forever(context, None, conf=None)
for module_id in module_sequence:
module = pipe["modules"][module_id]
# Plumb I/O
input_module = steps["forever"]
for wire in pipe["wires"]:
if (
util.pythonise(pipe["wires"][wire]["tgt"]["moduleid"]) == module_id
and pipe["wires"][wire]["tgt"]["id"] == "_INPUT"
and pipe["wires"][wire]["src"]["id"].startswith("_OUTPUT")
):
input_module = steps[util.pythonise(pipe["wires"][wire]["src"]["moduleid"])]
if module_id in pipe["embed"]:
assert input_module == steps["forever"], "input_module of an embedded module was already set"
input_module = "_INPUT"
pargs = [context, input_module]
kargs = {"conf": module["conf"]}
for wire in pipe["wires"]:
if (
util.pythonise(pipe["wires"][wire]["tgt"]["moduleid"]) == module_id
and pipe["wires"][wire]["tgt"]["id"] != "_INPUT"
and pipe["wires"][wire]["src"]["id"].startswith("_OUTPUT")
):
kargs["%(id)s" % {"id": util.pythonise(pipe["wires"][wire]["tgt"]["id"])}] = steps[
util.pythonise(pipe["wires"][wire]["src"]["moduleid"])
]
if module["type"] == "loop":
kargs["embed"] = steps[util.pythonise(module["conf"]["embed"]["value"]["id"])]
if module["type"] == "split":
kargs["splits"] = len(
[1 for w in pipe["wires"] if util.pythonise(pipe["wires"][w]["src"]["moduleid"]) == module_id]
)
# todo (re)import other pipes dynamically
pymodule_name = "pipe%(module_type)s" % {"module_type": module["type"]}
pymodule_generator_name = "pipe_%(module_type)s" % {"module_type": module["type"]}
if module["type"].startswith("pipe:"):
pymodule_name = "sys.modules['%(module_type)s']" % {"module_type": util.pythonise(module["type"])}
pymodule_generator_name = "%(module_type)s" % {"module_type": util.pythonise(module["type"])}
if module_id in pipe["embed"]:
# We need to wrap submodules (used by loops) so we can pass the input at runtime (as we can to subpipelines)
pypipe = (
"""def pipe_%(module_id)s(context, _INPUT, conf=None, **kwargs):\n"""
""" return %(pymodule_name)s.%(pymodule_generator_name)s(context, _INPUT, conf=%(conf)s, **kwargs)\n"""
% {
"module_id": module_id,
"pymodule_name": pymodule_name,
"pymodule_generator_name": pymodule_generator_name,
"conf": module["conf"],
# Note: no embed (so no subloops) or wire kargs are passed and outer kwargs are passed in
}
)
exec pypipe # Note: evaluated in current namespace - todo ok?
steps[module_id] = eval("pipe_%(module_id)s" % {"module_id": module_id})
else:
module_ref = eval(
"%(pymodule_name)s.%(pymodule_generator_name)s"
% {"pymodule_name": pymodule_name, "pymodule_generator_name": pymodule_generator_name}
)
steps[module_id] = module_ref(*pargs, **kargs)
if context.verbose:
print "%s (%s) = %s(%s)" % (steps[module_id], module_id, module_ref, str(pargs))
return steps[module_id]
def _parse_pipe(json_pipe, pipe_name="anonymous"):
"""Parse pipe JSON into internal structures
Keyword arguments:
json_pipe -- JSON representation of the pipe
pipe_name -- a name for the pipe (used for linking pipes)
Returns:
pipe -- an internal representation of a pipe
"""
pipe = {'name': util.pythonise(pipe_name)}
pipe['modules'] = {}
pipe['embed'] = {}
pipe['graph'] = {}
pipe['wires'] = {}
modules = json_pipe['modules']
if not isinstance(modules, list):
modules = [modules]
for module in modules:
pipe['modules'][util.pythonise(module['id'])] = module
pipe['graph'][util.pythonise(module['id'])] = []
if module['type'] == 'loop':
embed = module['conf']['embed']['value']
pipe['modules'][util.pythonise(embed['id'])] = embed
pipe['graph'][util.pythonise(embed['id'])] = []
pipe['embed'][util.pythonise(embed['id'])] = embed
#make the loop dependent on its embedded module
pipe['graph'][util.pythonise(embed['id'])].append(
util.pythonise(module['id']))
wires = json_pipe['wires']
if not isinstance(wires, list):
wires = [wires]
for wire in wires:
pipe['graph'][util.pythonise(wire['src']['moduleid'])].append(
util.pythonise(wire['tgt']['moduleid']))
#Remove any orphan nodes
for node in pipe['graph'].keys():
targetted = [node in pipe['graph'][k] for k in pipe['graph']]
if not pipe['graph'][node] and not any(targetted):
del pipe['graph'][node]
for wire in wires:
pipe['wires'][util.pythonise(wire['id'])] = wire
return pipe
def write_pipe(context, pipe):
"""Convert a pipe into Python script
If context.describe_input is passed to the script then it just returns the input requirements instead of the pipeline
"""
pypipe = (
"""#Pipe %(pipename)s generated by pipe2py\n"""
"""\n"""
"""from pipe2py import Context\n"""
"""from pipe2py.modules import *\n"""
"""\n""" % {"pipename": pipe["name"]}
)
pyinput = []
module_sequence = topological_sort(pipe["graph"])
# First pass to find any required subpipelines and user inputs
for module_id in module_sequence:
module = pipe["modules"][module_id]
if module["type"].startswith("pipe:"):
pypipe += """import %(module_type)s\n""" % {"module_type": util.pythonise(module["type"])}
if module["conf"] and "prompt" in module["conf"]:
pyinput.append(
(
module["conf"]["position"]["value"],
module["conf"]["name"]["value"],
module["conf"]["prompt"]["value"],
module["conf"]["default"]["type"],
module["conf"]["default"]["value"],
)
)
# Note: there seems to be no need to recursively collate inputs from subpipelines
pypipe += (
"""\n"""
"""def %(pipename)s(context, _INPUT, conf=None, **kwargs):\n"""
""" "Pipeline"\n""" # todo insert pipeline description here
""" if conf is None:\n"""
""" conf = {}\n"""
"""\n"""
""" if context.describe_input:\n"""
""" return %(inputs)s\n"""
"""\n"""
""" forever = pipeforever.pipe_forever(context, None, conf=None)\n"""
"""\n""" % {"pipename": pipe["name"], "inputs": unicode(sorted(pyinput))} # todo pprint this
)
prev_module = []
for module_id in module_sequence:
module = pipe["modules"][module_id]
# Plumb I/O
input_module = "forever"
for wire in pipe["wires"]:
if (
util.pythonise(pipe["wires"][wire]["tgt"]["moduleid"]) == module_id
and pipe["wires"][wire]["tgt"]["id"] == "_INPUT"
and pipe["wires"][wire]["src"]["id"].startswith("_OUTPUT")
):
input_module = util.pythonise(pipe["wires"][wire]["src"]["moduleid"])
if module_id in pipe["embed"]:
assert input_module == "forever", "input_module of an embedded module was already set"
input_module = "_INPUT"
pargs = [
"%(input_module)s" % {"input_module": input_module},
"conf=%(conf)s" % {"conf": module["conf"]}, # todo pprint this
]
for wire in pipe["wires"]:
if (
util.pythonise(pipe["wires"][wire]["tgt"]["moduleid"]) == module_id
and pipe["wires"][wire]["tgt"]["id"] != "_INPUT"
and pipe["wires"][wire]["src"]["id"].startswith("_OUTPUT")
):
pargs.append(
"%(id)s = %(secondary_module)s"
% {
"id": util.pythonise(pipe["wires"][wire]["tgt"]["id"]),
"secondary_module": util.pythonise(pipe["wires"][wire]["src"]["moduleid"]),
}
)
if module["type"] == "loop":
pargs.append(
"embed = pipe_%(embed_module)s"
% {"embed_module": util.pythonise(module["conf"]["embed"]["value"]["id"])}
)
if module["type"] == "split":
pargs.append(
"splits = %(splits)s"
% {
"splits": len(
[1 for w in pipe["wires"] if util.pythonise(pipe["wires"][w]["src"]["moduleid"]) == module_id]
)
}
)
pymodule_name = "pipe%(module_type)s" % {"module_type": module["type"]}
pymodule_generator_name = "pipe_%(module_type)s" % {"module_type": module["type"]}
if module["type"].startswith("pipe:"):
pymodule_name = "%(module_type)s" % {"module_type": util.pythonise(module["type"])}
pymodule_generator_name = "%(module_type)s" % {"module_type": util.pythonise(module["type"])}
indent = ""
if module_id in pipe["embed"]:
# We need to wrap submodules (used by loops) so we can pass the input at runtime (as we can to subpipelines)
pypipe += (
""" def pipe_%(module_id)s(context, _INPUT, conf=None, **kwargs):\n"""
""" "Submodule"\n""" % {"module_id": module_id} # todo insert submodule description here
)
indent = " "
pypipe += (
"""%(indent)s %(module_id)s = %(pymodule_name)s.%(pymodule_generator_name)s(context, %(pargs)s)\n"""
% {
"indent": indent,
"module_id": module_id,
"pymodule_name": pymodule_name,
"pymodule_generator_name": pymodule_generator_name,
"pargs": ", ".join(pargs),
}
)
if module_id in pipe["embed"]:
pypipe += """ return %(module_id)s\n""" % {"module_id": module_id}
prev_module = module_id
# todo? if context.verbose:
# print "%s = %s.%s(%s)" %(module_id, pymodule_name, pymodule_generator_name, str(pargs))
pypipe += """ return %(module_id)s\n""" % {"module_id": prev_module}
pypipe += (
"""\n"""
"""if __name__ == "__main__":\n"""
""" context = Context()\n"""
""" p = %(pipename)s(context, None)\n"""
""" for i in p:\n"""
""" print i\n""" % {"pipename": pipe["name"]}
)
return pypipe
def build_pipe(context, pipe):
"""Convert a pipe into an executable Python pipeline
If context.describe_input then just return the input requirements instead of the pipeline
Note: any subpipes must be available to import as .py files
current namespace can become polluted by submodule wrapper definitions
"""
pyinput = []
module_sequence = topological_sort(pipe['graph'])
#First pass to find and import any required subpipelines and user inputs
#Note: assumes they have already been compiled to accessible .py files
for module_id in module_sequence:
module = pipe['modules'][module_id]
if module['type'].startswith('pipe:'):
__import__(util.pythonise(module['type']))
if module['conf'] and 'prompt' in module[
'conf'] and context.describe_input:
pyinput.append((module['conf']['position']['value'],
module['conf']['name']['value'],
module['conf']['prompt']['value'],
module['conf']['default']['type'],
module['conf']['default']['value']))
#Note: there seems to be no need to recursively collate inputs from subpipelines
if context.describe_input:
return sorted(pyinput)
steps = {}
steps["forever"] = pipeforever.pipe_forever(context, None, conf=None)
for module_id in module_sequence:
module = pipe['modules'][module_id]
#Plumb I/O
input_module = steps["forever"]
for wire in pipe['wires']:
if util.pythonise(pipe['wires'][wire]['tgt']['moduleid']
) == module_id and pipe['wires'][wire]['tgt'][
'id'] == '_INPUT' and pipe['wires'][wire][
'src']['id'].startswith('_OUTPUT'):
input_module = steps[util.pythonise(
pipe['wires'][wire]['src']['moduleid'])]
if module_id in pipe['embed']:
assert input_module == steps[
"forever"], "input_module of an embedded module was already set"
input_module = "_INPUT"
pargs = [
context,
input_module,
]
kargs = {
"conf": module['conf'],
}
for wire in pipe['wires']:
if util.pythonise(pipe['wires'][wire]['tgt']['moduleid']
) == module_id and pipe['wires'][wire]['tgt'][
'id'] != '_INPUT' and pipe['wires'][wire][
'src']['id'].startswith('_OUTPUT'):
kargs["%(id)s" % {
'id': util.pythonise(pipe['wires'][wire]['tgt']['id'])
}] = steps[util.pythonise(
pipe['wires'][wire]['src']['moduleid'])]
if module['type'] == 'loop':
kargs["embed"] = steps[util.pythonise(
module['conf']['embed']['value']['id'])]
if module['type'] == 'split':
kargs["splits"] = len([
1 for w in pipe['wires'] if util.pythonise(
pipe['wires'][w]['src']['moduleid']) == module_id
])
#todo (re)import other pipes dynamically
pymodule_name = "pipe%(module_type)s" % {'module_type': module['type']}
pymodule_generator_name = "pipe_%(module_type)s" % {
'module_type': module['type']
}
if module['type'].startswith('pipe:'):
pymodule_name = "sys.modules['%(module_type)s']" % {
'module_type': util.pythonise(module['type'])
}
pymodule_generator_name = "%(module_type)s" % {
'module_type': util.pythonise(module['type'])
}
if module_id in pipe['embed']:
#We need to wrap submodules (used by loops) so we can pass the input at runtime (as we can to subpipelines)
pypipe = (
"""def pipe_%(module_id)s(context, _INPUT, conf=None, **kwargs):\n"""
""" return %(pymodule_name)s.%(pymodule_generator_name)s(context, _INPUT, conf=%(conf)s, **kwargs)\n"""
% {
'module_id': module_id,
'pymodule_name': pymodule_name,
'pymodule_generator_name': pymodule_generator_name,
'conf': module['conf'],
#Note: no embed (so no subloops) or wire kargs are passed and outer kwargs are passed in
})
exec pypipe #Note: evaluated in current namespace - todo ok?
steps[module_id] = eval("pipe_%(module_id)s" %
{'module_id': module_id})
else:
module_ref = eval(
"%(pymodule_name)s.%(pymodule_generator_name)s" % {
'pymodule_name': pymodule_name,
'pymodule_generator_name': pymodule_generator_name,
})
steps[module_id] = module_ref(*pargs, **kargs)
if context.verbose:
print "%s (%s) = %s(%s)" % (steps[module_id], module_id,
module_ref, str(pargs))
return steps[module_id]
def write_pipe(context, pipe):
"""Convert a pipe into Python script
If context.describe_input is passed to the script then it just returns the input requirements instead of the pipeline
"""
pypipe = ("""#Pipe %(pipename)s generated by pipe2py\n"""
"""\n"""
"""from pipe2py import Context\n"""
"""from pipe2py.modules import *\n"""
"""\n""" % {
'pipename': pipe['name']
})
pyinput = []
module_sequence = topological_sort(pipe['graph'])
#First pass to find any required subpipelines and user inputs
for module_id in module_sequence:
module = pipe['modules'][module_id]
if module['type'].startswith('pipe:'):
pypipe += """import %(module_type)s\n""" % {
'module_type': util.pythonise(module['type'])
}
if module['conf'] and 'prompt' in module['conf']:
pyinput.append((module['conf']['position']['value'],
module['conf']['name']['value'],
module['conf']['prompt']['value'],
module['conf']['default']['type'],
module['conf']['default']['value']))
#Note: there seems to be no need to recursively collate inputs from subpipelines
pypipe += (
"""\n"""
"""def %(pipename)s(context, _INPUT, conf=None, **kwargs):\n"""
""" "Pipeline"\n""" #todo insert pipeline description here
""" if conf is None:\n"""
""" conf = {}\n"""
"""\n"""
""" if context.describe_input:\n"""
""" return %(inputs)s\n"""
"""\n"""
""" forever = pipeforever.pipe_forever(context, None, conf=None)\n"""
"""\n""" % {
'pipename': pipe['name'],
'inputs': unicode(sorted(pyinput))
} #todo pprint this
)
prev_module = []
for module_id in module_sequence:
module = pipe['modules'][module_id]
#Plumb I/O
input_module = "forever"
for wire in pipe['wires']:
if util.pythonise(pipe['wires'][wire]['tgt']['moduleid']
) == module_id and pipe['wires'][wire]['tgt'][
'id'] == '_INPUT' and pipe['wires'][wire][
'src']['id'].startswith('_OUTPUT'):
input_module = util.pythonise(
pipe['wires'][wire]['src']['moduleid'])
if module_id in pipe['embed']:
assert input_module == "forever", "input_module of an embedded module was already set"
input_module = "_INPUT"
pargs = [
"%(input_module)s" % {
'input_module': input_module
},
"conf=%(conf)s" % {
'conf': module['conf']
}, #todo pprint this
]
for wire in pipe['wires']:
if util.pythonise(pipe['wires'][wire]['tgt']['moduleid']
) == module_id and pipe['wires'][wire]['tgt'][
'id'] != '_INPUT' and pipe['wires'][wire][
'src']['id'].startswith('_OUTPUT'):
pargs.append(
"%(id)s = %(secondary_module)s" % {
'id':
util.pythonise(pipe['wires'][wire]['tgt']['id']),
'secondary_module':
util.pythonise(pipe['wires'][wire]['src']['moduleid'])
})
if module['type'] == 'loop':
pargs.append(
"embed = pipe_%(embed_module)s" % {
'embed_module':
util.pythonise(module['conf']['embed']['value']['id'])
})
if module['type'] == 'split':
pargs.append(
"splits = %(splits)s" % {
'splits':
len([
1 for w in pipe['wires'] if util.pythonise(
pipe['wires'][w]['src']['moduleid']) == module_id
])
})
pymodule_name = "pipe%(module_type)s" % {'module_type': module['type']}
pymodule_generator_name = "pipe_%(module_type)s" % {
'module_type': module['type']
}
if module['type'].startswith('pipe:'):
pymodule_name = "%(module_type)s" % {
'module_type': util.pythonise(module['type'])
}
pymodule_generator_name = "%(module_type)s" % {
'module_type': util.pythonise(module['type'])
}
indent = ""
if module_id in pipe['embed']:
#We need to wrap submodules (used by loops) so we can pass the input at runtime (as we can to subpipelines)
pypipe += (
""" def pipe_%(module_id)s(context, _INPUT, conf=None, **kwargs):\n"""
""" "Submodule"\n""" #todo insert submodule description here
% {
'module_id': module_id
})
indent = " "
pypipe += """%(indent)s %(module_id)s = %(pymodule_name)s.%(pymodule_generator_name)s(context, %(pargs)s)\n""" % {
'indent': indent,
'module_id': module_id,
'pymodule_name': pymodule_name,
'pymodule_generator_name': pymodule_generator_name,
'pargs': ", ".join(pargs)
}
if module_id in pipe['embed']:
pypipe += """ return %(module_id)s\n""" % {
'module_id': module_id
}
prev_module = module_id
#todo? if context.verbose:
# print "%s = %s.%s(%s)" %(module_id, pymodule_name, pymodule_generator_name, str(pargs))
pypipe += """ return %(module_id)s\n""" % {'module_id': prev_module}
pypipe += ("""\n"""
"""if __name__ == "__main__":\n"""
""" context = Context()\n"""
""" p = %(pipename)s(context, None)\n"""
""" for i in p:\n"""
""" print i\n""" % {
'pipename': pipe['name']
})
return pypipe
def _parse_pipe(json_pipe, pipe_name="anonymous"):
"""Parse pipe JSON into internal structures
Keyword arguments:
json_pipe -- JSON representation of the pipe
pipe_name -- a name for the pipe (used for linking pipes)
Returns:
pipe -- an internal representation of a pipe
"""
pipe = {'name': util.pythonise(pipe_name)}
pipe['modules'] = {}
pipe['embed'] = {}
pipe['graph'] = {}
pipe['wires'] = {}
modules = json_pipe['modules']
if not isinstance(modules, list):
modules = [modules]
for module in modules:
pipe['modules'][util.pythonise(module['id'])] = module
pipe['graph'][util.pythonise(module['id'])] = []
if module['type'] == 'loop':
embed = module['conf']['embed']['value']
pipe['modules'][util.pythonise(embed['id'])] = embed
pipe['graph'][util.pythonise(embed['id'])] = []
pipe['embed'][util.pythonise(embed['id'])] = embed
#make the loop dependent on its embedded module
pipe['graph'][util.pythonise(embed['id'])].append(util.pythonise(module['id']))
wires = json_pipe['wires']
if not isinstance(wires, list):
wires = [wires]
for wire in wires:
pipe['graph'][util.pythonise(wire['src']['moduleid'])].append(util.pythonise(wire['tgt']['moduleid']))
#Remove any orphan nodes
for node in pipe['graph'].keys():
targetted = [node in pipe['graph'][k] for k in pipe['graph']]
if not pipe['graph'][node] and not any(targetted):
del pipe['graph'][node]
for wire in wires:
pipe['wires'][util.pythonise(wire['id'])] = wire
return pipe
def write_pipe(context, pipe):
"""Convert a pipe into Python script
If context.describe_input is passed to the script then it just returns the input requirements instead of the pipeline
"""
pypipe = ("""#Pipe %(pipename)s generated by pipe2py\n"""
"""\n"""
"""from pipe2py import Context\n"""
"""from pipe2py.modules import *\n"""
"""\n""" % {'pipename':pipe['name']}
)
pyinput = []
module_sequence = topological_sort(pipe['graph'])
#First pass to find any required subpipelines and user inputs
for module_id in module_sequence:
module = pipe['modules'][module_id]
if module['type'].startswith('pipe:'):
pypipe += """import %(module_type)s\n""" % {'module_type':util.pythonise(module['type'])}
if module['conf'] and 'prompt' in module['conf']:
pyinput.append((module['conf']['position']['value'],
module['conf']['name']['value'],
module['conf']['prompt']['value'],
module['conf']['default']['type'],
module['conf']['default']['value']))
#Note: there seems to be no need to recursively collate inputs from subpipelines
pypipe += ("""\n"""
"""def %(pipename)s(context, _INPUT, conf=None, **kwargs):\n"""
""" "Pipeline"\n""" #todo insert pipeline description here
""" if conf is None:\n"""
""" conf = {}\n"""
"""\n"""
""" if context.describe_input:\n"""
""" return %(inputs)s\n"""
"""\n"""
""" forever = pipeforever.pipe_forever(context, None, conf=None)\n"""
"""\n""" % {'pipename':pipe['name'],
'inputs':unicode(sorted(pyinput))} #todo pprint this
)
prev_module = []
for module_id in module_sequence:
module = pipe['modules'][module_id]
#Plumb I/O
input_module = "forever"
for wire in pipe['wires']:
if util.pythonise(pipe['wires'][wire]['tgt']['moduleid']) == module_id and pipe['wires'][wire]['tgt']['id'] == '_INPUT' and pipe['wires'][wire]['src']['id'].startswith('_OUTPUT'):
input_module = util.pythonise(pipe['wires'][wire]['src']['moduleid'])
if module_id in pipe['embed']:
assert input_module == "forever", "input_module of an embedded module was already set"
input_module = "_INPUT"
pargs = ["%(input_module)s" % {'input_module':input_module},
"conf=%(conf)s" % {'conf':module['conf']}, #todo pprint this
]
for wire in pipe['wires']:
if util.pythonise(pipe['wires'][wire]['tgt']['moduleid']) == module_id and pipe['wires'][wire]['tgt']['id'] != '_INPUT' and pipe['wires'][wire]['src']['id'].startswith('_OUTPUT'):
pargs.append("%(id)s = %(secondary_module)s" % {'id':util.pythonise(pipe['wires'][wire]['tgt']['id']), 'secondary_module':util.pythonise(pipe['wires'][wire]['src']['moduleid'])})
if module['type'] == 'loop':
pargs.append("embed = pipe_%(embed_module)s" % {'embed_module':util.pythonise(module['conf']['embed']['value']['id'])})
if module['type'] == 'split':
pargs.append("splits = %(splits)s" % {'splits':len([1 for w in pipe['wires'] if util.pythonise(pipe['wires'][w]['src']['moduleid']) == module_id])})
pymodule_name = "pipe%(module_type)s" % {'module_type':module['type']}
pymodule_generator_name = "pipe_%(module_type)s" % {'module_type':module['type']}
if module['type'].startswith('pipe:'):
pymodule_name = "%(module_type)s" % {'module_type':util.pythonise(module['type'])}
pymodule_generator_name = "%(module_type)s" % {'module_type':util.pythonise(module['type'])}
indent = ""
if module_id in pipe['embed']:
#We need to wrap submodules (used by loops) so we can pass the input at runtime (as we can to subpipelines)
pypipe += (""" def pipe_%(module_id)s(context, _INPUT, conf=None, **kwargs):\n"""
""" "Submodule"\n""" #todo insert submodule description here
% {'module_id':module_id}
)
indent = " "
pypipe += """%(indent)s %(module_id)s = %(pymodule_name)s.%(pymodule_generator_name)s(context, %(pargs)s)\n""" % {
'indent':indent,
'module_id':module_id,
'pymodule_name':pymodule_name,
'pymodule_generator_name':pymodule_generator_name,
'pargs':", ".join(pargs)}
if module_id in pipe['embed']:
pypipe += """ return %(module_id)s\n""" % {'module_id':module_id}
prev_module = module_id
#todo? if context.verbose:
# print "%s = %s.%s(%s)" %(module_id, pymodule_name, pymodule_generator_name, str(pargs))
pypipe += """ return %(module_id)s\n""" % {'module_id':prev_module}
pypipe += ("""\n"""
"""if __name__ == "__main__":\n"""
""" context = Context()\n"""
""" p = %(pipename)s(context, None)\n"""
""" for i in p:\n"""
""" print i\n""" % {'pipename':pipe['name']}
)
return pypipe
def _pipe_commons(context, pipe, module_id, pyinput=None, steps=None):
pyinput = pyinput or []
module = pipe['modules'][module_id]
module_type = module['type']
conf = module['conf']
kwargs = {'conf': conf}
output = None
if module_type.startswith('pipe:'):
# Import any required sub-pipelines and user inputs
# Note: assumes they have already been compiled to accessible .py files
import_module(util.pythonise(module_type)) if steps else None
pythonised_type = util.pythonise(module_type)
pymodule_name = '%s' % pythonised_type
pymodule_generator = '%s' % pythonised_type
else:
pymodule_name = 'pipe%s' % module_type
pymodule_generator = 'pipe_%s' % module_type
if context.describe_input or not steps:
# Find any required subpipelines and user inputs
if conf and 'prompt' in conf:
# Note: there seems to be no need to recursively collate inputs
# from subpipelines
module_confs = (
module['conf']['position']['value'],
module['conf']['name']['value'],
module['conf']['prompt']['value'],
module['conf']['default']['type'],
module['conf']['default']['value']
)
pyinput.append(module_confs)
if steps:
output = {
'pyinput': pyinput,
'pymodule_name': pymodule_name,
'pymodule_generator': pymodule_generator,
}
if not output:
# find the default input of this module
input_module = steps['forever'] if steps else 'forever'
for key, pipe_wire in pipe['wires'].items():
moduleid = util.pythonise(pipe_wire['src']['moduleid'])
# todo? this equates the outputs
is_default_in_and_out = (
util.pythonise(pipe_wire['tgt']['moduleid']) == module_id
and pipe_wire['tgt']['id'] == '_INPUT'
and pipe_wire['src']['id'].startswith('_OUTPUT')
)
# if the wire is to this module and it's the default input and it's
# the default output:
if is_default_in_and_out:
input_module = steps[moduleid] if steps else moduleid
# todo? this equates the outputs
is_default_out_only = (
util.pythonise(pipe_wire['tgt']['moduleid']) == module_id
and pipe_wire['tgt']['id'] != '_INPUT'
and pipe_wire['src']['id'].startswith('_OUTPUT')
)
# if the wire is to this module and it's *NOT* the default input
# and it's the default output
if is_default_out_only:
# set the extra inputs of this module as kwargs of this module
pipe_id = util.pythonise(pipe_wire['tgt']['id'])
updated = steps[moduleid] if steps else Id(moduleid)
kwargs.update({pipe_id: updated})
if module_id in pipe['embed']:
text = 'input_module of an embedded module was already set'
# what is this for???
# assert input_module == (steps['forever'], text)
input_module = '_INPUT'
args = [context, input_module] if steps else [
Id('context'), Id(input_module)]
# set the embedded module in the kwargs if this is loop module
if module_type == 'loop':
value = module['conf']['embed']['value']
pipe_id = util.pythonise(value['id'])
updated = steps[pipe_id] if steps else Id('pipe_%s' % pipe_id)
kwargs.update({'embed': updated})
# set splits in the kwargs if this is split module
if module_type == 'split':
filtered = filter(
lambda x: module_id == util.pythonise(x[1]['src']['moduleid']),
pipe['wires'].items()
)
count = len(filtered)
updated = count if steps else Id(count)
kwargs.update({'splits': updated})
output = {
'pyinput': pyinput,
'pymodule_name': pymodule_name,
'pymodule_generator': pymodule_generator,
'args': args,
'kwargs': kwargs,
}
return output
def build_pipe(context, pipe):
"""Convert a pipe into an executable Python pipeline
If context.describe_input then just return the input requirements instead of the pipeline
Note: any subpipes must be available to import as .py files
current namespace can become polluted by submodule wrapper definitions
"""
pyinput = []
module_sequence = topological_sort(pipe['graph'])
#First pass to find and import any required subpipelines and user inputs
#Note: assumes they have already been compiled to accessible .py files
for module_id in module_sequence:
module = pipe['modules'][module_id]
if module['type'].startswith('pipe:'):
__import__(util.pythonise(module['type']))
if 'prompt' in module['conf'] and context.describe_input:
pyinput.append((module['conf']['position']['value'],
module['conf']['name']['value'],
module['conf']['prompt']['value'],
module['conf']['default']['type'],
module['conf']['default']['value']))
#Note: there seems to be no need to recursively collate inputs from subpipelines
if context.describe_input:
return sorted(pyinput)
steps = {}
steps["forever"] = pipeforever.pipe_forever(context, None, conf=None)
for module_id in module_sequence:
module = pipe['modules'][module_id]
#Plumb I/O
# find the default input of this module
input_module = steps["forever"]
for wire in pipe['wires']:
# if the wire is to this module and it's the default input and it's the default output:
if util.pythonise(pipe['wires'][wire]['tgt']['moduleid']) == module_id and pipe['wires'][wire]['tgt']['id'] == '_INPUT' and pipe['wires'][wire]['src']['id'].startswith('_OUTPUT'): # todo? this equates the outputs
input_module = steps[util.pythonise(pipe['wires'][wire]['src']['moduleid'])]
if module_id in pipe['embed']:
assert input_module == steps["forever"], "input_module of an embedded module was already set"
input_module = "_INPUT"
pargs = [context,
input_module,
]
kargs = {"conf":module['conf'],
}
# set the extra inputs of this module as kargs of this module
for wire in pipe['wires']:
# if the wire is to this module and it's *not* the default input and it's the default output:
if util.pythonise(pipe['wires'][wire]['tgt']['moduleid']) == module_id and pipe['wires'][wire]['tgt']['id'] != '_INPUT' and pipe['wires'][wire]['src']['id'].startswith('_OUTPUT'): # todo? this equates the outputs
kargs["%(id)s" % {'id':util.pythonise(pipe['wires'][wire]['tgt']['id'])}] = steps[util.pythonise(pipe['wires'][wire]['src']['moduleid'])]
# set the embedded module in the kargs if this is loop module
if module['type'] == 'loop':
kargs["embed"] = steps[util.pythonise(module['conf']['embed']['value']['id'])]
#todo (re)import other pipes dynamically
pymodule_name = "pipe%(module_type)s" % {'module_type':module['type']}
pymodule_generator_name = "pipe_%(module_type)s" % {'module_type':module['type']}
if module['type'].startswith('pipe:'):
pymodule_name = "sys.modules['%(module_type)s']" % {'module_type':util.pythonise(module['type'])}
pymodule_generator_name = "%(module_type)s" % {'module_type':util.pythonise(module['type'])}
# if this module is an embedded module:
if module_id in pipe['embed']:
#We need to wrap submodules (used by loops) so we can pass the input at runtime (as we can to subpipelines)
pypipe = ("""def pipe_%(module_id)s(context, _INPUT, conf=None, **kwargs):\n"""
""" return %(pymodule_name)s.%(pymodule_generator_name)s(context, _INPUT, conf=%(conf)s, **kwargs)\n"""
% {'module_id':module_id,
'pymodule_name':pymodule_name,
'pymodule_generator_name':pymodule_generator_name,
'conf':module['conf'],
#Note: no embed (so no subloops) or wire kargs are passed and outer kwargs are passed in
}
)
exec pypipe #Note: evaluated in current namespace - todo ok?
steps[module_id] = eval("pipe_%(module_id)s" % {'module_id':module_id})
else: # else this module is not an embedded module:
module_ref = eval("%(pymodule_name)s.%(pymodule_generator_name)s" % {'pymodule_name':pymodule_name,
'pymodule_generator_name':pymodule_generator_name,})
steps[module_id] = module_ref(*pargs, **kargs)
if context.verbose:
print "%s (%s) = %s(%s)" %(steps[module_id], module_id, module_ref, str(pargs))
return steps[module_id]
def write_pipe(context, pipe):
"""Convert a pipe into Python script
If context.describe_input is passed to the script then it just returns the input requirements instead of the pipeline
"""
pypipe = ("""#Pipe %(pipename)s generated by pipe2py\n"""
"""\n"""
"""from pipe2py import Context\n"""
"""from pipe2py.modules import *\n"""
"""\n""" % {'pipename':pipe['name']}
)
pyinput = []
module_sequence = topological_sort(pipe['graph'])
#First pass to find any required subpipelines and user inputs
for module_id in module_sequence:
module = pipe['modules'][module_id]
if module['type'].startswith('pipe:'):
pypipe += """import %(module_type)s\n""" % {'module_type':util.pythonise(module['type'])}
if 'prompt' in module['conf']:
pyinput.append((module['conf']['position']['value'],
module['conf']['name']['value'],
module['conf']['prompt']['value'],
module['conf']['default']['type'],
module['conf']['default']['value']))
#Note: there seems to be no need to recursively collate inputs from subpipelines
pypipe += ("""\n"""
"""def %(pipename)s(context, _INPUT, conf=None, **kwargs):\n"""
""" "Pipeline"\n""" #todo insert pipeline description here
""" if conf is None:\n"""
""" conf = {}\n"""
"""\n"""
""" if context.describe_input:\n"""
""" return %(inputs)s\n"""
"""\n"""
""" forever = pipeforever.pipe_forever(context, None, conf=None)\n"""
"""\n""" % {'pipename':pipe['name'],
'inputs':unicode(sorted(pyinput))} #todo pprint this
)
prev_module = []
for module_id in module_sequence:
module = pipe['modules'][module_id]
#Plumb I/O
# find the default input of this module
input_module = "forever"
for wire in pipe['wires']:
# if the wire is to this module and it's the default input and it's the default output:
if util.pythonise(pipe['wires'][wire]['tgt']['moduleid']) == module_id and pipe['wires'][wire]['tgt']['id'] == '_INPUT' and pipe['wires'][wire]['src']['id'].startswith('_OUTPUT'): # todo? this equates the outputs
input_module = util.pythonise(pipe['wires'][wire]['src']['moduleid'])
if module_id in pipe['embed']:
assert input_module == "forever", "input_module of an embedded module was already set"
input_module = "_INPUT"
mod_args = [Id('context'), Id(input_module)]
mod_kwargs = [('conf', module['conf'])]
# set the extra inputs of this module as kwargs of this module
for wire in pipe['wires']:
# if the wire is to this module and it's *not* the default input and it's the default output:
if util.pythonise(pipe['wires'][wire]['tgt']['moduleid']) == module_id and pipe['wires'][wire]['tgt']['id'] != '_INPUT' and pipe['wires'][wire]['src']['id'].startswith('_OUTPUT'): # todo? this equates the outputs
mod_kwargs += [(util.pythonise(pipe['wires'][wire]['tgt']['id']), Id(util.pythonise(pipe['wires'][wire]['src']['moduleid'])))]
# set the embedded module in the kwargs if this is loop module
if module['type'] == 'loop':
mod_kwargs += [("embed", Id("pipe_%s" % util.pythonise(module['conf']['embed']['value']['id'])))]
pymodule_name = "pipe%(module_type)s" % {'module_type':module['type']}
pymodule_generator_name = "pipe_%(module_type)s" % {'module_type':module['type']}
if module['type'].startswith('pipe:'):
pymodule_name = "%(module_type)s" % {'module_type':util.pythonise(module['type'])}
pymodule_generator_name = "%(module_type)s" % {'module_type':util.pythonise(module['type'])}
indent = ""
if module_id in pipe['embed']:
#We need to wrap submodules (used by loops) so we can pass the input at runtime (as we can to subpipelines)
pypipe += (""" def pipe_%(module_id)s(context, _INPUT, conf=None, **kwargs):\n"""
""" "Submodule"\n""" #todo insert submodule description here
% {'module_id':module_id}
)
indent = " "
pypipe += """%(indent)s %(module_id)s = %(pymodule_name)s.%(pymodule_generator_name)s(%(pargs)s)\n""" % {
'indent':indent,
'module_id':module_id,
'pymodule_name':pymodule_name,
'pymodule_generator_name':pymodule_generator_name,
'pargs':repr_args(mod_args+mod_kwargs)}
if module_id in pipe['embed']:
pypipe += """ return %(module_id)s\n""" % {'module_id':module_id}
prev_module = module_id
if context.verbose:
print ("%s = %s(%s)" % (module_id, pymodule_generator_name,
str_args([arg for arg in mod_args if arg != Id('context')]+
[(key, value) for key, value in mod_kwargs if key != 'conf']+
[(key, value) for key, value in mod_kwargs if key == 'conf']))
).encode("utf-8")
pypipe += """ return %(module_id)s\n""" % {'module_id':prev_module}
pypipe += ("""\n"""
"""if __name__ == "__main__":\n"""
""" context = Context()\n"""
""" p = %(pipename)s(context, None)\n"""
""" for i in p:\n"""
""" print i\n""" % {'pipename':pipe['name']}
)
return pypipe
