def write_pipe(context, pipe):
"""Convert a pipe into Python script
If context.describe_input is passed to the script then it just returns the input requirements instead of the pipeline
"""
pypipe = (
"""#Pipe %(pipename)s generated by pipe2py\n"""
"""\n"""
"""from pipe2py import Context\n"""
"""from pipe2py.modules import *\n"""
"""\n""" % {"pipename": pipe["name"]}
)
pyinput = []
module_sequence = topological_sort(pipe["graph"])
# First pass to find any required subpipelines and user inputs
for module_id in module_sequence:
module = pipe["modules"][module_id]
if module["type"].startswith("pipe:"):
pypipe += """import %(module_type)s\n""" % {"module_type": util.pythonise(module["type"])}
if module["conf"] and "prompt" in module["conf"]:
pyinput.append(
(
module["conf"]["position"]["value"],
module["conf"]["name"]["value"],
module["conf"]["prompt"]["value"],
module["conf"]["default"]["type"],
module["conf"]["default"]["value"],
)
)
# Note: there seems to be no need to recursively collate inputs from subpipelines
pypipe += (
"""\n"""
"""def %(pipename)s(context, _INPUT, conf=None, **kwargs):\n"""
""" "Pipeline"\n""" # todo insert pipeline description here
""" if conf is None:\n"""
""" conf = {}\n"""
"""\n"""
""" if context.describe_input:\n"""
""" return %(inputs)s\n"""
"""\n"""
""" forever = pipeforever.pipe_forever(context, None, conf=None)\n"""
"""\n""" % {"pipename": pipe["name"], "inputs": unicode(sorted(pyinput))} # todo pprint this
)
prev_module = []
for module_id in module_sequence:
module = pipe["modules"][module_id]
# Plumb I/O
input_module = "forever"
for wire in pipe["wires"]:
if (
util.pythonise(pipe["wires"][wire]["tgt"]["moduleid"]) == module_id
and pipe["wires"][wire]["tgt"]["id"] == "_INPUT"
and pipe["wires"][wire]["src"]["id"].startswith("_OUTPUT")
):
input_module = util.pythonise(pipe["wires"][wire]["src"]["moduleid"])
if module_id in pipe["embed"]:
assert input_module == "forever", "input_module of an embedded module was already set"
input_module = "_INPUT"
pargs = [
"%(input_module)s" % {"input_module": input_module},
"conf=%(conf)s" % {"conf": module["conf"]}, # todo pprint this
]
for wire in pipe["wires"]:
if (
util.pythonise(pipe["wires"][wire]["tgt"]["moduleid"]) == module_id
and pipe["wires"][wire]["tgt"]["id"] != "_INPUT"
and pipe["wires"][wire]["src"]["id"].startswith("_OUTPUT")
):
pargs.append(
"%(id)s = %(secondary_module)s"
% {
"id": util.pythonise(pipe["wires"][wire]["tgt"]["id"]),
"secondary_module": util.pythonise(pipe["wires"][wire]["src"]["moduleid"]),
}
)
if module["type"] == "loop":
pargs.append(
"embed = pipe_%(embed_module)s"
% {"embed_module": util.pythonise(module["conf"]["embed"]["value"]["id"])}
)
if module["type"] == "split":
pargs.append(
"splits = %(splits)s"
% {
"splits": len(
[1 for w in pipe["wires"] if util.pythonise(pipe["wires"][w]["src"]["moduleid"]) == module_id]
)
}
)
pymodule_name = "pipe%(module_type)s" % {"module_type": module["type"]}
pymodule_generator_name = "pipe_%(module_type)s" % {"module_type": module["type"]}
if module["type"].startswith("pipe:"):
pymodule_name = "%(module_type)s" % {"module_type": util.pythonise(module["type"])}
pymodule_generator_name = "%(module_type)s" % {"module_type": util.pythonise(module["type"])}
indent = ""
if module_id in pipe["embed"]:
# We need to wrap submodules (used by loops) so we can pass the input at runtime (as we can to subpipelines)
pypipe += (
""" def pipe_%(module_id)s(context, _INPUT, conf=None, **kwargs):\n"""
""" "Submodule"\n""" % {"module_id": module_id} # todo insert submodule description here
)
indent = " "
pypipe += (
"""%(indent)s %(module_id)s = %(pymodule_name)s.%(pymodule_generator_name)s(context, %(pargs)s)\n"""
% {
"indent": indent,
"module_id": module_id,
"pymodule_name": pymodule_name,
"pymodule_generator_name": pymodule_generator_name,
"pargs": ", ".join(pargs),
}
)
if module_id in pipe["embed"]:
pypipe += """ return %(module_id)s\n""" % {"module_id": module_id}
prev_module = module_id
# todo? if context.verbose:
# print "%s = %s.%s(%s)" %(module_id, pymodule_name, pymodule_generator_name, str(pargs))
pypipe += """ return %(module_id)s\n""" % {"module_id": prev_module}
pypipe += (
"""\n"""
"""if __name__ == "__main__":\n"""
""" context = Context()\n"""
""" p = %(pipename)s(context, None)\n"""
""" for i in p:\n"""
""" print i\n""" % {"pipename": pipe["name"]}
)
return pypipe
def build_pipe(context, pipe):
"""Convert a pipe into an executable Python pipeline
If context.describe_input then just return the input requirements instead of the pipeline
Note: any subpipes must be available to import as .py files
current namespace can become polluted by submodule wrapper definitions
"""
pyinput = []
module_sequence = topological_sort(pipe["graph"])
# First pass to find and import any required subpipelines and user inputs
# Note: assumes they have already been compiled to accessible .py files
for module_id in module_sequence:
module = pipe["modules"][module_id]
if module["type"].startswith("pipe:"):
__import__(util.pythonise(module["type"]))
if module["conf"] and "prompt" in module["conf"] and context.describe_input:
pyinput.append(
(
module["conf"]["position"]["value"],
module["conf"]["name"]["value"],
module["conf"]["prompt"]["value"],
module["conf"]["default"]["type"],
module["conf"]["default"]["value"],
)
)
# Note: there seems to be no need to recursively collate inputs from subpipelines
if context.describe_input:
return sorted(pyinput)
steps = {}
steps["forever"] = pipeforever.pipe_forever(context, None, conf=None)
for module_id in module_sequence:
module = pipe["modules"][module_id]
# Plumb I/O
input_module = steps["forever"]
for wire in pipe["wires"]:
if (
util.pythonise(pipe["wires"][wire]["tgt"]["moduleid"]) == module_id
and pipe["wires"][wire]["tgt"]["id"] == "_INPUT"
and pipe["wires"][wire]["src"]["id"].startswith("_OUTPUT")
):
input_module = steps[util.pythonise(pipe["wires"][wire]["src"]["moduleid"])]
if module_id in pipe["embed"]:
assert input_module == steps["forever"], "input_module of an embedded module was already set"
input_module = "_INPUT"
pargs = [context, input_module]
kargs = {"conf": module["conf"]}
for wire in pipe["wires"]:
if (
util.pythonise(pipe["wires"][wire]["tgt"]["moduleid"]) == module_id
and pipe["wires"][wire]["tgt"]["id"] != "_INPUT"
and pipe["wires"][wire]["src"]["id"].startswith("_OUTPUT")
):
kargs["%(id)s" % {"id": util.pythonise(pipe["wires"][wire]["tgt"]["id"])}] = steps[
util.pythonise(pipe["wires"][wire]["src"]["moduleid"])
]
if module["type"] == "loop":
kargs["embed"] = steps[util.pythonise(module["conf"]["embed"]["value"]["id"])]
if module["type"] == "split":
kargs["splits"] = len(
[1 for w in pipe["wires"] if util.pythonise(pipe["wires"][w]["src"]["moduleid"]) == module_id]
)
# todo (re)import other pipes dynamically
pymodule_name = "pipe%(module_type)s" % {"module_type": module["type"]}
pymodule_generator_name = "pipe_%(module_type)s" % {"module_type": module["type"]}
if module["type"].startswith("pipe:"):
pymodule_name = "sys.modules['%(module_type)s']" % {"module_type": util.pythonise(module["type"])}
pymodule_generator_name = "%(module_type)s" % {"module_type": util.pythonise(module["type"])}
if module_id in pipe["embed"]:
# We need to wrap submodules (used by loops) so we can pass the input at runtime (as we can to subpipelines)
pypipe = (
"""def pipe_%(module_id)s(context, _INPUT, conf=None, **kwargs):\n"""
""" return %(pymodule_name)s.%(pymodule_generator_name)s(context, _INPUT, conf=%(conf)s, **kwargs)\n"""
% {
"module_id": module_id,
"pymodule_name": pymodule_name,
"pymodule_generator_name": pymodule_generator_name,
"conf": module["conf"],
# Note: no embed (so no subloops) or wire kargs are passed and outer kwargs are passed in
}
)
exec pypipe # Note: evaluated in current namespace - todo ok?
steps[module_id] = eval("pipe_%(module_id)s" % {"module_id": module_id})
else:
module_ref = eval(
"%(pymodule_name)s.%(pymodule_generator_name)s"
% {"pymodule_name": pymodule_name, "pymodule_generator_name": pymodule_generator_name}
)
steps[module_id] = module_ref(*pargs, **kargs)
if context.verbose:
print "%s (%s) = %s(%s)" % (steps[module_id], module_id, module_ref, str(pargs))
return steps[module_id]
def write_pipe(context, pipe):
"""Convert a pipe into Python script
If context.describe_input is passed to the script then it just returns the input requirements instead of the pipeline
"""
pypipe = ("""#Pipe %(pipename)s generated by pipe2py\n"""
"""\n"""
"""from pipe2py import Context\n"""
"""from pipe2py.modules import *\n"""
"""\n""" % {
'pipename': pipe['name']
})
pyinput = []
module_sequence = topological_sort(pipe['graph'])
#First pass to find any required subpipelines and user inputs
for module_id in module_sequence:
module = pipe['modules'][module_id]
if module['type'].startswith('pipe:'):
pypipe += """import %(module_type)s\n""" % {
'module_type': util.pythonise(module['type'])
}
if module['conf'] and 'prompt' in module['conf']:
pyinput.append((module['conf']['position']['value'],
module['conf']['name']['value'],
module['conf']['prompt']['value'],
module['conf']['default']['type'],
module['conf']['default']['value']))
#Note: there seems to be no need to recursively collate inputs from subpipelines
pypipe += (
"""\n"""
"""def %(pipename)s(context, _INPUT, conf=None, **kwargs):\n"""
""" "Pipeline"\n""" #todo insert pipeline description here
""" if conf is None:\n"""
""" conf = {}\n"""
"""\n"""
""" if context.describe_input:\n"""
""" return %(inputs)s\n"""
"""\n"""
""" forever = pipeforever.pipe_forever(context, None, conf=None)\n"""
"""\n""" % {
'pipename': pipe['name'],
'inputs': unicode(sorted(pyinput))
} #todo pprint this
)
prev_module = []
for module_id in module_sequence:
module = pipe['modules'][module_id]
#Plumb I/O
input_module = "forever"
for wire in pipe['wires']:
if util.pythonise(pipe['wires'][wire]['tgt']['moduleid']
) == module_id and pipe['wires'][wire]['tgt'][
'id'] == '_INPUT' and pipe['wires'][wire][
'src']['id'].startswith('_OUTPUT'):
input_module = util.pythonise(
pipe['wires'][wire]['src']['moduleid'])
if module_id in pipe['embed']:
assert input_module == "forever", "input_module of an embedded module was already set"
input_module = "_INPUT"
pargs = [
"%(input_module)s" % {
'input_module': input_module
},
"conf=%(conf)s" % {
'conf': module['conf']
}, #todo pprint this
]
for wire in pipe['wires']:
if util.pythonise(pipe['wires'][wire]['tgt']['moduleid']
) == module_id and pipe['wires'][wire]['tgt'][
'id'] != '_INPUT' and pipe['wires'][wire][
'src']['id'].startswith('_OUTPUT'):
pargs.append(
"%(id)s = %(secondary_module)s" % {
'id':
util.pythonise(pipe['wires'][wire]['tgt']['id']),
'secondary_module':
util.pythonise(pipe['wires'][wire]['src']['moduleid'])
})
if module['type'] == 'loop':
pargs.append(
"embed = pipe_%(embed_module)s" % {
'embed_module':
util.pythonise(module['conf']['embed']['value']['id'])
})
if module['type'] == 'split':
pargs.append(
"splits = %(splits)s" % {
'splits':
len([
1 for w in pipe['wires'] if util.pythonise(
pipe['wires'][w]['src']['moduleid']) == module_id
])
})
pymodule_name = "pipe%(module_type)s" % {'module_type': module['type']}
pymodule_generator_name = "pipe_%(module_type)s" % {
'module_type': module['type']
}
if module['type'].startswith('pipe:'):
pymodule_name = "%(module_type)s" % {
'module_type': util.pythonise(module['type'])
}
pymodule_generator_name = "%(module_type)s" % {
'module_type': util.pythonise(module['type'])
}
indent = ""
if module_id in pipe['embed']:
#We need to wrap submodules (used by loops) so we can pass the input at runtime (as we can to subpipelines)
pypipe += (
""" def pipe_%(module_id)s(context, _INPUT, conf=None, **kwargs):\n"""
""" "Submodule"\n""" #todo insert submodule description here
% {
'module_id': module_id
})
indent = " "
pypipe += """%(indent)s %(module_id)s = %(pymodule_name)s.%(pymodule_generator_name)s(context, %(pargs)s)\n""" % {
'indent': indent,
'module_id': module_id,
'pymodule_name': pymodule_name,
'pymodule_generator_name': pymodule_generator_name,
'pargs': ", ".join(pargs)
}
if module_id in pipe['embed']:
pypipe += """ return %(module_id)s\n""" % {
'module_id': module_id
}
prev_module = module_id
#todo? if context.verbose:
# print "%s = %s.%s(%s)" %(module_id, pymodule_name, pymodule_generator_name, str(pargs))
pypipe += """ return %(module_id)s\n""" % {'module_id': prev_module}
pypipe += ("""\n"""
"""if __name__ == "__main__":\n"""
""" context = Context()\n"""
""" p = %(pipename)s(context, None)\n"""
""" for i in p:\n"""
""" print i\n""" % {
'pipename': pipe['name']
})
return pypipe
def build_pipe(context, pipe):
"""Convert a pipe into an executable Python pipeline
If context.describe_input then just return the input requirements instead of the pipeline
Note: any subpipes must be available to import as .py files
current namespace can become polluted by submodule wrapper definitions
"""
pyinput = []
module_sequence = topological_sort(pipe['graph'])
#First pass to find and import any required subpipelines and user inputs
#Note: assumes they have already been compiled to accessible .py files
for module_id in module_sequence:
module = pipe['modules'][module_id]
if module['type'].startswith('pipe:'):
__import__(util.pythonise(module['type']))
if module['conf'] and 'prompt' in module[
'conf'] and context.describe_input:
pyinput.append((module['conf']['position']['value'],
module['conf']['name']['value'],
module['conf']['prompt']['value'],
module['conf']['default']['type'],
module['conf']['default']['value']))
#Note: there seems to be no need to recursively collate inputs from subpipelines
if context.describe_input:
return sorted(pyinput)
steps = {}
steps["forever"] = pipeforever.pipe_forever(context, None, conf=None)
for module_id in module_sequence:
module = pipe['modules'][module_id]
#Plumb I/O
input_module = steps["forever"]
for wire in pipe['wires']:
if util.pythonise(pipe['wires'][wire]['tgt']['moduleid']
) == module_id and pipe['wires'][wire]['tgt'][
'id'] == '_INPUT' and pipe['wires'][wire][
'src']['id'].startswith('_OUTPUT'):
input_module = steps[util.pythonise(
pipe['wires'][wire]['src']['moduleid'])]
if module_id in pipe['embed']:
assert input_module == steps[
"forever"], "input_module of an embedded module was already set"
input_module = "_INPUT"
pargs = [
context,
input_module,
]
kargs = {
"conf": module['conf'],
}
for wire in pipe['wires']:
if util.pythonise(pipe['wires'][wire]['tgt']['moduleid']
) == module_id and pipe['wires'][wire]['tgt'][
'id'] != '_INPUT' and pipe['wires'][wire][
'src']['id'].startswith('_OUTPUT'):
kargs["%(id)s" % {
'id': util.pythonise(pipe['wires'][wire]['tgt']['id'])
}] = steps[util.pythonise(
pipe['wires'][wire]['src']['moduleid'])]
if module['type'] == 'loop':
kargs["embed"] = steps[util.pythonise(
module['conf']['embed']['value']['id'])]
if module['type'] == 'split':
kargs["splits"] = len([
1 for w in pipe['wires'] if util.pythonise(
pipe['wires'][w]['src']['moduleid']) == module_id
])
#todo (re)import other pipes dynamically
pymodule_name = "pipe%(module_type)s" % {'module_type': module['type']}
pymodule_generator_name = "pipe_%(module_type)s" % {
'module_type': module['type']
}
if module['type'].startswith('pipe:'):
pymodule_name = "sys.modules['%(module_type)s']" % {
'module_type': util.pythonise(module['type'])
}
pymodule_generator_name = "%(module_type)s" % {
'module_type': util.pythonise(module['type'])
}
if module_id in pipe['embed']:
#We need to wrap submodules (used by loops) so we can pass the input at runtime (as we can to subpipelines)
pypipe = (
"""def pipe_%(module_id)s(context, _INPUT, conf=None, **kwargs):\n"""
""" return %(pymodule_name)s.%(pymodule_generator_name)s(context, _INPUT, conf=%(conf)s, **kwargs)\n"""
% {
'module_id': module_id,
'pymodule_name': pymodule_name,
'pymodule_generator_name': pymodule_generator_name,
'conf': module['conf'],
#Note: no embed (so no subloops) or wire kargs are passed and outer kwargs are passed in
})
exec pypipe #Note: evaluated in current namespace - todo ok?
steps[module_id] = eval("pipe_%(module_id)s" %
{'module_id': module_id})
else:
module_ref = eval(
"%(pymodule_name)s.%(pymodule_generator_name)s" % {
'pymodule_name': pymodule_name,
'pymodule_generator_name': pymodule_generator_name,
})
steps[module_id] = module_ref(*pargs, **kargs)
if context.verbose:
print "%s (%s) = %s(%s)" % (steps[module_id], module_id,
module_ref, str(pargs))
return steps[module_id]
def write_pipe(context, pipe):
"""Convert a pipe into Python script
If context.describe_input is passed to the script then it just returns the input requirements instead of the pipeline
"""
pypipe = ("""#Pipe %(pipename)s generated by pipe2py\n"""
"""\n"""
"""from pipe2py import Context\n"""
"""from pipe2py.modules import *\n"""
"""\n""" % {'pipename':pipe['name']}
)
pyinput = []
module_sequence = topological_sort(pipe['graph'])
#First pass to find any required subpipelines and user inputs
for module_id in module_sequence:
module = pipe['modules'][module_id]
if module['type'].startswith('pipe:'):
pypipe += """import %(module_type)s\n""" % {'module_type':util.pythonise(module['type'])}
if module['conf'] and 'prompt' in module['conf']:
pyinput.append((module['conf']['position']['value'],
module['conf']['name']['value'],
module['conf']['prompt']['value'],
module['conf']['default']['type'],
module['conf']['default']['value']))
#Note: there seems to be no need to recursively collate inputs from subpipelines
pypipe += ("""\n"""
"""def %(pipename)s(context, _INPUT, conf=None, **kwargs):\n"""
""" "Pipeline"\n""" #todo insert pipeline description here
""" if conf is None:\n"""
""" conf = {}\n"""
"""\n"""
""" if context.describe_input:\n"""
""" return %(inputs)s\n"""
"""\n"""
""" forever = pipeforever.pipe_forever(context, None, conf=None)\n"""
"""\n""" % {'pipename':pipe['name'],
'inputs':unicode(sorted(pyinput))} #todo pprint this
)
prev_module = []
for module_id in module_sequence:
module = pipe['modules'][module_id]
#Plumb I/O
input_module = "forever"
for wire in pipe['wires']:
if util.pythonise(pipe['wires'][wire]['tgt']['moduleid']) == module_id and pipe['wires'][wire]['tgt']['id'] == '_INPUT' and pipe['wires'][wire]['src']['id'].startswith('_OUTPUT'):
input_module = util.pythonise(pipe['wires'][wire]['src']['moduleid'])
if module_id in pipe['embed']:
assert input_module == "forever", "input_module of an embedded module was already set"
input_module = "_INPUT"
pargs = ["%(input_module)s" % {'input_module':input_module},
"conf=%(conf)s" % {'conf':module['conf']}, #todo pprint this
]
for wire in pipe['wires']:
if util.pythonise(pipe['wires'][wire]['tgt']['moduleid']) == module_id and pipe['wires'][wire]['tgt']['id'] != '_INPUT' and pipe['wires'][wire]['src']['id'].startswith('_OUTPUT'):
pargs.append("%(id)s = %(secondary_module)s" % {'id':util.pythonise(pipe['wires'][wire]['tgt']['id']), 'secondary_module':util.pythonise(pipe['wires'][wire]['src']['moduleid'])})
if module['type'] == 'loop':
pargs.append("embed = pipe_%(embed_module)s" % {'embed_module':util.pythonise(module['conf']['embed']['value']['id'])})
if module['type'] == 'split':
pargs.append("splits = %(splits)s" % {'splits':len([1 for w in pipe['wires'] if util.pythonise(pipe['wires'][w]['src']['moduleid']) == module_id])})
pymodule_name = "pipe%(module_type)s" % {'module_type':module['type']}
pymodule_generator_name = "pipe_%(module_type)s" % {'module_type':module['type']}
if module['type'].startswith('pipe:'):
pymodule_name = "%(module_type)s" % {'module_type':util.pythonise(module['type'])}
pymodule_generator_name = "%(module_type)s" % {'module_type':util.pythonise(module['type'])}
indent = ""
if module_id in pipe['embed']:
#We need to wrap submodules (used by loops) so we can pass the input at runtime (as we can to subpipelines)
pypipe += (""" def pipe_%(module_id)s(context, _INPUT, conf=None, **kwargs):\n"""
""" "Submodule"\n""" #todo insert submodule description here
% {'module_id':module_id}
)
indent = " "
pypipe += """%(indent)s %(module_id)s = %(pymodule_name)s.%(pymodule_generator_name)s(context, %(pargs)s)\n""" % {
'indent':indent,
'module_id':module_id,
'pymodule_name':pymodule_name,
'pymodule_generator_name':pymodule_generator_name,
'pargs':", ".join(pargs)}
if module_id in pipe['embed']:
pypipe += """ return %(module_id)s\n""" % {'module_id':module_id}
prev_module = module_id
#todo? if context.verbose:
# print "%s = %s.%s(%s)" %(module_id, pymodule_name, pymodule_generator_name, str(pargs))
pypipe += """ return %(module_id)s\n""" % {'module_id':prev_module}
pypipe += ("""\n"""
"""if __name__ == "__main__":\n"""
""" context = Context()\n"""
""" p = %(pipename)s(context, None)\n"""
""" for i in p:\n"""
""" print i\n""" % {'pipename':pipe['name']}
)
return pypipe
def build_pipe(context, pipe):
"""Convert a pipe into an executable Python pipeline
If context.describe_input then just return the input requirements instead of the pipeline
Note: any subpipes must be available to import as .py files
current namespace can become polluted by submodule wrapper definitions
"""
pyinput = []
module_sequence = topological_sort(pipe['graph'])
#First pass to find and import any required subpipelines and user inputs
#Note: assumes they have already been compiled to accessible .py files
for module_id in module_sequence:
module = pipe['modules'][module_id]
if module['type'].startswith('pipe:'):
__import__(util.pythonise(module['type']))
if 'prompt' in module['conf'] and context.describe_input:
pyinput.append((module['conf']['position']['value'],
module['conf']['name']['value'],
module['conf']['prompt']['value'],
module['conf']['default']['type'],
module['conf']['default']['value']))
#Note: there seems to be no need to recursively collate inputs from subpipelines
if context.describe_input:
return sorted(pyinput)
steps = {}
steps["forever"] = pipeforever.pipe_forever(context, None, conf=None)
for module_id in module_sequence:
module = pipe['modules'][module_id]
#Plumb I/O
# find the default input of this module
input_module = steps["forever"]
for wire in pipe['wires']:
# if the wire is to this module and it's the default input and it's the default output:
if util.pythonise(pipe['wires'][wire]['tgt']['moduleid']) == module_id and pipe['wires'][wire]['tgt']['id'] == '_INPUT' and pipe['wires'][wire]['src']['id'].startswith('_OUTPUT'): # todo? this equates the outputs
input_module = steps[util.pythonise(pipe['wires'][wire]['src']['moduleid'])]
if module_id in pipe['embed']:
assert input_module == steps["forever"], "input_module of an embedded module was already set"
input_module = "_INPUT"
pargs = [context,
input_module,
]
kargs = {"conf":module['conf'],
}
# set the extra inputs of this module as kargs of this module
for wire in pipe['wires']:
# if the wire is to this module and it's *not* the default input and it's the default output:
if util.pythonise(pipe['wires'][wire]['tgt']['moduleid']) == module_id and pipe['wires'][wire]['tgt']['id'] != '_INPUT' and pipe['wires'][wire]['src']['id'].startswith('_OUTPUT'): # todo? this equates the outputs
kargs["%(id)s" % {'id':util.pythonise(pipe['wires'][wire]['tgt']['id'])}] = steps[util.pythonise(pipe['wires'][wire]['src']['moduleid'])]
# set the embedded module in the kargs if this is loop module
if module['type'] == 'loop':
kargs["embed"] = steps[util.pythonise(module['conf']['embed']['value']['id'])]
#todo (re)import other pipes dynamically
pymodule_name = "pipe%(module_type)s" % {'module_type':module['type']}
pymodule_generator_name = "pipe_%(module_type)s" % {'module_type':module['type']}
if module['type'].startswith('pipe:'):
pymodule_name = "sys.modules['%(module_type)s']" % {'module_type':util.pythonise(module['type'])}
pymodule_generator_name = "%(module_type)s" % {'module_type':util.pythonise(module['type'])}
# if this module is an embedded module:
if module_id in pipe['embed']:
#We need to wrap submodules (used by loops) so we can pass the input at runtime (as we can to subpipelines)
pypipe = ("""def pipe_%(module_id)s(context, _INPUT, conf=None, **kwargs):\n"""
""" return %(pymodule_name)s.%(pymodule_generator_name)s(context, _INPUT, conf=%(conf)s, **kwargs)\n"""
% {'module_id':module_id,
'pymodule_name':pymodule_name,
'pymodule_generator_name':pymodule_generator_name,
'conf':module['conf'],
#Note: no embed (so no subloops) or wire kargs are passed and outer kwargs are passed in
}
)
exec pypipe #Note: evaluated in current namespace - todo ok?
steps[module_id] = eval("pipe_%(module_id)s" % {'module_id':module_id})
else: # else this module is not an embedded module:
module_ref = eval("%(pymodule_name)s.%(pymodule_generator_name)s" % {'pymodule_name':pymodule_name,
'pymodule_generator_name':pymodule_generator_name,})
steps[module_id] = module_ref(*pargs, **kargs)
if context.verbose:
print "%s (%s) = %s(%s)" %(steps[module_id], module_id, module_ref, str(pargs))
return steps[module_id]
def write_pipe(context, pipe):
"""Convert a pipe into Python script
If context.describe_input is passed to the script then it just returns the input requirements instead of the pipeline
"""
pypipe = ("""#Pipe %(pipename)s generated by pipe2py\n"""
"""\n"""
"""from pipe2py import Context\n"""
"""from pipe2py.modules import *\n"""
"""\n""" % {'pipename':pipe['name']}
)
pyinput = []
module_sequence = topological_sort(pipe['graph'])
#First pass to find any required subpipelines and user inputs
for module_id in module_sequence:
module = pipe['modules'][module_id]
if module['type'].startswith('pipe:'):
pypipe += """import %(module_type)s\n""" % {'module_type':util.pythonise(module['type'])}
if 'prompt' in module['conf']:
pyinput.append((module['conf']['position']['value'],
module['conf']['name']['value'],
module['conf']['prompt']['value'],
module['conf']['default']['type'],
module['conf']['default']['value']))
#Note: there seems to be no need to recursively collate inputs from subpipelines
pypipe += ("""\n"""
"""def %(pipename)s(context, _INPUT, conf=None, **kwargs):\n"""
""" "Pipeline"\n""" #todo insert pipeline description here
""" if conf is None:\n"""
""" conf = {}\n"""
"""\n"""
""" if context.describe_input:\n"""
""" return %(inputs)s\n"""
"""\n"""
""" forever = pipeforever.pipe_forever(context, None, conf=None)\n"""
"""\n""" % {'pipename':pipe['name'],
'inputs':unicode(sorted(pyinput))} #todo pprint this
)
prev_module = []
for module_id in module_sequence:
module = pipe['modules'][module_id]
#Plumb I/O
# find the default input of this module
input_module = "forever"
for wire in pipe['wires']:
# if the wire is to this module and it's the default input and it's the default output:
if util.pythonise(pipe['wires'][wire]['tgt']['moduleid']) == module_id and pipe['wires'][wire]['tgt']['id'] == '_INPUT' and pipe['wires'][wire]['src']['id'].startswith('_OUTPUT'): # todo? this equates the outputs
input_module = util.pythonise(pipe['wires'][wire]['src']['moduleid'])
if module_id in pipe['embed']:
assert input_module == "forever", "input_module of an embedded module was already set"
input_module = "_INPUT"
mod_args = [Id('context'), Id(input_module)]
mod_kwargs = [('conf', module['conf'])]
# set the extra inputs of this module as kwargs of this module
for wire in pipe['wires']:
# if the wire is to this module and it's *not* the default input and it's the default output:
if util.pythonise(pipe['wires'][wire]['tgt']['moduleid']) == module_id and pipe['wires'][wire]['tgt']['id'] != '_INPUT' and pipe['wires'][wire]['src']['id'].startswith('_OUTPUT'): # todo? this equates the outputs
mod_kwargs += [(util.pythonise(pipe['wires'][wire]['tgt']['id']), Id(util.pythonise(pipe['wires'][wire]['src']['moduleid'])))]
# set the embedded module in the kwargs if this is loop module
if module['type'] == 'loop':
mod_kwargs += [("embed", Id("pipe_%s" % util.pythonise(module['conf']['embed']['value']['id'])))]
pymodule_name = "pipe%(module_type)s" % {'module_type':module['type']}
pymodule_generator_name = "pipe_%(module_type)s" % {'module_type':module['type']}
if module['type'].startswith('pipe:'):
pymodule_name = "%(module_type)s" % {'module_type':util.pythonise(module['type'])}
pymodule_generator_name = "%(module_type)s" % {'module_type':util.pythonise(module['type'])}
indent = ""
if module_id in pipe['embed']:
#We need to wrap submodules (used by loops) so we can pass the input at runtime (as we can to subpipelines)
pypipe += (""" def pipe_%(module_id)s(context, _INPUT, conf=None, **kwargs):\n"""
""" "Submodule"\n""" #todo insert submodule description here
% {'module_id':module_id}
)
indent = " "
pypipe += """%(indent)s %(module_id)s = %(pymodule_name)s.%(pymodule_generator_name)s(%(pargs)s)\n""" % {
'indent':indent,
'module_id':module_id,
'pymodule_name':pymodule_name,
'pymodule_generator_name':pymodule_generator_name,
'pargs':repr_args(mod_args+mod_kwargs)}
if module_id in pipe['embed']:
pypipe += """ return %(module_id)s\n""" % {'module_id':module_id}
prev_module = module_id
if context.verbose:
print ("%s = %s(%s)" % (module_id, pymodule_generator_name,
str_args([arg for arg in mod_args if arg != Id('context')]+
[(key, value) for key, value in mod_kwargs if key != 'conf']+
[(key, value) for key, value in mod_kwargs if key == 'conf']))
).encode("utf-8")
pypipe += """ return %(module_id)s\n""" % {'module_id':prev_module}
pypipe += ("""\n"""
"""if __name__ == "__main__":\n"""
""" context = Context()\n"""
""" p = %(pipename)s(context, None)\n"""
""" for i in p:\n"""
""" print i\n""" % {'pipename':pipe['name']}
)
return pypipe
