def _parse_pipe(json_pipe, pipe_name="anonymous"): """Parse pipe JSON into internal structures Keyword arguments: json_pipe -- JSON representation of the pipe pipe_name -- a name for the pipe (used for linking pipes) Returns: pipe -- an internal representation of a pipe """ pipe = {"name": util.pythonise(pipe_name)} pipe["modules"] = {} pipe["embed"] = {} pipe["graph"] = {} pipe["wires"] = {} modules = json_pipe["modules"] if not isinstance(modules, list): modules = [modules] for module in modules: pipe["modules"][util.pythonise(module["id"])] = module pipe["graph"][util.pythonise(module["id"])] = [] if module["type"] == "loop": embed = module["conf"]["embed"]["value"] pipe["modules"][util.pythonise(embed["id"])] = embed pipe["graph"][util.pythonise(embed["id"])] = [] pipe["embed"][util.pythonise(embed["id"])] = embed # make the loop dependent on its embedded module pipe["graph"][util.pythonise(embed["id"])].append(util.pythonise(module["id"])) wires = json_pipe["wires"] if not isinstance(wires, list): wires = [wires] for wire in wires: pipe["graph"][util.pythonise(wire["src"]["moduleid"])].append(util.pythonise(wire["tgt"]["moduleid"])) # Remove any orphan nodes for node in pipe["graph"].keys(): targetted = [node in pipe["graph"][k] for k in pipe["graph"]] if not pipe["graph"][node] and not any(targetted): del pipe["graph"][node] for wire in wires: pipe["wires"][util.pythonise(wire["id"])] = wire return pipe
def build_pipe(context, pipe): """Convert a pipe into an executable Python pipeline If context.describe_input then just return the input requirements instead of the pipeline Note: any subpipes must be available to import as .py files current namespace can become polluted by submodule wrapper definitions """ pyinput = [] module_sequence = topological_sort(pipe["graph"]) # First pass to find and import any required subpipelines and user inputs # Note: assumes they have already been compiled to accessible .py files for module_id in module_sequence: module = pipe["modules"][module_id] if module["type"].startswith("pipe:"): __import__(util.pythonise(module["type"])) if module["conf"] and "prompt" in module["conf"] and context.describe_input: pyinput.append( ( module["conf"]["position"]["value"], module["conf"]["name"]["value"], module["conf"]["prompt"]["value"], module["conf"]["default"]["type"], module["conf"]["default"]["value"], ) ) # Note: there seems to be no need to recursively collate inputs from subpipelines if context.describe_input: return sorted(pyinput) steps = {} steps["forever"] = pipeforever.pipe_forever(context, None, conf=None) for module_id in module_sequence: module = pipe["modules"][module_id] # Plumb I/O input_module = steps["forever"] for wire in pipe["wires"]: if ( util.pythonise(pipe["wires"][wire]["tgt"]["moduleid"]) == module_id and pipe["wires"][wire]["tgt"]["id"] == "_INPUT" and pipe["wires"][wire]["src"]["id"].startswith("_OUTPUT") ): input_module = steps[util.pythonise(pipe["wires"][wire]["src"]["moduleid"])] if module_id in pipe["embed"]: assert input_module == steps["forever"], "input_module of an embedded module was already set" input_module = "_INPUT" pargs = [context, input_module] kargs = {"conf": module["conf"]} for wire in pipe["wires"]: if ( util.pythonise(pipe["wires"][wire]["tgt"]["moduleid"]) == module_id and pipe["wires"][wire]["tgt"]["id"] != "_INPUT" and pipe["wires"][wire]["src"]["id"].startswith("_OUTPUT") ): kargs["%(id)s" % {"id": util.pythonise(pipe["wires"][wire]["tgt"]["id"])}] = steps[ util.pythonise(pipe["wires"][wire]["src"]["moduleid"]) ] if module["type"] == "loop": kargs["embed"] = steps[util.pythonise(module["conf"]["embed"]["value"]["id"])] if module["type"] == "split": kargs["splits"] = len( [1 for w in pipe["wires"] if util.pythonise(pipe["wires"][w]["src"]["moduleid"]) == module_id] ) # todo (re)import other pipes dynamically pymodule_name = "pipe%(module_type)s" % {"module_type": module["type"]} pymodule_generator_name = "pipe_%(module_type)s" % {"module_type": module["type"]} if module["type"].startswith("pipe:"): pymodule_name = "sys.modules['%(module_type)s']" % {"module_type": util.pythonise(module["type"])} pymodule_generator_name = "%(module_type)s" % {"module_type": util.pythonise(module["type"])} if module_id in pipe["embed"]: # We need to wrap submodules (used by loops) so we can pass the input at runtime (as we can to subpipelines) pypipe = ( """def pipe_%(module_id)s(context, _INPUT, conf=None, **kwargs):\n""" """ return %(pymodule_name)s.%(pymodule_generator_name)s(context, _INPUT, conf=%(conf)s, **kwargs)\n""" % { "module_id": module_id, "pymodule_name": pymodule_name, "pymodule_generator_name": pymodule_generator_name, "conf": module["conf"], # Note: no embed (so no subloops) or wire kargs are passed and outer kwargs are passed in } ) exec pypipe # Note: evaluated in current namespace - todo ok? steps[module_id] = eval("pipe_%(module_id)s" % {"module_id": module_id}) else: module_ref = eval( "%(pymodule_name)s.%(pymodule_generator_name)s" % {"pymodule_name": pymodule_name, "pymodule_generator_name": pymodule_generator_name} ) steps[module_id] = module_ref(*pargs, **kargs) if context.verbose: print "%s (%s) = %s(%s)" % (steps[module_id], module_id, module_ref, str(pargs)) return steps[module_id]
def _parse_pipe(json_pipe, pipe_name="anonymous"): """Parse pipe JSON into internal structures Keyword arguments: json_pipe -- JSON representation of the pipe pipe_name -- a name for the pipe (used for linking pipes) Returns: pipe -- an internal representation of a pipe """ pipe = {'name': util.pythonise(pipe_name)} pipe['modules'] = {} pipe['embed'] = {} pipe['graph'] = {} pipe['wires'] = {} modules = json_pipe['modules'] if not isinstance(modules, list): modules = [modules] for module in modules: pipe['modules'][util.pythonise(module['id'])] = module pipe['graph'][util.pythonise(module['id'])] = [] if module['type'] == 'loop': embed = module['conf']['embed']['value'] pipe['modules'][util.pythonise(embed['id'])] = embed pipe['graph'][util.pythonise(embed['id'])] = [] pipe['embed'][util.pythonise(embed['id'])] = embed #make the loop dependent on its embedded module pipe['graph'][util.pythonise(embed['id'])].append( util.pythonise(module['id'])) wires = json_pipe['wires'] if not isinstance(wires, list): wires = [wires] for wire in wires: pipe['graph'][util.pythonise(wire['src']['moduleid'])].append( util.pythonise(wire['tgt']['moduleid'])) #Remove any orphan nodes for node in pipe['graph'].keys(): targetted = [node in pipe['graph'][k] for k in pipe['graph']] if not pipe['graph'][node] and not any(targetted): del pipe['graph'][node] for wire in wires: pipe['wires'][util.pythonise(wire['id'])] = wire return pipe
def write_pipe(context, pipe): """Convert a pipe into Python script If context.describe_input is passed to the script then it just returns the input requirements instead of the pipeline """ pypipe = ( """#Pipe %(pipename)s generated by pipe2py\n""" """\n""" """from pipe2py import Context\n""" """from pipe2py.modules import *\n""" """\n""" % {"pipename": pipe["name"]} ) pyinput = [] module_sequence = topological_sort(pipe["graph"]) # First pass to find any required subpipelines and user inputs for module_id in module_sequence: module = pipe["modules"][module_id] if module["type"].startswith("pipe:"): pypipe += """import %(module_type)s\n""" % {"module_type": util.pythonise(module["type"])} if module["conf"] and "prompt" in module["conf"]: pyinput.append( ( module["conf"]["position"]["value"], module["conf"]["name"]["value"], module["conf"]["prompt"]["value"], module["conf"]["default"]["type"], module["conf"]["default"]["value"], ) ) # Note: there seems to be no need to recursively collate inputs from subpipelines pypipe += ( """\n""" """def %(pipename)s(context, _INPUT, conf=None, **kwargs):\n""" """ "Pipeline"\n""" # todo insert pipeline description here """ if conf is None:\n""" """ conf = {}\n""" """\n""" """ if context.describe_input:\n""" """ return %(inputs)s\n""" """\n""" """ forever = pipeforever.pipe_forever(context, None, conf=None)\n""" """\n""" % {"pipename": pipe["name"], "inputs": unicode(sorted(pyinput))} # todo pprint this ) prev_module = [] for module_id in module_sequence: module = pipe["modules"][module_id] # Plumb I/O input_module = "forever" for wire in pipe["wires"]: if ( util.pythonise(pipe["wires"][wire]["tgt"]["moduleid"]) == module_id and pipe["wires"][wire]["tgt"]["id"] == "_INPUT" and pipe["wires"][wire]["src"]["id"].startswith("_OUTPUT") ): input_module = util.pythonise(pipe["wires"][wire]["src"]["moduleid"]) if module_id in pipe["embed"]: assert input_module == "forever", "input_module of an embedded module was already set" input_module = "_INPUT" pargs = [ "%(input_module)s" % {"input_module": input_module}, "conf=%(conf)s" % {"conf": module["conf"]}, # todo pprint this ] for wire in pipe["wires"]: if ( util.pythonise(pipe["wires"][wire]["tgt"]["moduleid"]) == module_id and pipe["wires"][wire]["tgt"]["id"] != "_INPUT" and pipe["wires"][wire]["src"]["id"].startswith("_OUTPUT") ): pargs.append( "%(id)s = %(secondary_module)s" % { "id": util.pythonise(pipe["wires"][wire]["tgt"]["id"]), "secondary_module": util.pythonise(pipe["wires"][wire]["src"]["moduleid"]), } ) if module["type"] == "loop": pargs.append( "embed = pipe_%(embed_module)s" % {"embed_module": util.pythonise(module["conf"]["embed"]["value"]["id"])} ) if module["type"] == "split": pargs.append( "splits = %(splits)s" % { "splits": len( [1 for w in pipe["wires"] if util.pythonise(pipe["wires"][w]["src"]["moduleid"]) == module_id] ) } ) pymodule_name = "pipe%(module_type)s" % {"module_type": module["type"]} pymodule_generator_name = "pipe_%(module_type)s" % {"module_type": module["type"]} if module["type"].startswith("pipe:"): pymodule_name = "%(module_type)s" % {"module_type": util.pythonise(module["type"])} pymodule_generator_name = "%(module_type)s" % {"module_type": util.pythonise(module["type"])} indent = "" if module_id in pipe["embed"]: # We need to wrap submodules (used by loops) so we can pass the input at runtime (as we can to subpipelines) pypipe += ( """ def pipe_%(module_id)s(context, _INPUT, conf=None, **kwargs):\n""" """ "Submodule"\n""" % {"module_id": module_id} # todo insert submodule description here ) indent = " " pypipe += ( """%(indent)s %(module_id)s = %(pymodule_name)s.%(pymodule_generator_name)s(context, %(pargs)s)\n""" % { "indent": indent, "module_id": module_id, "pymodule_name": pymodule_name, "pymodule_generator_name": pymodule_generator_name, "pargs": ", ".join(pargs), } ) if module_id in pipe["embed"]: pypipe += """ return %(module_id)s\n""" % {"module_id": module_id} prev_module = module_id # todo? if context.verbose: # print "%s = %s.%s(%s)" %(module_id, pymodule_name, pymodule_generator_name, str(pargs)) pypipe += """ return %(module_id)s\n""" % {"module_id": prev_module} pypipe += ( """\n""" """if __name__ == "__main__":\n""" """ context = Context()\n""" """ p = %(pipename)s(context, None)\n""" """ for i in p:\n""" """ print i\n""" % {"pipename": pipe["name"]} ) return pypipe
def build_pipe(context, pipe): """Convert a pipe into an executable Python pipeline If context.describe_input then just return the input requirements instead of the pipeline Note: any subpipes must be available to import as .py files current namespace can become polluted by submodule wrapper definitions """ pyinput = [] module_sequence = topological_sort(pipe['graph']) #First pass to find and import any required subpipelines and user inputs #Note: assumes they have already been compiled to accessible .py files for module_id in module_sequence: module = pipe['modules'][module_id] if module['type'].startswith('pipe:'): __import__(util.pythonise(module['type'])) if module['conf'] and 'prompt' in module[ 'conf'] and context.describe_input: pyinput.append((module['conf']['position']['value'], module['conf']['name']['value'], module['conf']['prompt']['value'], module['conf']['default']['type'], module['conf']['default']['value'])) #Note: there seems to be no need to recursively collate inputs from subpipelines if context.describe_input: return sorted(pyinput) steps = {} steps["forever"] = pipeforever.pipe_forever(context, None, conf=None) for module_id in module_sequence: module = pipe['modules'][module_id] #Plumb I/O input_module = steps["forever"] for wire in pipe['wires']: if util.pythonise(pipe['wires'][wire]['tgt']['moduleid'] ) == module_id and pipe['wires'][wire]['tgt'][ 'id'] == '_INPUT' and pipe['wires'][wire][ 'src']['id'].startswith('_OUTPUT'): input_module = steps[util.pythonise( pipe['wires'][wire]['src']['moduleid'])] if module_id in pipe['embed']: assert input_module == steps[ "forever"], "input_module of an embedded module was already set" input_module = "_INPUT" pargs = [ context, input_module, ] kargs = { "conf": module['conf'], } for wire in pipe['wires']: if util.pythonise(pipe['wires'][wire]['tgt']['moduleid'] ) == module_id and pipe['wires'][wire]['tgt'][ 'id'] != '_INPUT' and pipe['wires'][wire][ 'src']['id'].startswith('_OUTPUT'): kargs["%(id)s" % { 'id': util.pythonise(pipe['wires'][wire]['tgt']['id']) }] = steps[util.pythonise( pipe['wires'][wire]['src']['moduleid'])] if module['type'] == 'loop': kargs["embed"] = steps[util.pythonise( module['conf']['embed']['value']['id'])] if module['type'] == 'split': kargs["splits"] = len([ 1 for w in pipe['wires'] if util.pythonise( pipe['wires'][w]['src']['moduleid']) == module_id ]) #todo (re)import other pipes dynamically pymodule_name = "pipe%(module_type)s" % {'module_type': module['type']} pymodule_generator_name = "pipe_%(module_type)s" % { 'module_type': module['type'] } if module['type'].startswith('pipe:'): pymodule_name = "sys.modules['%(module_type)s']" % { 'module_type': util.pythonise(module['type']) } pymodule_generator_name = "%(module_type)s" % { 'module_type': util.pythonise(module['type']) } if module_id in pipe['embed']: #We need to wrap submodules (used by loops) so we can pass the input at runtime (as we can to subpipelines) pypipe = ( """def pipe_%(module_id)s(context, _INPUT, conf=None, **kwargs):\n""" """ return %(pymodule_name)s.%(pymodule_generator_name)s(context, _INPUT, conf=%(conf)s, **kwargs)\n""" % { 'module_id': module_id, 'pymodule_name': pymodule_name, 'pymodule_generator_name': pymodule_generator_name, 'conf': module['conf'], #Note: no embed (so no subloops) or wire kargs are passed and outer kwargs are passed in }) exec pypipe #Note: evaluated in current namespace - todo ok? steps[module_id] = eval("pipe_%(module_id)s" % {'module_id': module_id}) else: module_ref = eval( "%(pymodule_name)s.%(pymodule_generator_name)s" % { 'pymodule_name': pymodule_name, 'pymodule_generator_name': pymodule_generator_name, }) steps[module_id] = module_ref(*pargs, **kargs) if context.verbose: print "%s (%s) = %s(%s)" % (steps[module_id], module_id, module_ref, str(pargs)) return steps[module_id]
def write_pipe(context, pipe): """Convert a pipe into Python script If context.describe_input is passed to the script then it just returns the input requirements instead of the pipeline """ pypipe = ("""#Pipe %(pipename)s generated by pipe2py\n""" """\n""" """from pipe2py import Context\n""" """from pipe2py.modules import *\n""" """\n""" % { 'pipename': pipe['name'] }) pyinput = [] module_sequence = topological_sort(pipe['graph']) #First pass to find any required subpipelines and user inputs for module_id in module_sequence: module = pipe['modules'][module_id] if module['type'].startswith('pipe:'): pypipe += """import %(module_type)s\n""" % { 'module_type': util.pythonise(module['type']) } if module['conf'] and 'prompt' in module['conf']: pyinput.append((module['conf']['position']['value'], module['conf']['name']['value'], module['conf']['prompt']['value'], module['conf']['default']['type'], module['conf']['default']['value'])) #Note: there seems to be no need to recursively collate inputs from subpipelines pypipe += ( """\n""" """def %(pipename)s(context, _INPUT, conf=None, **kwargs):\n""" """ "Pipeline"\n""" #todo insert pipeline description here """ if conf is None:\n""" """ conf = {}\n""" """\n""" """ if context.describe_input:\n""" """ return %(inputs)s\n""" """\n""" """ forever = pipeforever.pipe_forever(context, None, conf=None)\n""" """\n""" % { 'pipename': pipe['name'], 'inputs': unicode(sorted(pyinput)) } #todo pprint this ) prev_module = [] for module_id in module_sequence: module = pipe['modules'][module_id] #Plumb I/O input_module = "forever" for wire in pipe['wires']: if util.pythonise(pipe['wires'][wire]['tgt']['moduleid'] ) == module_id and pipe['wires'][wire]['tgt'][ 'id'] == '_INPUT' and pipe['wires'][wire][ 'src']['id'].startswith('_OUTPUT'): input_module = util.pythonise( pipe['wires'][wire]['src']['moduleid']) if module_id in pipe['embed']: assert input_module == "forever", "input_module of an embedded module was already set" input_module = "_INPUT" pargs = [ "%(input_module)s" % { 'input_module': input_module }, "conf=%(conf)s" % { 'conf': module['conf'] }, #todo pprint this ] for wire in pipe['wires']: if util.pythonise(pipe['wires'][wire]['tgt']['moduleid'] ) == module_id and pipe['wires'][wire]['tgt'][ 'id'] != '_INPUT' and pipe['wires'][wire][ 'src']['id'].startswith('_OUTPUT'): pargs.append( "%(id)s = %(secondary_module)s" % { 'id': util.pythonise(pipe['wires'][wire]['tgt']['id']), 'secondary_module': util.pythonise(pipe['wires'][wire]['src']['moduleid']) }) if module['type'] == 'loop': pargs.append( "embed = pipe_%(embed_module)s" % { 'embed_module': util.pythonise(module['conf']['embed']['value']['id']) }) if module['type'] == 'split': pargs.append( "splits = %(splits)s" % { 'splits': len([ 1 for w in pipe['wires'] if util.pythonise( pipe['wires'][w]['src']['moduleid']) == module_id ]) }) pymodule_name = "pipe%(module_type)s" % {'module_type': module['type']} pymodule_generator_name = "pipe_%(module_type)s" % { 'module_type': module['type'] } if module['type'].startswith('pipe:'): pymodule_name = "%(module_type)s" % { 'module_type': util.pythonise(module['type']) } pymodule_generator_name = "%(module_type)s" % { 'module_type': util.pythonise(module['type']) } indent = "" if module_id in pipe['embed']: #We need to wrap submodules (used by loops) so we can pass the input at runtime (as we can to subpipelines) pypipe += ( """ def pipe_%(module_id)s(context, _INPUT, conf=None, **kwargs):\n""" """ "Submodule"\n""" #todo insert submodule description here % { 'module_id': module_id }) indent = " " pypipe += """%(indent)s %(module_id)s = %(pymodule_name)s.%(pymodule_generator_name)s(context, %(pargs)s)\n""" % { 'indent': indent, 'module_id': module_id, 'pymodule_name': pymodule_name, 'pymodule_generator_name': pymodule_generator_name, 'pargs': ", ".join(pargs) } if module_id in pipe['embed']: pypipe += """ return %(module_id)s\n""" % { 'module_id': module_id } prev_module = module_id #todo? if context.verbose: # print "%s = %s.%s(%s)" %(module_id, pymodule_name, pymodule_generator_name, str(pargs)) pypipe += """ return %(module_id)s\n""" % {'module_id': prev_module} pypipe += ("""\n""" """if __name__ == "__main__":\n""" """ context = Context()\n""" """ p = %(pipename)s(context, None)\n""" """ for i in p:\n""" """ print i\n""" % { 'pipename': pipe['name'] }) return pypipe
def _parse_pipe(json_pipe, pipe_name="anonymous"): """Parse pipe JSON into internal structures Keyword arguments: json_pipe -- JSON representation of the pipe pipe_name -- a name for the pipe (used for linking pipes) Returns: pipe -- an internal representation of a pipe """ pipe = {'name': util.pythonise(pipe_name)} pipe['modules'] = {} pipe['embed'] = {} pipe['graph'] = {} pipe['wires'] = {} modules = json_pipe['modules'] if not isinstance(modules, list): modules = [modules] for module in modules: pipe['modules'][util.pythonise(module['id'])] = module pipe['graph'][util.pythonise(module['id'])] = [] if module['type'] == 'loop': embed = module['conf']['embed']['value'] pipe['modules'][util.pythonise(embed['id'])] = embed pipe['graph'][util.pythonise(embed['id'])] = [] pipe['embed'][util.pythonise(embed['id'])] = embed #make the loop dependent on its embedded module pipe['graph'][util.pythonise(embed['id'])].append(util.pythonise(module['id'])) wires = json_pipe['wires'] if not isinstance(wires, list): wires = [wires] for wire in wires: pipe['graph'][util.pythonise(wire['src']['moduleid'])].append(util.pythonise(wire['tgt']['moduleid'])) #Remove any orphan nodes for node in pipe['graph'].keys(): targetted = [node in pipe['graph'][k] for k in pipe['graph']] if not pipe['graph'][node] and not any(targetted): del pipe['graph'][node] for wire in wires: pipe['wires'][util.pythonise(wire['id'])] = wire return pipe
def write_pipe(context, pipe): """Convert a pipe into Python script If context.describe_input is passed to the script then it just returns the input requirements instead of the pipeline """ pypipe = ("""#Pipe %(pipename)s generated by pipe2py\n""" """\n""" """from pipe2py import Context\n""" """from pipe2py.modules import *\n""" """\n""" % {'pipename':pipe['name']} ) pyinput = [] module_sequence = topological_sort(pipe['graph']) #First pass to find any required subpipelines and user inputs for module_id in module_sequence: module = pipe['modules'][module_id] if module['type'].startswith('pipe:'): pypipe += """import %(module_type)s\n""" % {'module_type':util.pythonise(module['type'])} if module['conf'] and 'prompt' in module['conf']: pyinput.append((module['conf']['position']['value'], module['conf']['name']['value'], module['conf']['prompt']['value'], module['conf']['default']['type'], module['conf']['default']['value'])) #Note: there seems to be no need to recursively collate inputs from subpipelines pypipe += ("""\n""" """def %(pipename)s(context, _INPUT, conf=None, **kwargs):\n""" """ "Pipeline"\n""" #todo insert pipeline description here """ if conf is None:\n""" """ conf = {}\n""" """\n""" """ if context.describe_input:\n""" """ return %(inputs)s\n""" """\n""" """ forever = pipeforever.pipe_forever(context, None, conf=None)\n""" """\n""" % {'pipename':pipe['name'], 'inputs':unicode(sorted(pyinput))} #todo pprint this ) prev_module = [] for module_id in module_sequence: module = pipe['modules'][module_id] #Plumb I/O input_module = "forever" for wire in pipe['wires']: if util.pythonise(pipe['wires'][wire]['tgt']['moduleid']) == module_id and pipe['wires'][wire]['tgt']['id'] == '_INPUT' and pipe['wires'][wire]['src']['id'].startswith('_OUTPUT'): input_module = util.pythonise(pipe['wires'][wire]['src']['moduleid']) if module_id in pipe['embed']: assert input_module == "forever", "input_module of an embedded module was already set" input_module = "_INPUT" pargs = ["%(input_module)s" % {'input_module':input_module}, "conf=%(conf)s" % {'conf':module['conf']}, #todo pprint this ] for wire in pipe['wires']: if util.pythonise(pipe['wires'][wire]['tgt']['moduleid']) == module_id and pipe['wires'][wire]['tgt']['id'] != '_INPUT' and pipe['wires'][wire]['src']['id'].startswith('_OUTPUT'): pargs.append("%(id)s = %(secondary_module)s" % {'id':util.pythonise(pipe['wires'][wire]['tgt']['id']), 'secondary_module':util.pythonise(pipe['wires'][wire]['src']['moduleid'])}) if module['type'] == 'loop': pargs.append("embed = pipe_%(embed_module)s" % {'embed_module':util.pythonise(module['conf']['embed']['value']['id'])}) if module['type'] == 'split': pargs.append("splits = %(splits)s" % {'splits':len([1 for w in pipe['wires'] if util.pythonise(pipe['wires'][w]['src']['moduleid']) == module_id])}) pymodule_name = "pipe%(module_type)s" % {'module_type':module['type']} pymodule_generator_name = "pipe_%(module_type)s" % {'module_type':module['type']} if module['type'].startswith('pipe:'): pymodule_name = "%(module_type)s" % {'module_type':util.pythonise(module['type'])} pymodule_generator_name = "%(module_type)s" % {'module_type':util.pythonise(module['type'])} indent = "" if module_id in pipe['embed']: #We need to wrap submodules (used by loops) so we can pass the input at runtime (as we can to subpipelines) pypipe += (""" def pipe_%(module_id)s(context, _INPUT, conf=None, **kwargs):\n""" """ "Submodule"\n""" #todo insert submodule description here % {'module_id':module_id} ) indent = " " pypipe += """%(indent)s %(module_id)s = %(pymodule_name)s.%(pymodule_generator_name)s(context, %(pargs)s)\n""" % { 'indent':indent, 'module_id':module_id, 'pymodule_name':pymodule_name, 'pymodule_generator_name':pymodule_generator_name, 'pargs':", ".join(pargs)} if module_id in pipe['embed']: pypipe += """ return %(module_id)s\n""" % {'module_id':module_id} prev_module = module_id #todo? if context.verbose: # print "%s = %s.%s(%s)" %(module_id, pymodule_name, pymodule_generator_name, str(pargs)) pypipe += """ return %(module_id)s\n""" % {'module_id':prev_module} pypipe += ("""\n""" """if __name__ == "__main__":\n""" """ context = Context()\n""" """ p = %(pipename)s(context, None)\n""" """ for i in p:\n""" """ print i\n""" % {'pipename':pipe['name']} ) return pypipe
def _pipe_commons(context, pipe, module_id, pyinput=None, steps=None): pyinput = pyinput or [] module = pipe['modules'][module_id] module_type = module['type'] conf = module['conf'] kwargs = {'conf': conf} output = None if module_type.startswith('pipe:'): # Import any required sub-pipelines and user inputs # Note: assumes they have already been compiled to accessible .py files import_module(util.pythonise(module_type)) if steps else None pythonised_type = util.pythonise(module_type) pymodule_name = '%s' % pythonised_type pymodule_generator = '%s' % pythonised_type else: pymodule_name = 'pipe%s' % module_type pymodule_generator = 'pipe_%s' % module_type if context.describe_input or not steps: # Find any required subpipelines and user inputs if conf and 'prompt' in conf: # Note: there seems to be no need to recursively collate inputs # from subpipelines module_confs = ( module['conf']['position']['value'], module['conf']['name']['value'], module['conf']['prompt']['value'], module['conf']['default']['type'], module['conf']['default']['value'] ) pyinput.append(module_confs) if steps: output = { 'pyinput': pyinput, 'pymodule_name': pymodule_name, 'pymodule_generator': pymodule_generator, } if not output: # find the default input of this module input_module = steps['forever'] if steps else 'forever' for key, pipe_wire in pipe['wires'].items(): moduleid = util.pythonise(pipe_wire['src']['moduleid']) # todo? this equates the outputs is_default_in_and_out = ( util.pythonise(pipe_wire['tgt']['moduleid']) == module_id and pipe_wire['tgt']['id'] == '_INPUT' and pipe_wire['src']['id'].startswith('_OUTPUT') ) # if the wire is to this module and it's the default input and it's # the default output: if is_default_in_and_out: input_module = steps[moduleid] if steps else moduleid # todo? this equates the outputs is_default_out_only = ( util.pythonise(pipe_wire['tgt']['moduleid']) == module_id and pipe_wire['tgt']['id'] != '_INPUT' and pipe_wire['src']['id'].startswith('_OUTPUT') ) # if the wire is to this module and it's *NOT* the default input # and it's the default output if is_default_out_only: # set the extra inputs of this module as kwargs of this module pipe_id = util.pythonise(pipe_wire['tgt']['id']) updated = steps[moduleid] if steps else Id(moduleid) kwargs.update({pipe_id: updated}) if module_id in pipe['embed']: text = 'input_module of an embedded module was already set' # what is this for??? # assert input_module == (steps['forever'], text) input_module = '_INPUT' args = [context, input_module] if steps else [ Id('context'), Id(input_module)] # set the embedded module in the kwargs if this is loop module if module_type == 'loop': value = module['conf']['embed']['value'] pipe_id = util.pythonise(value['id']) updated = steps[pipe_id] if steps else Id('pipe_%s' % pipe_id) kwargs.update({'embed': updated}) # set splits in the kwargs if this is split module if module_type == 'split': filtered = filter( lambda x: module_id == util.pythonise(x[1]['src']['moduleid']), pipe['wires'].items() ) count = len(filtered) updated = count if steps else Id(count) kwargs.update({'splits': updated}) output = { 'pyinput': pyinput, 'pymodule_name': pymodule_name, 'pymodule_generator': pymodule_generator, 'args': args, 'kwargs': kwargs, } return output
def build_pipe(context, pipe): """Convert a pipe into an executable Python pipeline If context.describe_input then just return the input requirements instead of the pipeline Note: any subpipes must be available to import as .py files current namespace can become polluted by submodule wrapper definitions """ pyinput = [] module_sequence = topological_sort(pipe['graph']) #First pass to find and import any required subpipelines and user inputs #Note: assumes they have already been compiled to accessible .py files for module_id in module_sequence: module = pipe['modules'][module_id] if module['type'].startswith('pipe:'): __import__(util.pythonise(module['type'])) if 'prompt' in module['conf'] and context.describe_input: pyinput.append((module['conf']['position']['value'], module['conf']['name']['value'], module['conf']['prompt']['value'], module['conf']['default']['type'], module['conf']['default']['value'])) #Note: there seems to be no need to recursively collate inputs from subpipelines if context.describe_input: return sorted(pyinput) steps = {} steps["forever"] = pipeforever.pipe_forever(context, None, conf=None) for module_id in module_sequence: module = pipe['modules'][module_id] #Plumb I/O # find the default input of this module input_module = steps["forever"] for wire in pipe['wires']: # if the wire is to this module and it's the default input and it's the default output: if util.pythonise(pipe['wires'][wire]['tgt']['moduleid']) == module_id and pipe['wires'][wire]['tgt']['id'] == '_INPUT' and pipe['wires'][wire]['src']['id'].startswith('_OUTPUT'): # todo? this equates the outputs input_module = steps[util.pythonise(pipe['wires'][wire]['src']['moduleid'])] if module_id in pipe['embed']: assert input_module == steps["forever"], "input_module of an embedded module was already set" input_module = "_INPUT" pargs = [context, input_module, ] kargs = {"conf":module['conf'], } # set the extra inputs of this module as kargs of this module for wire in pipe['wires']: # if the wire is to this module and it's *not* the default input and it's the default output: if util.pythonise(pipe['wires'][wire]['tgt']['moduleid']) == module_id and pipe['wires'][wire]['tgt']['id'] != '_INPUT' and pipe['wires'][wire]['src']['id'].startswith('_OUTPUT'): # todo? this equates the outputs kargs["%(id)s" % {'id':util.pythonise(pipe['wires'][wire]['tgt']['id'])}] = steps[util.pythonise(pipe['wires'][wire]['src']['moduleid'])] # set the embedded module in the kargs if this is loop module if module['type'] == 'loop': kargs["embed"] = steps[util.pythonise(module['conf']['embed']['value']['id'])] #todo (re)import other pipes dynamically pymodule_name = "pipe%(module_type)s" % {'module_type':module['type']} pymodule_generator_name = "pipe_%(module_type)s" % {'module_type':module['type']} if module['type'].startswith('pipe:'): pymodule_name = "sys.modules['%(module_type)s']" % {'module_type':util.pythonise(module['type'])} pymodule_generator_name = "%(module_type)s" % {'module_type':util.pythonise(module['type'])} # if this module is an embedded module: if module_id in pipe['embed']: #We need to wrap submodules (used by loops) so we can pass the input at runtime (as we can to subpipelines) pypipe = ("""def pipe_%(module_id)s(context, _INPUT, conf=None, **kwargs):\n""" """ return %(pymodule_name)s.%(pymodule_generator_name)s(context, _INPUT, conf=%(conf)s, **kwargs)\n""" % {'module_id':module_id, 'pymodule_name':pymodule_name, 'pymodule_generator_name':pymodule_generator_name, 'conf':module['conf'], #Note: no embed (so no subloops) or wire kargs are passed and outer kwargs are passed in } ) exec pypipe #Note: evaluated in current namespace - todo ok? steps[module_id] = eval("pipe_%(module_id)s" % {'module_id':module_id}) else: # else this module is not an embedded module: module_ref = eval("%(pymodule_name)s.%(pymodule_generator_name)s" % {'pymodule_name':pymodule_name, 'pymodule_generator_name':pymodule_generator_name,}) steps[module_id] = module_ref(*pargs, **kargs) if context.verbose: print "%s (%s) = %s(%s)" %(steps[module_id], module_id, module_ref, str(pargs)) return steps[module_id]
def write_pipe(context, pipe): """Convert a pipe into Python script If context.describe_input is passed to the script then it just returns the input requirements instead of the pipeline """ pypipe = ("""#Pipe %(pipename)s generated by pipe2py\n""" """\n""" """from pipe2py import Context\n""" """from pipe2py.modules import *\n""" """\n""" % {'pipename':pipe['name']} ) pyinput = [] module_sequence = topological_sort(pipe['graph']) #First pass to find any required subpipelines and user inputs for module_id in module_sequence: module = pipe['modules'][module_id] if module['type'].startswith('pipe:'): pypipe += """import %(module_type)s\n""" % {'module_type':util.pythonise(module['type'])} if 'prompt' in module['conf']: pyinput.append((module['conf']['position']['value'], module['conf']['name']['value'], module['conf']['prompt']['value'], module['conf']['default']['type'], module['conf']['default']['value'])) #Note: there seems to be no need to recursively collate inputs from subpipelines pypipe += ("""\n""" """def %(pipename)s(context, _INPUT, conf=None, **kwargs):\n""" """ "Pipeline"\n""" #todo insert pipeline description here """ if conf is None:\n""" """ conf = {}\n""" """\n""" """ if context.describe_input:\n""" """ return %(inputs)s\n""" """\n""" """ forever = pipeforever.pipe_forever(context, None, conf=None)\n""" """\n""" % {'pipename':pipe['name'], 'inputs':unicode(sorted(pyinput))} #todo pprint this ) prev_module = [] for module_id in module_sequence: module = pipe['modules'][module_id] #Plumb I/O # find the default input of this module input_module = "forever" for wire in pipe['wires']: # if the wire is to this module and it's the default input and it's the default output: if util.pythonise(pipe['wires'][wire]['tgt']['moduleid']) == module_id and pipe['wires'][wire]['tgt']['id'] == '_INPUT' and pipe['wires'][wire]['src']['id'].startswith('_OUTPUT'): # todo? this equates the outputs input_module = util.pythonise(pipe['wires'][wire]['src']['moduleid']) if module_id in pipe['embed']: assert input_module == "forever", "input_module of an embedded module was already set" input_module = "_INPUT" mod_args = [Id('context'), Id(input_module)] mod_kwargs = [('conf', module['conf'])] # set the extra inputs of this module as kwargs of this module for wire in pipe['wires']: # if the wire is to this module and it's *not* the default input and it's the default output: if util.pythonise(pipe['wires'][wire]['tgt']['moduleid']) == module_id and pipe['wires'][wire]['tgt']['id'] != '_INPUT' and pipe['wires'][wire]['src']['id'].startswith('_OUTPUT'): # todo? this equates the outputs mod_kwargs += [(util.pythonise(pipe['wires'][wire]['tgt']['id']), Id(util.pythonise(pipe['wires'][wire]['src']['moduleid'])))] # set the embedded module in the kwargs if this is loop module if module['type'] == 'loop': mod_kwargs += [("embed", Id("pipe_%s" % util.pythonise(module['conf']['embed']['value']['id'])))] pymodule_name = "pipe%(module_type)s" % {'module_type':module['type']} pymodule_generator_name = "pipe_%(module_type)s" % {'module_type':module['type']} if module['type'].startswith('pipe:'): pymodule_name = "%(module_type)s" % {'module_type':util.pythonise(module['type'])} pymodule_generator_name = "%(module_type)s" % {'module_type':util.pythonise(module['type'])} indent = "" if module_id in pipe['embed']: #We need to wrap submodules (used by loops) so we can pass the input at runtime (as we can to subpipelines) pypipe += (""" def pipe_%(module_id)s(context, _INPUT, conf=None, **kwargs):\n""" """ "Submodule"\n""" #todo insert submodule description here % {'module_id':module_id} ) indent = " " pypipe += """%(indent)s %(module_id)s = %(pymodule_name)s.%(pymodule_generator_name)s(%(pargs)s)\n""" % { 'indent':indent, 'module_id':module_id, 'pymodule_name':pymodule_name, 'pymodule_generator_name':pymodule_generator_name, 'pargs':repr_args(mod_args+mod_kwargs)} if module_id in pipe['embed']: pypipe += """ return %(module_id)s\n""" % {'module_id':module_id} prev_module = module_id if context.verbose: print ("%s = %s(%s)" % (module_id, pymodule_generator_name, str_args([arg for arg in mod_args if arg != Id('context')]+ [(key, value) for key, value in mod_kwargs if key != 'conf']+ [(key, value) for key, value in mod_kwargs if key == 'conf'])) ).encode("utf-8") pypipe += """ return %(module_id)s\n""" % {'module_id':prev_module} pypipe += ("""\n""" """if __name__ == "__main__":\n""" """ context = Context()\n""" """ p = %(pipename)s(context, None)\n""" """ for i in p:\n""" """ print i\n""" % {'pipename':pipe['name']} ) return pypipe