示例#1
0
def write_pipe(context, pipe):
    """Convert a pipe into Python script
    
       If context.describe_input is passed to the script then it just returns the input requirements instead of the pipeline
    """

    pypipe = (
        """#Pipe %(pipename)s generated by pipe2py\n"""
        """\n"""
        """from pipe2py import Context\n"""
        """from pipe2py.modules import *\n"""
        """\n""" % {"pipename": pipe["name"]}
    )
    pyinput = []

    module_sequence = topological_sort(pipe["graph"])

    # First pass to find any required subpipelines and user inputs
    for module_id in module_sequence:
        module = pipe["modules"][module_id]
        if module["type"].startswith("pipe:"):
            pypipe += """import %(module_type)s\n""" % {"module_type": util.pythonise(module["type"])}
        if module["conf"] and "prompt" in module["conf"]:
            pyinput.append(
                (
                    module["conf"]["position"]["value"],
                    module["conf"]["name"]["value"],
                    module["conf"]["prompt"]["value"],
                    module["conf"]["default"]["type"],
                    module["conf"]["default"]["value"],
                )
            )
            # Note: there seems to be no need to recursively collate inputs from subpipelines

    pypipe += (
        """\n"""
        """def %(pipename)s(context, _INPUT, conf=None, **kwargs):\n"""
        """    "Pipeline"\n"""  # todo insert pipeline description here
        """    if conf is None:\n"""
        """        conf = {}\n"""
        """\n"""
        """    if context.describe_input:\n"""
        """        return %(inputs)s\n"""
        """\n"""
        """    forever = pipeforever.pipe_forever(context, None, conf=None)\n"""
        """\n""" % {"pipename": pipe["name"], "inputs": unicode(sorted(pyinput))}  # todo pprint this
    )

    prev_module = []
    for module_id in module_sequence:
        module = pipe["modules"][module_id]

        # Plumb I/O
        input_module = "forever"
        for wire in pipe["wires"]:
            if (
                util.pythonise(pipe["wires"][wire]["tgt"]["moduleid"]) == module_id
                and pipe["wires"][wire]["tgt"]["id"] == "_INPUT"
                and pipe["wires"][wire]["src"]["id"].startswith("_OUTPUT")
            ):
                input_module = util.pythonise(pipe["wires"][wire]["src"]["moduleid"])

        if module_id in pipe["embed"]:
            assert input_module == "forever", "input_module of an embedded module was already set"
            input_module = "_INPUT"

        pargs = [
            "%(input_module)s" % {"input_module": input_module},
            "conf=%(conf)s" % {"conf": module["conf"]},  # todo pprint this
        ]

        for wire in pipe["wires"]:
            if (
                util.pythonise(pipe["wires"][wire]["tgt"]["moduleid"]) == module_id
                and pipe["wires"][wire]["tgt"]["id"] != "_INPUT"
                and pipe["wires"][wire]["src"]["id"].startswith("_OUTPUT")
            ):
                pargs.append(
                    "%(id)s = %(secondary_module)s"
                    % {
                        "id": util.pythonise(pipe["wires"][wire]["tgt"]["id"]),
                        "secondary_module": util.pythonise(pipe["wires"][wire]["src"]["moduleid"]),
                    }
                )

        if module["type"] == "loop":
            pargs.append(
                "embed = pipe_%(embed_module)s"
                % {"embed_module": util.pythonise(module["conf"]["embed"]["value"]["id"])}
            )

        if module["type"] == "split":
            pargs.append(
                "splits = %(splits)s"
                % {
                    "splits": len(
                        [1 for w in pipe["wires"] if util.pythonise(pipe["wires"][w]["src"]["moduleid"]) == module_id]
                    )
                }
            )

        pymodule_name = "pipe%(module_type)s" % {"module_type": module["type"]}
        pymodule_generator_name = "pipe_%(module_type)s" % {"module_type": module["type"]}
        if module["type"].startswith("pipe:"):
            pymodule_name = "%(module_type)s" % {"module_type": util.pythonise(module["type"])}
            pymodule_generator_name = "%(module_type)s" % {"module_type": util.pythonise(module["type"])}

        indent = ""
        if module_id in pipe["embed"]:
            # We need to wrap submodules (used by loops) so we can pass the input at runtime (as we can to subpipelines)
            pypipe += (
                """    def pipe_%(module_id)s(context, _INPUT, conf=None, **kwargs):\n"""
                """        "Submodule"\n""" % {"module_id": module_id}  # todo insert submodule description here
            )
            indent = "    "

        pypipe += (
            """%(indent)s    %(module_id)s = %(pymodule_name)s.%(pymodule_generator_name)s(context, %(pargs)s)\n"""
            % {
                "indent": indent,
                "module_id": module_id,
                "pymodule_name": pymodule_name,
                "pymodule_generator_name": pymodule_generator_name,
                "pargs": ", ".join(pargs),
            }
        )
        if module_id in pipe["embed"]:
            pypipe += """        return %(module_id)s\n""" % {"module_id": module_id}

        prev_module = module_id

        # todo? if context.verbose:
        #    print "%s = %s.%s(%s)" %(module_id, pymodule_name, pymodule_generator_name, str(pargs))

    pypipe += """    return %(module_id)s\n""" % {"module_id": prev_module}
    pypipe += (
        """\n"""
        """if __name__ == "__main__":\n"""
        """    context = Context()\n"""
        """    p = %(pipename)s(context, None)\n"""
        """    for i in p:\n"""
        """        print i\n""" % {"pipename": pipe["name"]}
    )

    return pypipe
示例#2
0
def build_pipe(context, pipe):
    """Convert a pipe into an executable Python pipeline
    
       If context.describe_input then just return the input requirements instead of the pipeline
    
       Note: any subpipes must be available to import as .py files
             current namespace can become polluted by submodule wrapper definitions
    """
    pyinput = []

    module_sequence = topological_sort(pipe["graph"])

    # First pass to find and import any required subpipelines and user inputs
    # Note: assumes they have already been compiled to accessible .py files
    for module_id in module_sequence:
        module = pipe["modules"][module_id]
        if module["type"].startswith("pipe:"):
            __import__(util.pythonise(module["type"]))
        if module["conf"] and "prompt" in module["conf"] and context.describe_input:
            pyinput.append(
                (
                    module["conf"]["position"]["value"],
                    module["conf"]["name"]["value"],
                    module["conf"]["prompt"]["value"],
                    module["conf"]["default"]["type"],
                    module["conf"]["default"]["value"],
                )
            )
            # Note: there seems to be no need to recursively collate inputs from subpipelines

    if context.describe_input:
        return sorted(pyinput)

    steps = {}
    steps["forever"] = pipeforever.pipe_forever(context, None, conf=None)
    for module_id in module_sequence:
        module = pipe["modules"][module_id]

        # Plumb I/O
        input_module = steps["forever"]
        for wire in pipe["wires"]:
            if (
                util.pythonise(pipe["wires"][wire]["tgt"]["moduleid"]) == module_id
                and pipe["wires"][wire]["tgt"]["id"] == "_INPUT"
                and pipe["wires"][wire]["src"]["id"].startswith("_OUTPUT")
            ):
                input_module = steps[util.pythonise(pipe["wires"][wire]["src"]["moduleid"])]

        if module_id in pipe["embed"]:
            assert input_module == steps["forever"], "input_module of an embedded module was already set"
            input_module = "_INPUT"

        pargs = [context, input_module]
        kargs = {"conf": module["conf"]}

        for wire in pipe["wires"]:
            if (
                util.pythonise(pipe["wires"][wire]["tgt"]["moduleid"]) == module_id
                and pipe["wires"][wire]["tgt"]["id"] != "_INPUT"
                and pipe["wires"][wire]["src"]["id"].startswith("_OUTPUT")
            ):
                kargs["%(id)s" % {"id": util.pythonise(pipe["wires"][wire]["tgt"]["id"])}] = steps[
                    util.pythonise(pipe["wires"][wire]["src"]["moduleid"])
                ]

        if module["type"] == "loop":
            kargs["embed"] = steps[util.pythonise(module["conf"]["embed"]["value"]["id"])]

        if module["type"] == "split":
            kargs["splits"] = len(
                [1 for w in pipe["wires"] if util.pythonise(pipe["wires"][w]["src"]["moduleid"]) == module_id]
            )

        # todo (re)import other pipes dynamically
        pymodule_name = "pipe%(module_type)s" % {"module_type": module["type"]}
        pymodule_generator_name = "pipe_%(module_type)s" % {"module_type": module["type"]}
        if module["type"].startswith("pipe:"):
            pymodule_name = "sys.modules['%(module_type)s']" % {"module_type": util.pythonise(module["type"])}
            pymodule_generator_name = "%(module_type)s" % {"module_type": util.pythonise(module["type"])}

        if module_id in pipe["embed"]:
            # We need to wrap submodules (used by loops) so we can pass the input at runtime (as we can to subpipelines)
            pypipe = (
                """def pipe_%(module_id)s(context, _INPUT, conf=None, **kwargs):\n"""
                """    return %(pymodule_name)s.%(pymodule_generator_name)s(context, _INPUT, conf=%(conf)s, **kwargs)\n"""
                % {
                    "module_id": module_id,
                    "pymodule_name": pymodule_name,
                    "pymodule_generator_name": pymodule_generator_name,
                    "conf": module["conf"],
                    # Note: no embed (so no subloops) or wire kargs are passed and outer kwargs are passed in
                }
            )
            exec pypipe  # Note: evaluated in current namespace - todo ok?
            steps[module_id] = eval("pipe_%(module_id)s" % {"module_id": module_id})
        else:
            module_ref = eval(
                "%(pymodule_name)s.%(pymodule_generator_name)s"
                % {"pymodule_name": pymodule_name, "pymodule_generator_name": pymodule_generator_name}
            )
            steps[module_id] = module_ref(*pargs, **kargs)

        if context.verbose:
            print "%s (%s) = %s(%s)" % (steps[module_id], module_id, module_ref, str(pargs))

    return steps[module_id]
示例#3
0
def write_pipe(context, pipe):
    """Convert a pipe into Python script
    
       If context.describe_input is passed to the script then it just returns the input requirements instead of the pipeline
    """

    pypipe = ("""#Pipe %(pipename)s generated by pipe2py\n"""
              """\n"""
              """from pipe2py import Context\n"""
              """from pipe2py.modules import *\n"""
              """\n""" % {
                  'pipename': pipe['name']
              })
    pyinput = []

    module_sequence = topological_sort(pipe['graph'])

    #First pass to find any required subpipelines and user inputs
    for module_id in module_sequence:
        module = pipe['modules'][module_id]
        if module['type'].startswith('pipe:'):
            pypipe += """import %(module_type)s\n""" % {
                'module_type': util.pythonise(module['type'])
            }
        if module['conf'] and 'prompt' in module['conf']:
            pyinput.append((module['conf']['position']['value'],
                            module['conf']['name']['value'],
                            module['conf']['prompt']['value'],
                            module['conf']['default']['type'],
                            module['conf']['default']['value']))
            #Note: there seems to be no need to recursively collate inputs from subpipelines

    pypipe += (
        """\n"""
        """def %(pipename)s(context, _INPUT, conf=None, **kwargs):\n"""
        """    "Pipeline"\n"""  #todo insert pipeline description here
        """    if conf is None:\n"""
        """        conf = {}\n"""
        """\n"""
        """    if context.describe_input:\n"""
        """        return %(inputs)s\n"""
        """\n"""
        """    forever = pipeforever.pipe_forever(context, None, conf=None)\n"""
        """\n""" % {
            'pipename': pipe['name'],
            'inputs': unicode(sorted(pyinput))
        }  #todo pprint this
    )

    prev_module = []
    for module_id in module_sequence:
        module = pipe['modules'][module_id]

        #Plumb I/O
        input_module = "forever"
        for wire in pipe['wires']:
            if util.pythonise(pipe['wires'][wire]['tgt']['moduleid']
                              ) == module_id and pipe['wires'][wire]['tgt'][
                                  'id'] == '_INPUT' and pipe['wires'][wire][
                                      'src']['id'].startswith('_OUTPUT'):
                input_module = util.pythonise(
                    pipe['wires'][wire]['src']['moduleid'])

        if module_id in pipe['embed']:
            assert input_module == "forever", "input_module of an embedded module was already set"
            input_module = "_INPUT"

        pargs = [
            "%(input_module)s" % {
                'input_module': input_module
            },
            "conf=%(conf)s" % {
                'conf': module['conf']
            },  #todo pprint this
        ]

        for wire in pipe['wires']:
            if util.pythonise(pipe['wires'][wire]['tgt']['moduleid']
                              ) == module_id and pipe['wires'][wire]['tgt'][
                                  'id'] != '_INPUT' and pipe['wires'][wire][
                                      'src']['id'].startswith('_OUTPUT'):
                pargs.append(
                    "%(id)s = %(secondary_module)s" % {
                        'id':
                        util.pythonise(pipe['wires'][wire]['tgt']['id']),
                        'secondary_module':
                        util.pythonise(pipe['wires'][wire]['src']['moduleid'])
                    })

        if module['type'] == 'loop':
            pargs.append(
                "embed = pipe_%(embed_module)s" % {
                    'embed_module':
                    util.pythonise(module['conf']['embed']['value']['id'])
                })

        if module['type'] == 'split':
            pargs.append(
                "splits = %(splits)s" % {
                    'splits':
                    len([
                        1 for w in pipe['wires'] if util.pythonise(
                            pipe['wires'][w]['src']['moduleid']) == module_id
                    ])
                })

        pymodule_name = "pipe%(module_type)s" % {'module_type': module['type']}
        pymodule_generator_name = "pipe_%(module_type)s" % {
            'module_type': module['type']
        }
        if module['type'].startswith('pipe:'):
            pymodule_name = "%(module_type)s" % {
                'module_type': util.pythonise(module['type'])
            }
            pymodule_generator_name = "%(module_type)s" % {
                'module_type': util.pythonise(module['type'])
            }

        indent = ""
        if module_id in pipe['embed']:
            #We need to wrap submodules (used by loops) so we can pass the input at runtime (as we can to subpipelines)
            pypipe += (
                """    def pipe_%(module_id)s(context, _INPUT, conf=None, **kwargs):\n"""
                """        "Submodule"\n"""  #todo insert submodule description here
                % {
                    'module_id': module_id
                })
            indent = "    "

        pypipe += """%(indent)s    %(module_id)s = %(pymodule_name)s.%(pymodule_generator_name)s(context, %(pargs)s)\n""" % {
            'indent': indent,
            'module_id': module_id,
            'pymodule_name': pymodule_name,
            'pymodule_generator_name': pymodule_generator_name,
            'pargs': ", ".join(pargs)
        }
        if module_id in pipe['embed']:
            pypipe += """        return %(module_id)s\n""" % {
                'module_id': module_id
            }

        prev_module = module_id

        #todo? if context.verbose:
        #    print "%s = %s.%s(%s)" %(module_id, pymodule_name, pymodule_generator_name, str(pargs))

    pypipe += """    return %(module_id)s\n""" % {'module_id': prev_module}
    pypipe += ("""\n"""
               """if __name__ == "__main__":\n"""
               """    context = Context()\n"""
               """    p = %(pipename)s(context, None)\n"""
               """    for i in p:\n"""
               """        print i\n""" % {
                   'pipename': pipe['name']
               })

    return pypipe
示例#4
0
def build_pipe(context, pipe):
    """Convert a pipe into an executable Python pipeline
    
       If context.describe_input then just return the input requirements instead of the pipeline
    
       Note: any subpipes must be available to import as .py files
             current namespace can become polluted by submodule wrapper definitions
    """
    pyinput = []

    module_sequence = topological_sort(pipe['graph'])

    #First pass to find and import any required subpipelines and user inputs
    #Note: assumes they have already been compiled to accessible .py files
    for module_id in module_sequence:
        module = pipe['modules'][module_id]
        if module['type'].startswith('pipe:'):
            __import__(util.pythonise(module['type']))
        if module['conf'] and 'prompt' in module[
                'conf'] and context.describe_input:
            pyinput.append((module['conf']['position']['value'],
                            module['conf']['name']['value'],
                            module['conf']['prompt']['value'],
                            module['conf']['default']['type'],
                            module['conf']['default']['value']))
            #Note: there seems to be no need to recursively collate inputs from subpipelines

    if context.describe_input:
        return sorted(pyinput)

    steps = {}
    steps["forever"] = pipeforever.pipe_forever(context, None, conf=None)
    for module_id in module_sequence:
        module = pipe['modules'][module_id]

        #Plumb I/O
        input_module = steps["forever"]
        for wire in pipe['wires']:
            if util.pythonise(pipe['wires'][wire]['tgt']['moduleid']
                              ) == module_id and pipe['wires'][wire]['tgt'][
                                  'id'] == '_INPUT' and pipe['wires'][wire][
                                      'src']['id'].startswith('_OUTPUT'):
                input_module = steps[util.pythonise(
                    pipe['wires'][wire]['src']['moduleid'])]

        if module_id in pipe['embed']:
            assert input_module == steps[
                "forever"], "input_module of an embedded module was already set"
            input_module = "_INPUT"

        pargs = [
            context,
            input_module,
        ]
        kargs = {
            "conf": module['conf'],
        }

        for wire in pipe['wires']:
            if util.pythonise(pipe['wires'][wire]['tgt']['moduleid']
                              ) == module_id and pipe['wires'][wire]['tgt'][
                                  'id'] != '_INPUT' and pipe['wires'][wire][
                                      'src']['id'].startswith('_OUTPUT'):
                kargs["%(id)s" % {
                    'id': util.pythonise(pipe['wires'][wire]['tgt']['id'])
                }] = steps[util.pythonise(
                    pipe['wires'][wire]['src']['moduleid'])]

        if module['type'] == 'loop':
            kargs["embed"] = steps[util.pythonise(
                module['conf']['embed']['value']['id'])]

        if module['type'] == 'split':
            kargs["splits"] = len([
                1 for w in pipe['wires'] if util.pythonise(
                    pipe['wires'][w]['src']['moduleid']) == module_id
            ])

        #todo (re)import other pipes dynamically
        pymodule_name = "pipe%(module_type)s" % {'module_type': module['type']}
        pymodule_generator_name = "pipe_%(module_type)s" % {
            'module_type': module['type']
        }
        if module['type'].startswith('pipe:'):
            pymodule_name = "sys.modules['%(module_type)s']" % {
                'module_type': util.pythonise(module['type'])
            }
            pymodule_generator_name = "%(module_type)s" % {
                'module_type': util.pythonise(module['type'])
            }

        if module_id in pipe['embed']:
            #We need to wrap submodules (used by loops) so we can pass the input at runtime (as we can to subpipelines)
            pypipe = (
                """def pipe_%(module_id)s(context, _INPUT, conf=None, **kwargs):\n"""
                """    return %(pymodule_name)s.%(pymodule_generator_name)s(context, _INPUT, conf=%(conf)s, **kwargs)\n"""
                % {
                    'module_id': module_id,
                    'pymodule_name': pymodule_name,
                    'pymodule_generator_name': pymodule_generator_name,
                    'conf': module['conf'],
                    #Note: no embed (so no subloops) or wire kargs are passed and outer kwargs are passed in
                })
            exec pypipe  #Note: evaluated in current namespace - todo ok?
            steps[module_id] = eval("pipe_%(module_id)s" %
                                    {'module_id': module_id})
        else:
            module_ref = eval(
                "%(pymodule_name)s.%(pymodule_generator_name)s" % {
                    'pymodule_name': pymodule_name,
                    'pymodule_generator_name': pymodule_generator_name,
                })
            steps[module_id] = module_ref(*pargs, **kargs)

        if context.verbose:
            print "%s (%s) = %s(%s)" % (steps[module_id], module_id,
                                        module_ref, str(pargs))

    return steps[module_id]
示例#5
0
def write_pipe(context, pipe):
    """Convert a pipe into Python script
    
       If context.describe_input is passed to the script then it just returns the input requirements instead of the pipeline
    """

    pypipe = ("""#Pipe %(pipename)s generated by pipe2py\n"""
              """\n"""
              """from pipe2py import Context\n"""
              """from pipe2py.modules import *\n"""
              """\n""" % {'pipename':pipe['name']}
             )
    pyinput = []
    
    module_sequence = topological_sort(pipe['graph'])
    
    #First pass to find any required subpipelines and user inputs
    for module_id in module_sequence:
        module = pipe['modules'][module_id]
        if module['type'].startswith('pipe:'):
            pypipe += """import %(module_type)s\n""" % {'module_type':util.pythonise(module['type'])}
        if module['conf'] and 'prompt' in module['conf']:
            pyinput.append((module['conf']['position']['value'],
                            module['conf']['name']['value'],
                            module['conf']['prompt']['value'],
                            module['conf']['default']['type'],
                            module['conf']['default']['value']))
            #Note: there seems to be no need to recursively collate inputs from subpipelines
            
    pypipe += ("""\n"""
               """def %(pipename)s(context, _INPUT, conf=None, **kwargs):\n"""
               """    "Pipeline"\n"""     #todo insert pipeline description here
               """    if conf is None:\n"""
               """        conf = {}\n"""
               """\n"""
               """    if context.describe_input:\n"""
               """        return %(inputs)s\n"""
               """\n"""
               """    forever = pipeforever.pipe_forever(context, None, conf=None)\n"""
               """\n""" % {'pipename':pipe['name'],
                           'inputs':unicode(sorted(pyinput))}  #todo pprint this
              )

    prev_module = []
    for module_id in module_sequence:
        module = pipe['modules'][module_id]

        #Plumb I/O
        input_module = "forever"
        for wire in pipe['wires']:
            if util.pythonise(pipe['wires'][wire]['tgt']['moduleid']) == module_id and pipe['wires'][wire]['tgt']['id'] == '_INPUT' and pipe['wires'][wire]['src']['id'].startswith('_OUTPUT'):
                input_module = util.pythonise(pipe['wires'][wire]['src']['moduleid'])

        if module_id in pipe['embed']:
            assert input_module == "forever", "input_module of an embedded module was already set"
            input_module = "_INPUT"
        
        pargs = ["%(input_module)s" % {'input_module':input_module}, 
                 "conf=%(conf)s" % {'conf':module['conf']},  #todo pprint this
                ]
        
        for wire in pipe['wires']:
            if util.pythonise(pipe['wires'][wire]['tgt']['moduleid']) == module_id and pipe['wires'][wire]['tgt']['id'] != '_INPUT' and pipe['wires'][wire]['src']['id'].startswith('_OUTPUT'):
                pargs.append("%(id)s = %(secondary_module)s" % {'id':util.pythonise(pipe['wires'][wire]['tgt']['id']), 'secondary_module':util.pythonise(pipe['wires'][wire]['src']['moduleid'])})
                
        if module['type'] == 'loop':
            pargs.append("embed = pipe_%(embed_module)s" % {'embed_module':util.pythonise(module['conf']['embed']['value']['id'])})

        if module['type'] == 'split':
            pargs.append("splits = %(splits)s" % {'splits':len([1 for w in pipe['wires'] if util.pythonise(pipe['wires'][w]['src']['moduleid']) == module_id])})
            
        pymodule_name = "pipe%(module_type)s" % {'module_type':module['type']}
        pymodule_generator_name = "pipe_%(module_type)s" % {'module_type':module['type']}
        if module['type'].startswith('pipe:'):
            pymodule_name = "%(module_type)s" % {'module_type':util.pythonise(module['type'])}
            pymodule_generator_name = "%(module_type)s" % {'module_type':util.pythonise(module['type'])}            

        indent = ""
        if module_id in pipe['embed']:
            #We need to wrap submodules (used by loops) so we can pass the input at runtime (as we can to subpipelines)
            pypipe += ("""    def pipe_%(module_id)s(context, _INPUT, conf=None, **kwargs):\n"""
                       """        "Submodule"\n"""     #todo insert submodule description here
                       % {'module_id':module_id}
                       )
            indent = "    "
            
        pypipe += """%(indent)s    %(module_id)s = %(pymodule_name)s.%(pymodule_generator_name)s(context, %(pargs)s)\n""" % {
                                                 'indent':indent,
                                                 'module_id':module_id,
                                                 'pymodule_name':pymodule_name,
                                                 'pymodule_generator_name':pymodule_generator_name,
                                                 'pargs':", ".join(pargs)}
        if module_id in pipe['embed']:
            pypipe += """        return %(module_id)s\n""" % {'module_id':module_id}

        prev_module = module_id
        
        #todo? if context.verbose:
        #    print "%s = %s.%s(%s)" %(module_id, pymodule_name, pymodule_generator_name, str(pargs))
    
    pypipe += """    return %(module_id)s\n""" % {'module_id':prev_module}
    pypipe += ("""\n"""
               """if __name__ == "__main__":\n"""
               """    context = Context()\n"""
               """    p = %(pipename)s(context, None)\n"""
               """    for i in p:\n"""
               """        print i\n""" % {'pipename':pipe['name']}
              )
        
    return pypipe
示例#6
0
文件: compile.py 项目: tuukka/pipe2py
def build_pipe(context, pipe):
    """Convert a pipe into an executable Python pipeline
    
       If context.describe_input then just return the input requirements instead of the pipeline
    
       Note: any subpipes must be available to import as .py files
             current namespace can become polluted by submodule wrapper definitions
    """
    pyinput = []

    module_sequence = topological_sort(pipe['graph'])

    #First pass to find and import any required subpipelines and user inputs
    #Note: assumes they have already been compiled to accessible .py files
    for module_id in module_sequence:
        module = pipe['modules'][module_id]
        if module['type'].startswith('pipe:'):
            __import__(util.pythonise(module['type']))
        if 'prompt' in module['conf'] and context.describe_input:
            pyinput.append((module['conf']['position']['value'],
                            module['conf']['name']['value'],
                            module['conf']['prompt']['value'],
                            module['conf']['default']['type'],
                            module['conf']['default']['value']))
            #Note: there seems to be no need to recursively collate inputs from subpipelines
            
    if context.describe_input:
        return sorted(pyinput)
    
    steps = {}
    steps["forever"] = pipeforever.pipe_forever(context, None, conf=None)
    for module_id in module_sequence:
        module = pipe['modules'][module_id]
        
        #Plumb I/O

        # find the default input of this module
        input_module = steps["forever"]
        for wire in pipe['wires']:
            # if the wire is to this module and it's the default input and it's the default output:
            if util.pythonise(pipe['wires'][wire]['tgt']['moduleid']) == module_id and pipe['wires'][wire]['tgt']['id'] == '_INPUT' and pipe['wires'][wire]['src']['id'].startswith('_OUTPUT'): # todo? this equates the outputs
                input_module = steps[util.pythonise(pipe['wires'][wire]['src']['moduleid'])]

        if module_id in pipe['embed']:
            assert input_module == steps["forever"], "input_module of an embedded module was already set"
            input_module = "_INPUT"
                
        pargs = [context,
                 input_module,
                ]
        kargs = {"conf":module['conf'],
                }
            
        # set the extra inputs of this module as kargs of this module
        for wire in pipe['wires']:
            # if the wire is to this module and it's *not* the default input and it's the default output:
            if util.pythonise(pipe['wires'][wire]['tgt']['moduleid']) == module_id and pipe['wires'][wire]['tgt']['id'] != '_INPUT' and pipe['wires'][wire]['src']['id'].startswith('_OUTPUT'): # todo? this equates the outputs
                kargs["%(id)s" % {'id':util.pythonise(pipe['wires'][wire]['tgt']['id'])}] = steps[util.pythonise(pipe['wires'][wire]['src']['moduleid'])]
                
        # set the embedded module in the kargs if this is loop module
        if module['type'] == 'loop':
            kargs["embed"] = steps[util.pythonise(module['conf']['embed']['value']['id'])]
                
        #todo (re)import other pipes dynamically
        pymodule_name = "pipe%(module_type)s" % {'module_type':module['type']}
        pymodule_generator_name = "pipe_%(module_type)s" % {'module_type':module['type']}
        if module['type'].startswith('pipe:'):
            pymodule_name = "sys.modules['%(module_type)s']" % {'module_type':util.pythonise(module['type'])}
            pymodule_generator_name = "%(module_type)s" % {'module_type':util.pythonise(module['type'])}            
            
        # if this module is an embedded module:
        if module_id in pipe['embed']:
            #We need to wrap submodules (used by loops) so we can pass the input at runtime (as we can to subpipelines)
            pypipe = ("""def pipe_%(module_id)s(context, _INPUT, conf=None, **kwargs):\n"""
                      """    return %(pymodule_name)s.%(pymodule_generator_name)s(context, _INPUT, conf=%(conf)s, **kwargs)\n"""
                       % {'module_id':module_id,
                          'pymodule_name':pymodule_name, 
                          'pymodule_generator_name':pymodule_generator_name,
                          'conf':module['conf'], 
                          #Note: no embed (so no subloops) or wire kargs are passed and outer kwargs are passed in
                         }
                     )
            exec pypipe   #Note: evaluated in current namespace - todo ok?
            steps[module_id] = eval("pipe_%(module_id)s" % {'module_id':module_id})
        else: # else this module is not an embedded module:
            module_ref = eval("%(pymodule_name)s.%(pymodule_generator_name)s" % {'pymodule_name':pymodule_name, 
                                                                                 'pymodule_generator_name':pymodule_generator_name,})
            steps[module_id] = module_ref(*pargs, **kargs)

        if context.verbose:
            print "%s (%s) = %s(%s)" %(steps[module_id], module_id, module_ref, str(pargs))

    return steps[module_id]
示例#7
0
文件: compile.py 项目: tuukka/pipe2py
def write_pipe(context, pipe):
    """Convert a pipe into Python script
    
       If context.describe_input is passed to the script then it just returns the input requirements instead of the pipeline
    """

    pypipe = ("""#Pipe %(pipename)s generated by pipe2py\n"""
              """\n"""
              """from pipe2py import Context\n"""
              """from pipe2py.modules import *\n"""
              """\n""" % {'pipename':pipe['name']}
             )
    pyinput = []
    
    module_sequence = topological_sort(pipe['graph'])
    
    #First pass to find any required subpipelines and user inputs
    for module_id in module_sequence:
        module = pipe['modules'][module_id]
        if module['type'].startswith('pipe:'):
            pypipe += """import %(module_type)s\n""" % {'module_type':util.pythonise(module['type'])}
        if 'prompt' in module['conf']:
            pyinput.append((module['conf']['position']['value'],
                            module['conf']['name']['value'],
                            module['conf']['prompt']['value'],
                            module['conf']['default']['type'],
                            module['conf']['default']['value']))
            #Note: there seems to be no need to recursively collate inputs from subpipelines
            
    pypipe += ("""\n"""
               """def %(pipename)s(context, _INPUT, conf=None, **kwargs):\n"""
               """    "Pipeline"\n"""     #todo insert pipeline description here
               """    if conf is None:\n"""
               """        conf = {}\n"""
               """\n"""
               """    if context.describe_input:\n"""
               """        return %(inputs)s\n"""
               """\n"""
               """    forever = pipeforever.pipe_forever(context, None, conf=None)\n"""
               """\n""" % {'pipename':pipe['name'],
                           'inputs':unicode(sorted(pyinput))}  #todo pprint this
              )

    prev_module = []
    for module_id in module_sequence:
        module = pipe['modules'][module_id]

        #Plumb I/O

        # find the default input of this module
        input_module = "forever"
        for wire in pipe['wires']:
            # if the wire is to this module and it's the default input and it's the default output:
            if util.pythonise(pipe['wires'][wire]['tgt']['moduleid']) == module_id and pipe['wires'][wire]['tgt']['id'] == '_INPUT' and pipe['wires'][wire]['src']['id'].startswith('_OUTPUT'): # todo? this equates the outputs
                input_module = util.pythonise(pipe['wires'][wire]['src']['moduleid'])

        if module_id in pipe['embed']:
            assert input_module == "forever", "input_module of an embedded module was already set"
            input_module = "_INPUT"
        
        mod_args = [Id('context'), Id(input_module)]
        mod_kwargs = [('conf', module['conf'])]

        # set the extra inputs of this module as kwargs of this module
        for wire in pipe['wires']:
            # if the wire is to this module and it's *not* the default input and it's the default output:
            if util.pythonise(pipe['wires'][wire]['tgt']['moduleid']) == module_id and pipe['wires'][wire]['tgt']['id'] != '_INPUT' and pipe['wires'][wire]['src']['id'].startswith('_OUTPUT'): # todo? this equates the outputs
                mod_kwargs += [(util.pythonise(pipe['wires'][wire]['tgt']['id']), Id(util.pythonise(pipe['wires'][wire]['src']['moduleid'])))]

        # set the embedded module in the kwargs if this is loop module
        if module['type'] == 'loop':
            mod_kwargs += [("embed", Id("pipe_%s" % util.pythonise(module['conf']['embed']['value']['id'])))]
        
        pymodule_name = "pipe%(module_type)s" % {'module_type':module['type']}
        pymodule_generator_name = "pipe_%(module_type)s" % {'module_type':module['type']}
        if module['type'].startswith('pipe:'):
            pymodule_name = "%(module_type)s" % {'module_type':util.pythonise(module['type'])}
            pymodule_generator_name = "%(module_type)s" % {'module_type':util.pythonise(module['type'])}            

        indent = ""
        if module_id in pipe['embed']:
            #We need to wrap submodules (used by loops) so we can pass the input at runtime (as we can to subpipelines)
            pypipe += ("""    def pipe_%(module_id)s(context, _INPUT, conf=None, **kwargs):\n"""
                       """        "Submodule"\n"""     #todo insert submodule description here
                       % {'module_id':module_id}
                       )
            indent = "    "
            
        pypipe += """%(indent)s    %(module_id)s = %(pymodule_name)s.%(pymodule_generator_name)s(%(pargs)s)\n""" % {
                                                 'indent':indent,
                                                 'module_id':module_id,
                                                 'pymodule_name':pymodule_name,
                                                 'pymodule_generator_name':pymodule_generator_name,
                                                 'pargs':repr_args(mod_args+mod_kwargs)}
        if module_id in pipe['embed']:
            pypipe += """        return %(module_id)s\n""" % {'module_id':module_id}

        prev_module = module_id
        
        if context.verbose:
            print ("%s = %s(%s)" % (module_id, pymodule_generator_name,
                                    str_args([arg for arg in mod_args if arg != Id('context')]+
                                             [(key, value) for key, value in mod_kwargs if key != 'conf']+
                                             [(key, value) for key, value in mod_kwargs if key == 'conf']))
                   ).encode("utf-8")
            
    pypipe += """    return %(module_id)s\n""" % {'module_id':prev_module}
    pypipe += ("""\n"""
               """if __name__ == "__main__":\n"""
               """    context = Context()\n"""
               """    p = %(pipename)s(context, None)\n"""
               """    for i in p:\n"""
               """        print i\n""" % {'pipename':pipe['name']}
              )
        
    return pypipe