Exemple #1
0
    def _load(self, pipeline, pipe_name, value=0, check=1):
        length = len(pipeline)
        switch = {1: '>', -1: '<', 0: '=='}

        # compare pipeline length to baseline value and obtain the following
        # result
        # 1 if length > value
        # -1 if length < value
        # 0 if length == value
        compared = cmp(length, value)

        try:
            module = import_module('tests.pypipelines.%s' % pipe_name)
        except ImportError:
            parent = p.dirname(__file__)
            pipe_file_name = p.join(parent, 'pipelines', '%s.json' % pipe_name)

            with open(pipe_file_name) as f:
                pjson = f.read()

            pydeps = extract_dependencies(loads(pjson))
        else:
            pipe_generator = getattr(module, pipe_name)
            pydeps = extract_dependencies(pipe_generator=pipe_generator)

        print 'pipeline length %s %i, but expected %s %i.' % (
            switch.get(compared), value, switch.get(check), value)

        print 'Modules used in %s: %s' % (pipe_name, pydeps)

        # assert that pipeline length is as expected
        return self.assertEqual(compared, check)
Exemple #2
0
    def _load(self, pipeline, pipe_name, value=0, check=1):
        length = len(pipeline)
        switch = {1: '>', -1: '<', 0: '=='}

        # compare pipeline length to baseline value and obtain the following
        # result
        # 1 if length > value
        # -1 if length < value
        # 0 if length == value
        compared = cmp(length, value)

        try:
            module = import_module('tests.pypipelines.%s' % pipe_name)
        except ImportError:
            parent = p.dirname(__file__)
            pipe_file_name = p.join(parent, 'pipelines', '%s.json' % pipe_name)

            with open(pipe_file_name) as f:
                pjson = f.read()

            pydeps = extract_dependencies(loads(pjson))
        else:
            pipe_generator = getattr(module, pipe_name)
            pydeps = extract_dependencies(pipe_generator=pipe_generator)

        print 'pipeline length %s %i, but expected %s %i.' % (
            switch.get(compared), value, switch.get(check), value)

        print 'Modules used in %s: %s' % (pipe_name, pydeps)

        # assert that pipeline length is as expected
        return self.assertEqual(compared, check)
Exemple #3
0
def stringify_pipe(context, pipe, pipe_def):
    """Convert a pipe into Python script
    """
    module_ids = topological_sort(pipe['graph'])

    kwargs = {
        'module_ids': module_ids,
        'module_names': utils.gen_names(module_ids, pipe),
        'pipe_names': utils.gen_names(module_ids, pipe, 'pipe'),
    }

    zipped = _get_zipped(context, pipe, **kwargs)
    modules = list(_gen_string_modules(context, pipe, zipped))
    pydeps = utils.extract_dependencies(pipe_def)
    pyinput = utils.extract_input(pipe_def)
    env = Environment(loader=PackageLoader('pipe2py'))
    template = env.get_template('pypipe.txt')

    tmpl_kwargs = {
        'modules': modules,
        'pipe_name': pipe['name'],
        'inputs': unicode(pyinput),
        'dependencies': unicode(pydeps),
        'embedded_pipes': pipe['embed'],
        'last_module': module_ids[-1],
    }

    return template.render(**tmpl_kwargs)
Exemple #4
0
def build_pipeline(context, pipe, pipe_def):
    """Convert a pipe into an executable Python pipeline

        If context.describe_input or context.describe_dependencies then just
        return that instead of the pipeline

        Note: any subpipes must be available to import from pipe2py.pypipelines
    """
    module_ids = topological_sort(pipe['graph'])
    pydeps = utils.extract_dependencies(pipe_def)
    pyinput = utils.extract_input(pipe_def)

    if not (context.describe_input or context.describe_dependencies):
        kwargs = {
            'module_ids': module_ids,
            'module_names': utils.gen_names(module_ids, pipe),
            'pipe_names': utils.gen_names(module_ids, pipe, 'pipe'),
        }

        zipped = _get_zipped(context, pipe, **kwargs)
        steps = _get_steps(context, pipe, zipped)

    if context.describe_input and context.describe_dependencies:
        pipeline = [{'inputs': pyinput, 'dependencies': pydeps}]
    elif context.describe_input:
        pipeline = pyinput
    elif context.describe_dependencies:
        pipeline = pydeps
    else:
        pipeline = steps[module_ids[-1]]

    for i in pipeline:
        yield i
Exemple #5
0
def stringify_pipe(context, pipe, pipe_def):
    """Convert a pipe into Python script
    """
    module_ids = topological_sort(pipe['graph'])

    kwargs = {
        'module_ids': module_ids,
        'module_names': utils.gen_names(module_ids, pipe),
        'pipe_names': utils.gen_names(module_ids, pipe, 'pipe'),
    }

    zipped = _get_zipped(context, pipe, **kwargs)
    modules = list(_gen_string_modules(context, pipe, zipped))
    pydeps = utils.extract_dependencies(pipe_def)
    pyinput = utils.extract_input(pipe_def)
    env = Environment(loader=PackageLoader('pipe2py'))
    template = env.get_template('pypipe.txt')

    tmpl_kwargs = {
        'modules': modules,
        'pipe_name': pipe['name'],
        'inputs': unicode(pyinput),
        'dependencies': unicode(pydeps),
        'embedded_pipes': pipe['embed'],
        'last_module': module_ids[-1],
    }

    return template.render(**tmpl_kwargs)
Exemple #6
0
def build_pipeline(context, pipe, pipe_def):
    """Convert a pipe into an executable Python pipeline

        If context.describe_input or context.describe_dependencies then just
        return that instead of the pipeline

        Note: any subpipes must be available to import from pipe2py.pypipelines
    """
    module_ids = topological_sort(pipe['graph'])
    pydeps = utils.extract_dependencies(pipe_def)
    pyinput = utils.extract_input(pipe_def)

    if not (context.describe_input or context.describe_dependencies):
        kwargs = {
            'module_ids': module_ids,
            'module_names': utils.gen_names(module_ids, pipe),
            'pipe_names': utils.gen_names(module_ids, pipe, 'pipe'),
        }

        zipped = _get_zipped(context, pipe, **kwargs)
        steps = _get_steps(context, pipe, zipped)

    if context.describe_input and context.describe_dependencies:
        pipeline = [{'inputs': pyinput, 'dependencies': pydeps}]
    elif context.describe_input:
        pipeline = pyinput
    elif context.describe_dependencies:
        pipeline = pydeps
    else:
        pipeline = steps[module_ids[-1]]

    for i in pipeline:
        yield i