def _load(self, pipeline, pipe_name, value=0, check=1):
length = len(pipeline)
switch = {1: '>', -1: '<', 0: '=='}
# compare pipeline length to baseline value and obtain the following
# result
# 1 if length > value
# -1 if length < value
# 0 if length == value
compared = cmp(length, value)
try:
module = import_module('tests.pypipelines.%s' % pipe_name)
except ImportError:
parent = p.dirname(__file__)
pipe_file_name = p.join(parent, 'pipelines', '%s.json' % pipe_name)
with open(pipe_file_name) as f:
pjson = f.read()
pydeps = extract_dependencies(loads(pjson))
else:
pipe_generator = getattr(module, pipe_name)
pydeps = extract_dependencies(pipe_generator=pipe_generator)
print 'pipeline length %s %i, but expected %s %i.' % (
switch.get(compared), value, switch.get(check), value)
print 'Modules used in %s: %s' % (pipe_name, pydeps)
# assert that pipeline length is as expected
return self.assertEqual(compared, check)
def _load(self, pipeline, pipe_name, value=0, check=1):
length = len(pipeline)
switch = {1: '>', -1: '<', 0: '=='}
# compare pipeline length to baseline value and obtain the following
# result
# 1 if length > value
# -1 if length < value
# 0 if length == value
compared = cmp(length, value)
try:
module = import_module('tests.pypipelines.%s' % pipe_name)
except ImportError:
parent = p.dirname(__file__)
pipe_file_name = p.join(parent, 'pipelines', '%s.json' % pipe_name)
with open(pipe_file_name) as f:
pjson = f.read()
pydeps = extract_dependencies(loads(pjson))
else:
pipe_generator = getattr(module, pipe_name)
pydeps = extract_dependencies(pipe_generator=pipe_generator)
print 'pipeline length %s %i, but expected %s %i.' % (
switch.get(compared), value, switch.get(check), value)
print 'Modules used in %s: %s' % (pipe_name, pydeps)
# assert that pipeline length is as expected
return self.assertEqual(compared, check)
def stringify_pipe(context, pipe, pipe_def):
"""Convert a pipe into Python script
"""
module_ids = topological_sort(pipe['graph'])
kwargs = {
'module_ids': module_ids,
'module_names': utils.gen_names(module_ids, pipe),
'pipe_names': utils.gen_names(module_ids, pipe, 'pipe'),
}
zipped = _get_zipped(context, pipe, **kwargs)
modules = list(_gen_string_modules(context, pipe, zipped))
pydeps = utils.extract_dependencies(pipe_def)
pyinput = utils.extract_input(pipe_def)
env = Environment(loader=PackageLoader('pipe2py'))
template = env.get_template('pypipe.txt')
tmpl_kwargs = {
'modules': modules,
'pipe_name': pipe['name'],
'inputs': unicode(pyinput),
'dependencies': unicode(pydeps),
'embedded_pipes': pipe['embed'],
'last_module': module_ids[-1],
}
return template.render(**tmpl_kwargs)
def build_pipeline(context, pipe, pipe_def):
"""Convert a pipe into an executable Python pipeline
If context.describe_input or context.describe_dependencies then just
return that instead of the pipeline
Note: any subpipes must be available to import from pipe2py.pypipelines
"""
module_ids = topological_sort(pipe['graph'])
pydeps = utils.extract_dependencies(pipe_def)
pyinput = utils.extract_input(pipe_def)
if not (context.describe_input or context.describe_dependencies):
kwargs = {
'module_ids': module_ids,
'module_names': utils.gen_names(module_ids, pipe),
'pipe_names': utils.gen_names(module_ids, pipe, 'pipe'),
}
zipped = _get_zipped(context, pipe, **kwargs)
steps = _get_steps(context, pipe, zipped)
if context.describe_input and context.describe_dependencies:
pipeline = [{'inputs': pyinput, 'dependencies': pydeps}]
elif context.describe_input:
pipeline = pyinput
elif context.describe_dependencies:
pipeline = pydeps
else:
pipeline = steps[module_ids[-1]]
for i in pipeline:
yield i
def stringify_pipe(context, pipe, pipe_def):
"""Convert a pipe into Python script
"""
module_ids = topological_sort(pipe['graph'])
kwargs = {
'module_ids': module_ids,
'module_names': utils.gen_names(module_ids, pipe),
'pipe_names': utils.gen_names(module_ids, pipe, 'pipe'),
}
zipped = _get_zipped(context, pipe, **kwargs)
modules = list(_gen_string_modules(context, pipe, zipped))
pydeps = utils.extract_dependencies(pipe_def)
pyinput = utils.extract_input(pipe_def)
env = Environment(loader=PackageLoader('pipe2py'))
template = env.get_template('pypipe.txt')
tmpl_kwargs = {
'modules': modules,
'pipe_name': pipe['name'],
'inputs': unicode(pyinput),
'dependencies': unicode(pydeps),
'embedded_pipes': pipe['embed'],
'last_module': module_ids[-1],
}
return template.render(**tmpl_kwargs)
def build_pipeline(context, pipe, pipe_def):
"""Convert a pipe into an executable Python pipeline
If context.describe_input or context.describe_dependencies then just
return that instead of the pipeline
Note: any subpipes must be available to import from pipe2py.pypipelines
"""
module_ids = topological_sort(pipe['graph'])
pydeps = utils.extract_dependencies(pipe_def)
pyinput = utils.extract_input(pipe_def)
if not (context.describe_input or context.describe_dependencies):
kwargs = {
'module_ids': module_ids,
'module_names': utils.gen_names(module_ids, pipe),
'pipe_names': utils.gen_names(module_ids, pipe, 'pipe'),
}
zipped = _get_zipped(context, pipe, **kwargs)
steps = _get_steps(context, pipe, zipped)
if context.describe_input and context.describe_dependencies:
pipeline = [{'inputs': pyinput, 'dependencies': pydeps}]
elif context.describe_input:
pipeline = pyinput
elif context.describe_dependencies:
pipeline = pydeps
else:
pipeline = steps[module_ids[-1]]
for i in pipeline:
yield i