def cli(ctx, pipeline, env, binary, clear):
""" Adds pipeline requirements to .popper.yml """
# try to get pipeline from current directory
if not pipeline:
get_pipe = pu.in_pipeline(name=True)
if get_pipe is not None:
pipeline = get_pipe
else:
pu.fail("No pipeline detected")
config = pu.read_config()
if pipeline not in config['pipelines']:
pu.fail(
'Pipeline {} does not exist. Check your .popper.yml file.'.format(
pipeline))
# merge current requirements
reqs = config['pipelines'][pipeline].get('requirements', {})
var_reqs = set([]) if clear else set(reqs.get('vars', []))
var_reqs |= set(env)
var_reqs = list(var_reqs)
reqs['vars'] = var_reqs
bin_reqs = set([]) if clear else set(reqs.get('bin', []))
bin_reqs |= set(binary)
bin_reqs = list(bin_reqs)
reqs['bin'] = bin_reqs
pu.update_config(pipeline, reqs=reqs)
def cli(ctx, pipeline, set):
"""View or change the stages of a pipeline.
"""
if not pipeline:
get_pipe = pu.in_pipeline(name=True)
if get_pipe is not None:
pipeline = get_pipe
else:
pu.fail("This is not a pipeline")
config = pu.read_config()
if pipeline in config['pipelines']:
if set:
config['pipelines'][pipeline]['stages'] = set.split(',')
pu.write_config(config)
pu.info("\nStages:", fg="yellow")
pu.print_yaml(config['pipelines'][pipeline]['stages'], fg="white")
else:
pu.fail("The pipeline {} is not defined. \nSee popper.yml file to see "
"which pipelines are defined.".format(pipeline))
def cli(ctx, pipeline, add, rm, ls, argument):
"""Manipulates the environments that are associated to a pipeline. An
environment is a docker image where a pipeline runs when 'popper run' is
executed. The 'host' environment is a special case that corresponds to
the running directly on the environment where the 'popper' command runs,
i.e. running directly on the host without docker. When a new pipeline is
created using, the default environment is 'host' (see 'popper init --help'
for more).
Examples:
popper env mypipeline # show environments for pipeline
popper env mypipeline --add ubuntu-xenial,centos-7.2
popper env mypipeline --rm host
:argument Used to pass an argument to Docker through popper.
Can be given multiple times (Ignored for 'host').
An example of usage is as follows:
popper env mypipeline --add debian-9 -arg --runtime=runc -arg --ipc=host
This will add to the environment 'debian-9' the set of
arguments runtime=runc and ipc=host.
"""
config = pu.read_config()
if ls:
try:
response = requests.get("https://hub.docker.com/v2/repositories/"
"falsifiable/popper/tags")
environments = []
for result in response.json()['results']:
environments.append(result['name'])
pu.info('environments:')
pu.print_yaml(environments)
except requests.exceptions.RequestException as e:
click.echo(click.style("Error: " + str(e), fg='red'), err=True)
sys.exit(0)
if not pipeline:
get_pipe = pu.in_pipeline(name=True)
if get_pipe is not None:
pipeline = get_pipe
else:
pu.fail("This is not a pipeline")
if not add and not rm:
if pipeline not in config['pipelines']:
pu.fail("Pipeline '{}' not found in .popper.yml".format(pipeline))
pu.print_yaml(config['pipelines'][pipeline]['envs'], fg='yellow')
sys.exit(0)
envs = config['pipelines'][pipeline]['envs']
args = set(argument)
if add:
elems = add.split(',')
environments = set(elems) - set(envs)
envs.update({env: {'args': []} for env in environments})
for env in elems:
envs[env]['args'] = args
if rm:
for env in rm.split(','):
if env in envs:
envs.pop(env)
else:
pu.warn('Environment {} not found in {}'.format(env, pipeline))
config['pipelines'][pipeline]['envs'] = envs
pu.write_config(config)
def cli(ctx, pipeline, add, rm):
"""Define or remove executions of a pipeline."""
if not pipeline:
get_pipe = pu.in_pipeline(name=True)
if get_pipe is not None:
pipeline = get_pipe
else:
pu.fail("This is not a pipeline")
config, pipeline_config = pu.read_config(pipeline)
if add and rm:
raise UsageError("Both add and rm cannot be given at the same time. "
"See popper env-vars --help for more information.")
if add:
env_vars = pipeline_config.get('parameters', [])
vars_add = {}
for var in add:
key, val = var.split('=')
vars_add[key] = val
env_vars.append(vars_add)
pu.update_config(pipeline, parameters=env_vars)
sys.exit(0)
if rm:
env_vars = pipeline_config.get('parameters', None)
if not env_vars:
pu.fail("No parameters defined for this pipeline.")
vars_del = {}
for var in rm:
key, val = var.split('=')
vars_del[key] = val
index = -1
for vars in env_vars:
if len(vars.keys()) != len(vars_del.keys()):
continue
else:
successful = True
for key in vars_del:
if vars[key] != vars_del[key]:
successful = False
if successful:
index = env_vars.index(vars)
break
if index == -1:
pu.fail("Unable to find this parametrization in this pipeline.")
env_vars.pop(index)
pu.update_config(pipeline, parameters=env_vars)
sys.exit(0)
try:
env_vars = pipeline_config['parameters']
if len(env_vars) == 0:
raise KeyError
pu.print_yaml(env_vars)
except KeyError:
pu.info("No parameterization defined for this pipeline.")
def cli(ctx, pipeline):
"""Generates a workflow diagram corresponding to a Popper pipeline, in the
.dot format. The string defining the graph is printed to stdout so it can
be piped into other tools. For example, to generate a png file, one can
make use of the graphviz CLI tools:
popper workflow mypipe | dot -T png -o mypipe.png
"""
pipes = pu.read_config()['pipelines']
if not pipeline:
get_pipe = pu.in_pipeline(name=True)
if get_pipe is not None:
pipeline = get_pipe
else:
pu.fail("This is not a pipeline")
if pipeline not in pipes:
pu.fail("Cannot find pipeline {} in .popper.yml".format(pipeline))
project_root = pu.get_project_root()
abs_path = os.path.join(project_root, pipes[pipeline]['path'])
transformer = ObtainDotGraph()
parser = Lark(bash_grammar,
parser='lalr',
lexer='contextual',
transformer=transformer)
for stage in pipes[pipeline]['stages']:
transformer.current_stage = stage
stage_file = pu.get_filename(abs_path, stage)
with open(stage_file, 'r') as f:
s = f.read()
parser.parse(s)
cs = transformer.comment_stack
cs = remove_redundant_if(cs)
# print(cs)
print('digraph pipeline {')
curr_node = None
prev_node = None
node_id = 's{}'.format(0)
curr_if_node = None
if_created = False
if_index = None
for i, item in enumerate(cs):
if item == '[wf]#stage#':
prev_item = cs[i - 1]
next_item = cs[i + 1]
label = '"{' + '{} | {}'.format(next_item, prev_item) + '}"'
curr_node = (next_item.replace('-', '_')).replace('.sh', ' ')
# create the stage node
print('{} [{}];'.format(curr_node, 'shape=record, label=' + label))
if prev_node:
print('{} -> {};'.format(prev_node, curr_node))
prev_node = curr_node
continue
# initialize the if-node
elif item == '[wf]#if#':
if_created = False
c = 'condition'
if i > 1 and (not cs[i - 1].startswith('[wf]#')
and '.sh' not in cs[i - 1]):
c += ' : {}'.format(cs[i - 1])
if_index = i - 1
curr_if_node = node_id
# inside if-elif-else construct
elif (item == '[wf]#else#' or item == '[wf]#elif#'
or item == '[wf]#fi#'):
if not cs[i - 1].startswith('[wf]#'):
if not if_created:
if_created, node_id = create_if_node(node_id, c, prev_node)
print('{} [shape=record, label="{}"];'.format(
node_id, cs[i - 1]))
print('{} -> {};'.format(curr_if_node, node_id))
node_id = increment(node_id)
if item == '[wf]#fi':
if_created = False
continue
# inside loop
elif item == '[wf]#done':
c = 'loop'
if not cs[i - 1].startswith('[wf]#') and '.sh' not in cs[i - 1]:
c += ' : {}'.format(cs[i - 1])
print('{} [shape=record,label="{}"];'.format(node_id, c))
print('{} -> {};'.format(prev_node, node_id))
node_id = increment(node_id)
# is a comment outside any control structures
elif not item.startswith('[wf]#') and '.sh' not in item:
if i == len(cs) - 1 and not cs[i - 1] == '[wf]#stage#':
print('{} [shape=record,label="{}"];'.format(node_id, item))
print('{} -> {};'.format(prev_node, node_id))
node_id = increment(node_id)
elif i < len(cs) - 1:
if (not cs[i + 1].startswith('[wf]#')
and not cs[i - 1] == '[wf]#stage#'):
print('{} [shape=record,label="{}"];'.format(
node_id, item))
print('{} -> {};'.format(prev_node, node_id))
node_id = increment(node_id)
print('}')