def cli(ctx, pipeline, env, binary, clear):
""" Adds pipeline requirements to .popper.yml """
# try to get pipeline from current directory
if not pipeline:
get_pipe = pu.in_pipeline(name=True)
if get_pipe is not None:
pipeline = get_pipe
else:
pu.fail("No pipeline detected")
config = pu.read_config()
if pipeline not in config['pipelines']:
pu.fail(
'Pipeline {} does not exist. Check your .popper.yml file.'.format(
pipeline))
# merge current requirements
reqs = config['pipelines'][pipeline].get('requirements', {})
var_reqs = set([]) if clear else set(reqs.get('vars', []))
var_reqs |= set(env)
var_reqs = list(var_reqs)
reqs['vars'] = var_reqs
bin_reqs = set([]) if clear else set(reqs.get('bin', []))
bin_reqs |= set(binary)
bin_reqs = list(bin_reqs)
reqs['bin'] = bin_reqs
pu.update_config(pipeline, reqs=reqs)
def cli(ctx, name, stages, envs, existing, infer_stages):
"""Initializes a repository or a pipeline. Without an argument, this
command initializes a popper repository. If an argument is given, a
pipeline or paper folder is initialized. If the given name is 'paper',
then a 'paper' folder is created. Otherwise, a pipeline named NAME is
created and initialized inside the 'pipelines' folder.
By default, the stages of a pipeline are: setup, run, post-run, validate
and teardown. To override these, the `--stages` flag can be provided, which
expects a comma-separated list of stage names.
The teardown stage is to be provided at the end if the --stages flag is
being used.
If the --existing flag is given, the NAME argument is treated as a path to
a folder, which is assumed to contain bash scripts. --stages must be given.
"""
# check if the the teardown stage is the last stage of the pipeline
if stages and 'teardown' in stages and stages.split(',')[-1] != 'teardown':
raise BadArgumentUsage(
'--stages = Teardown should be the last stage.' +
' Consider renaming it or putting it at the end.')
project_root = pu.get_project_root()
# init repo
if name is None:
initialize_repo(project_root)
return
if not pu.is_popperized():
pu.fail("Repository has not been popperized yet. See 'init --help'")
if isdir(os.path.join(project_root, name)) and existing:
# existing pipeline
abs_path = os.path.join(project_root, name)
relative_path = name
if infer_stages:
stages = ",".join(
map(lambda x: x[:-3], sorted(glob.glob1(abs_path, '*.sh'))))
else:
initialize_existing_pipeline(abs_path, stages, envs)
elif name == 'paper':
# create a paper pipeline
abs_path = os.path.join(project_root, 'paper')
relative_path = os.path.join('paper')
initialize_paper(abs_path, envs)
else:
# new pipeline
abs_path = os.path.join(project_root, 'pipelines', name)
relative_path = os.path.join('pipelines', name)
initialize_new_pipeline(abs_path, stages, envs)
pu.update_config(name, stages, envs, relative_path)
pu.info('Initialized pipeline ' + name, fg='blue', bold=True)
def cli(ctx, pipeline, add, rm):
"""Define or remove executions of a pipeline."""
config, pipeline_config = pu.read_config(pipeline)
if add and rm:
raise UsageError("Both add and rm cannot be given at the same time. "
"See popper env-vars --help for more information.")
if add:
env_vars = pipeline_config.get('vars', [])
vars_add = {}
for var in add:
key, val = var.split('=')
vars_add[key] = val
env_vars.append(vars_add)
pu.update_config(pipeline, vars=env_vars)
elif rm:
env_vars = pipeline_config.get('vars', None)
if not env_vars:
pu.fail("No environment variables defined for this pipeline.")
vars_del = {}
for var in rm:
key, val = var.split('=')
vars_del[key] = val
index = -1
for vars in env_vars:
if len(vars.keys()) != len(vars_del.keys()):
continue
else:
successful = True
for key in vars_del:
if vars[key] != vars_del[key]:
successful = False
if successful:
index = env_vars.index(vars)
break
if index != -1:
env_vars.pop(index)
pu.update_config(pipeline, vars=env_vars)
else:
pu.fail("The environment variable list does "
"not exist for this pipeline.")
else:
try:
env_vars = pipeline_config['vars']
if len(env_vars) == 0:
raise KeyError
pu.print_yaml(env_vars)
except KeyError:
pu.info("No environment variables defined for this pipeline.")
def cli(ctx, name, stages, envs, existing):
"""Initializes a repository or a pipeline. Without an argument, this
command initializes a popper repository. If an argument is given, a
pipeline or paper folder is initialized. If the given name is 'paper',
then a 'paper' folder is created. Otherwise, a pipeline named NAME is
created and initialized inside the 'pipelines' folder.
By default, the stages of a pipeline are: setup, run, post-run, validate
and teardown. To override these, the `--stages` flag can be provided, which
expects a comma-separated list of stage names.
If the --existing flag is given, the NAME argument is treated as a path to
a folder, which is assumed to contain bash scripts. --stages must be given.
"""
project_root = pu.get_project_root()
# init repo
if name is None:
initialize_repo(project_root)
return
if not pu.is_popperized():
pu.fail("Repository has not been popperized yet. See 'init --help'")
if isdir(os.path.join(project_root, name)) and existing:
# existing pipeline
abs_path = os.path.join(project_root, name)
relative_path = name
initialize_existing_pipeline(abs_path, stages, envs)
elif name == 'paper':
# create a paper pipeline
abs_path = os.path.join(project_root, 'paper')
relative_path = os.path.join('paper')
initialize_paper(abs_path, envs)
else:
# new pipeline
abs_path = os.path.join(project_root, 'pipelines', name)
relative_path = os.path.join('pipelines', name)
initialize_new_pipeline(abs_path, stages, envs)
pu.update_config(name, stages, envs, relative_path)
pu.info('Initialized pipeline ' + name, fg='blue', bold=True)
def cli(ctx, pipeline, add, rm):
"""Define or remove executions of a pipeline."""
if not pipeline:
get_pipe = pu.in_pipeline(name=True)
if get_pipe is not None:
pipeline = get_pipe
else:
pu.fail("This is not a pipeline")
config, pipeline_config = pu.read_config(pipeline)
if add and rm:
raise UsageError("Both add and rm cannot be given at the same time. "
"See popper env-vars --help for more information.")
if add:
env_vars = pipeline_config.get('parameters', [])
vars_add = {}
for var in add:
key, val = var.split('=')
vars_add[key] = val
env_vars.append(vars_add)
pu.update_config(pipeline, parameters=env_vars)
sys.exit(0)
if rm:
env_vars = pipeline_config.get('parameters', None)
if not env_vars:
pu.fail("No parameters defined for this pipeline.")
vars_del = {}
for var in rm:
key, val = var.split('=')
vars_del[key] = val
index = -1
for vars in env_vars:
if len(vars.keys()) != len(vars_del.keys()):
continue
else:
successful = True
for key in vars_del:
if vars[key] != vars_del[key]:
successful = False
if successful:
index = env_vars.index(vars)
break
if index == -1:
pu.fail("Unable to find this parametrization in this pipeline.")
env_vars.pop(index)
pu.update_config(pipeline, parameters=env_vars)
sys.exit(0)
try:
env_vars = pipeline_config['parameters']
if len(env_vars) == 0:
raise KeyError
pu.print_yaml(env_vars)
except KeyError:
pu.info("No parameterization defined for this pipeline.")