def cli(ctx, add, rm):
"""Manipulates the metadata associated to a Popper repository. A metadata
entry is an arbitrary key-value pair. Without any options, it displays all
the existing metadata entries.
Examples:
popper metadata # show all entries
popper metadata --add author='Jane Doe' --add year=2018
popper metadata --rm author
"""
config = pu.read_config()
if not add and not rm:
pu.print_yaml(config['metadata'], fg='yellow')
if add:
for kv_str in add:
kv_list = kv_str.split('=')
config['metadata'][kv_list[0]] = kv_list[1]
if rm:
for k in rm:
config['metadata'].pop(k)
pu.write_config(config)
def get_info(pipeline):
"""Prints the information about the specified pipeline.
Args:
pipeline (list): [org,repo,pipeline_name]
"""
# Checking if the popperized repositories are present or not
info = {}
org = pipeline[0]
repo = pipeline[1]
pipe = pipeline[2]
commit_url = 'https://api.github.com/repos'
commit_url += '/{}/{}/git/refs/heads/master'.format(org, repo)
r = pu.make_gh_request(commit_url,
msg="Please check if the specified pipeline exists "
"and the internet is connected.")
r = r.json()
info['name'] = pipe
info['url'] = 'https://github.com/{}/{}'.format(org, repo)
info['sha'] = r['object']['sha']
temp = {}
contents = " ".join(pu.read_gh_pipeline(org, repo, pipe)[1:])
if len(contents) != 0:
temp['description'] = contents
pu.print_yaml(info)
if 'description' in temp:
pu.print_yaml(temp)
def cli(ctx, pipeline, add, rm):
"""Manipulates the environments that are associated to a pipeline. An
environment is a docker image where a pipeline runs when 'popper run' is
executed. The 'host' environment is a special case that corresponds to
the running directly on the environment where the 'popper' command runs,
i.e. running directly on the host without docker. When a new pipeline is
created using, the default environment is 'host' (see 'popper init --help'
for more).
Examples:
popper env mypipeline # show environments for pipeline
popper env mypipeline --add ubuntu-xenial,centos-7.2
popper env mypipeline --rm host
"""
config = pu.read_config()
if not add and not rm:
pu.print_yaml(config['pipelines'][pipeline]['envs'], fg='yellow')
if add:
config['pipelines'][pipeline]['envs'] += add.split(',')
if rm:
for e in rm.split(','):
config['pipelines'][pipeline]['envs'].remove(e)
pu.write_config(config)
def cli(ctx, service, history, inplace):
"""Generates markdown for the badge of a service. Currently available
services are: CloudLab, Chameleon, Google Cloud Engine and Popper.
"""
if history and service:
raise BadArgumentUsage("--history can't be combined with other flags.")
remote_url = pu.get_remote_url()
if not remote_url:
pu.fail("Failed to infer remote URL for git repository.")
org, repo = remote_url.split('/')[-2:]
if history:
baseurl = pu.read_config().get('badge-server-url',
'http://badges.falsifiable.us')
try:
r = requests.get('{}/{}/{}/list'.format(baseurl, org, repo))
if r.json():
pu.print_yaml(r.json())
else:
pu.info("No records to show")
except requests.exceptions.RequestException:
pu.fail("Could not communicate with the badge server")
sys.exit(0)
if not service and inplace:
raise BadArgumentUsage("--inplace must be given with --service")
if service is None:
pu.fail('Please specify a service name.')
if service not in services:
pu.fail('Unknown service {}.'.format(service))
if service == 'popper':
org, repo = remote_url.split('/')[-2:]
markup = '[]({})'.format(
services[service][0], services[service][1].format(org, repo),
services[service][2])
else:
markup = '[]({})'.format(*services[service])
if not inplace:
pu.info(markup)
sys.exit(0)
try:
os.chdir(pu.get_project_root())
with open('README.md', 'r+') as f:
content = f.read()
f.seek(0, 0)
f.write(markup + '\n\n' + content)
except IOError as e:
if e.errno == ENOENT:
pu.fail("README.md does not exist at the root of the project")
def cli(ctx, pipeline, add, rm):
"""Define or remove executions of a pipeline."""
config, pipeline_config = pu.read_config(pipeline)
if add and rm:
raise UsageError("Both add and rm cannot be given at the same time. "
"See popper env-vars --help for more information.")
if add:
env_vars = pipeline_config.get('vars', [])
vars_add = {}
for var in add:
key, val = var.split('=')
vars_add[key] = val
env_vars.append(vars_add)
pu.update_config(pipeline, vars=env_vars)
elif rm:
env_vars = pipeline_config.get('vars', None)
if not env_vars:
pu.fail("No environment variables defined for this pipeline.")
vars_del = {}
for var in rm:
key, val = var.split('=')
vars_del[key] = val
index = -1
for vars in env_vars:
if len(vars.keys()) != len(vars_del.keys()):
continue
else:
successful = True
for key in vars_del:
if vars[key] != vars_del[key]:
successful = False
if successful:
index = env_vars.index(vars)
break
if index != -1:
env_vars.pop(index)
pu.update_config(pipeline, vars=env_vars)
else:
pu.fail("The environment variable list does "
"not exist for this pipeline.")
else:
try:
env_vars = pipeline_config['vars']
if len(env_vars) == 0:
raise KeyError
pu.print_yaml(env_vars)
except KeyError:
pu.info("No environment variables defined for this pipeline.")
def cli(ctx, pipeline, set):
"""View or change the stages of a pipeline.
"""
config = pu.read_config()
if pipeline in config['pipelines']:
if set:
config['pipelines'][pipeline]['stages'] = set.split(',')
pu.write_config(config)
pu.info("\nStages:", fg="yellow")
pu.print_yaml(config['pipelines'][pipeline]['stages'], fg="white")
else:
pu.fail("The pipeline {} is not defined. \nSee popper.yml file to see "
"which pipelines are defined.".format(pipeline))
def cli(ctx, pipeline, add, rm, ls):
"""Manipulates the environments that are associated to a pipeline. An
environment is a docker image where a pipeline runs when 'popper run' is
executed. The 'host' environment is a special case that corresponds to
the running directly on the environment where the 'popper' command runs,
i.e. running directly on the host without docker. When a new pipeline is
created using, the default environment is 'host' (see 'popper init --help'
for more).
Examples:
popper env mypipeline # show environments for pipeline
popper env mypipeline --add ubuntu-xenial,centos-7.2
popper env mypipeline --rm host
"""
config = pu.read_config()
if not add and not rm and not ls:
if not pipeline:
raise BadArgumentUsage('Expecting name of a pipeline')
if pipeline not in config['pipelines']:
pu.fail("Pipeline '{}' not found in .popper.yml".format(pipeline))
pu.print_yaml(config['pipelines'][pipeline]['envs'], fg='yellow')
sys.exit(0)
if add:
config['pipelines'][pipeline]['envs'] += add.split(',')
if rm:
for e in rm.split(','):
config['pipelines'][pipeline]['envs'].remove(e)
if ls:
try:
response = requests.get("https://hub.docker.com/v2/repositories/"
"falsifiable/poppercheck/tags")
environments = []
for result in response.json()['results']:
environments.append(result['name'])
pu.info('environments:')
pu.print_yaml(environments)
except requests.exceptions.RequestException as e:
click.echo(click.style("Error: " + str(e), fg='red'), err=True)
pu.write_config(config)
def cli(ctx):
"""Used to list down the avaliable pipelines
in a popper repository
Example:
> popper ls
- experiment-1
- experiment-2
"""
pipelines = list(pu.read_config()['pipelines'].keys())
if 'paper' in pipelines:
pipelines.remove('paper')
if len(pipelines) == 0:
pu.info("There are no pipelines in this repository", fg="red")
else:
pu.info("The available pipelines are :\n", fg="cyan")
pu.print_yaml(pipelines, fg="cyan")
def get_info(query):
# Checking if the popperized repositories are present or not
info = {}
org = query[0]
repo = query[1]
pipe = query[2]
commit_url = 'https://api.github.com/repos/{}/{}/commits'.format(org, repo)
r = requests.get(commit_url)
if r.status_code == 200:
commits = r.json()
info['name'] = pipe
info['url'] = 'https://github.com/{}/{}'.format(org, repo)
if len(commits) > 0 and isinstance(commits[0], type({})):
info['version'] = commits[0].get('sha')
pu.print_yaml(info)
else:
pu.fail("Please check if the specified pipeline exists " +
" and the internet is connected")
def cli(ctx, keywords, skip_update, add, rm, ls, include_readme):
"""Searches for pipelines on GitHub matching the given keyword(s).
The list of repositories or organizations scraped for Popper pipelines is
specified in the 'popperized' list in the .popper.yml file. By default,
https://github.com/popperized is added to the configuration.
If no keywords are specified, a list of all the pipelines from all
organizations (in the .popper.yml file) and repositories will be returned.
Example:
popper search quiho
would result in:
popperized/quiho-popper
To add or remove orgs/repos to/from the 'popperized' ,
use the --add and --rm flags while searching.
popper search --add org/repo
To remove an organization/person do:
popper search --rm org/repo
To view the list repositories that are available to the search command:
popper search --ls
"""
if (rm or add or ls) and (keywords):
raise BadArgumentUsage(
"'add', 'rm' and 'ls' flags cannot be combined with others.")
project_root = pu.get_project_root()
config = pu.read_config()
popperized_list = config['popperized']
if add:
add = 'github/' + add
if add not in popperized_list:
popperized_list.append(add)
config['popperized'] = popperized_list
pu.write_config(config)
sys.exit(0)
if rm:
rm = 'github/' + rm
if rm in popperized_list:
popperized_list.remove(rm)
config['popperized'] = popperized_list
pu.write_config(config)
sys.exit(0)
result = [] # to store the result of the search query as a list
if ls:
for p in popperized_list:
if p.count('/') == 1:
org_name = p.split('/')[1]
org_url = ('https://api.github.com/users/{}/repos')
org_url = org_url.format(org_name)
response = pu.make_gh_request(org_url)
repos = response.json()
temp = [r["full_name"] for r in repos]
result.extend(temp)
else:
result.extend(p[7:])
if len(result) > 0:
pu.info("The list of available poppperized repositories are:\n")
pu.print_yaml(result)
sys.exit()
else:
fail_msg = "There are no popperized repositores available"
"for search. Use the --add flag to add an org/repo."
pu.fail(fail_msg)
sys.exit(0)
search_params = {}
if not keywords: # checks if the query is empty or not
search_params['empty_query'] = True
else:
search_params['empty_query'] = False
cache_dir = os.path.join(project_root, '.cache')
search_params["keywords"] = keywords
search_params["cache_dir"] = cache_dir
search_params["skip_update"] = True if skip_update else False
search_params["in_readme"] = True if include_readme else False
if not os.path.exists(cache_dir):
os.makedirs(cache_dir)
for popperized in popperized_list:
if popperized.count('/') == 1:
# it is an organization
org_name = popperized.split('/')[1]
repos = ""
if not skip_update:
org_url = (
'https://api.github.com/users/{}/repos'.format(org_name))
response = pu.make_gh_request(org_url)
with open(os.path.join(cache_dir, org_name + '_repos.json'),
'w') as f:
json.dump(response.json(), f)
try:
with open(os.path.join(cache_dir, org_name + '_repos.json'),
'r') as f:
repos = json.load(f)
except FileNotFoundError:
pu.fail('No cached metadata has been downloaded')
with click.progressbar(
repos,
show_eta=False,
label='Searching in ' + org_name,
bar_template='[%(bar)s] %(label)s | %(info)s',
show_percent=True) as bar:
for r in bar:
if search_params["empty_query"]:
temp = ' {}/{}'\
.format(org_name, r['name'])
result.append(temp)
elif l_distance(r["name"].lower(), keywords.lower()) < 1:
temp = ' {}/{}' \
.format(org_name, r['name'])
result.append(temp)
else:
search_params["repo_url"] = r["url"]
search_params["uname"] = org_name
result.extend(search_pipeline(search_params))
else:
# it is a repository
user, repo = popperized.split('/')[1:]
repo_url = ('https://api.github.com/repos/{}/{}'.format(
user, repo))
search_params["repo_url"] = repo_url
search_params["uname"] = user
pu.info("Searching in repository : {}".format(repo))
result.extend(search_pipeline(search_params))
if len(result) != 0:
pu.info("\nSearch results:\n", fg="green")
for res in result:
pu.info("> " + res + "\n")
if search_params["in_readme"]:
pu.info("Use popper info command to view the"
" details of a pipeline. See popper info --"
"help")
else:
pu.fail("Unable to find any matching pipelines")
def cli(ctx, pipeline_name, skip_update):
"""Searches for the specified pipelines inside the GitHub
organizations specified in the .popper.yml file.
The organization at -> https://github.com/popperized is present
by default.
If no pipeline name is specified by the user, then a list of all the
pipelines from all the listed organizations and their respective
repositories will be returned.
Example:
popper search quiho
would result in:
popperized/quiho-popper
"""
project_root = pu.get_project_root()
empty_query = False
if not pipeline_name: # checks if the query is empty or not
empty_query = True
POPPER_GITHUB_API_TOKEN = ""
# check if the github personal access token has been specified by the user
if 'POPPER_GITHUB_API_TOKEN' in os.environ:
POPPER_GITHUB_API_TOKEN = os.environ['POPPER_GITHUB_API_TOKEN']
config = pu.read_config()
popperized_list = config['popperized']
result = [] # to store the result of the search query as a list
cache_dir = os.path.join(project_root, '.cache')
if not os.path.exists(cache_dir):
os.makedirs(cache_dir)
for popperized in popperized_list:
if popperized.count('/') == 1:
# it is an organization
org_name = popperized.split('/')[1]
repos = ""
if not skip_update:
org_url = ('https://api.github.com/users/{}/repos'
.format(org_name))
headers = {}
if POPPER_GITHUB_API_TOKEN != "":
headers = {
'Authorization': 'token %s' % POPPER_GITHUB_API_TOKEN
}
response = requests.get(org_url, headers=headers)
if response.status_code != 200:
pu.fail("Unable to connect. Please check your network"
" and try again. Response code = {} {}"
.format(response.status_code, org_url))
#
with open(os.path.join(cache_dir, org_name + '_repos.json'),
'w') as f:
json.dump(response.json(), f)
try:
with open(os.path.join(cache_dir, org_name + '_repos.json'),
'r') as f:
repos = json.load(f)
except FileNotFoundError:
pu.fail('No cached metadata has been downloaded')
with click.progressbar(
repos,
show_eta=False,
label='Searching {}'.format(org_name),
bar_template='[%(bar)s] %(label)s | %(info)s',
show_percent=True) as bar:
for r in bar:
if l_distance(r["name"].lower(),
pipeline_name.lower()) < 1:
temp = ' {}/{}' \
.format(org_name, r['name'])
result.append(temp)
else:
result.extend(
search_pipeline(
r["url"],
pipeline_name,
org_name,
empty_query,
POPPER_GITHUB_API_TOKEN,
cache_dir,
skip_update))
else:
# it is a repository
user, repo = popperized.split('/')[1:]
repo_url = ('https://api.github.com/repos/{}/{}'
.format(user, repo))
headers = {}
pu.info("Searching {}".format(repo))
result.extend(
search_pipeline(repo_url, pipeline_name,
user, empty_query, POPPER_GITHUB_API_TOKEN,
cache_dir, skip_update)
)
if len(result) != 0:
pu.info("Search results:\n", fg="green")
pu.print_yaml(result)
else:
pu.fail("Unable to find any matching pipelines")
def cli(ctx, keywords, skip_update, add, rm, ls, include_readme):
"""Searches for pipelines on Github matching the given keyword(s).
The list of repositories or organizations scraped for pipelines is
specified in the 'popperized' list in the .popper.yml file. By default,
https://github.com/popperized is added to the list.
If no keywords are specified, a list of all the pipelines from all
organizations (in the .popper.yml file) and repositories will be shown.
This commands makes use of Github's API, which has a limit on the number of
requests per hour that an unauthenticated user can make. If you reach this
limit, you can provide a Github API token via a POPPER_GITHUB_API_TOKEN
environment variable. If defined, this variable is used to obtain the token
when executing HTTP requests.
Example:
popper search quiho
Would result in:
popperized/quiho-popper/single-node
The format of search output is <org>/<repo>/<pipeline-name>. To add
organizations or repositories to the list of pipeline sources:
popper search --add org/repo
popper search --add entireorg
To remove one:
popper search --rm org/repo
popper search --rm entireorg
To view the list repositories that are available to the search command:
popper search --ls
"""
if ((rm and add) or (rm and ls) or (add and ls)):
raise BadArgumentUsage("Only one of 'add', 'rm' and 'ls' accepted.")
if (rm or add or ls) and keywords:
raise BadArgumentUsage("Search cannot be combined with other flags.")
config = pu.read_config()
sources = pu.get_search_sources(config)
if add:
if len(add.split('/')) > 2:
pu.fail("Bad source naming format. See 'popper search --help'.")
if add in sources:
pu.info('{} is already a search source.'.format(add))
sys.exist(0)
sources.append(add)
config['search_sources'] = sources
pu.write_config(config)
sys.exit(0)
if rm:
if rm not in sources:
pu.info("'{}' is not a search source.".format(rm))
sys.exit(0)
sources.remove(rm)
config['search_sources'] = sources
pu.write_config(config)
sys.exit(0)
if len(sources) == 0:
pu.fail('No source for popper pipelines defined! Add one first.')
pipeline_meta = pu.fetch_pipeline_metadata(skip_update)
result = search_pipelines(pipeline_meta, keywords, include_readme)
pu.info('Matching pipelines:')
pu.print_yaml(result)
def cli(ctx, pipeline, add, rm):
"""Define or remove executions of a pipeline."""
if not pipeline:
get_pipe = pu.in_pipeline(name=True)
if get_pipe is not None:
pipeline = get_pipe
else:
pu.fail("This is not a pipeline")
config, pipeline_config = pu.read_config(pipeline)
if add and rm:
raise UsageError("Both add and rm cannot be given at the same time. "
"See popper env-vars --help for more information.")
if add:
env_vars = pipeline_config.get('parameters', [])
vars_add = {}
for var in add:
key, val = var.split('=')
vars_add[key] = val
env_vars.append(vars_add)
pu.update_config(pipeline, parameters=env_vars)
sys.exit(0)
if rm:
env_vars = pipeline_config.get('parameters', None)
if not env_vars:
pu.fail("No parameters defined for this pipeline.")
vars_del = {}
for var in rm:
key, val = var.split('=')
vars_del[key] = val
index = -1
for vars in env_vars:
if len(vars.keys()) != len(vars_del.keys()):
continue
else:
successful = True
for key in vars_del:
if vars[key] != vars_del[key]:
successful = False
if successful:
index = env_vars.index(vars)
break
if index == -1:
pu.fail("Unable to find this parametrization in this pipeline.")
env_vars.pop(index)
pu.update_config(pipeline, parameters=env_vars)
sys.exit(0)
try:
env_vars = pipeline_config['parameters']
if len(env_vars) == 0:
raise KeyError
pu.print_yaml(env_vars)
except KeyError:
pu.info("No parameterization defined for this pipeline.")
def cli(ctx, pipeline, add, rm, ls, argument):
"""Manipulates the environments that are associated to a pipeline. An
environment is a docker image where a pipeline runs when 'popper run' is
executed. The 'host' environment is a special case that corresponds to
the running directly on the environment where the 'popper' command runs,
i.e. running directly on the host without docker. When a new pipeline is
created using, the default environment is 'host' (see 'popper init --help'
for more).
Examples:
popper env mypipeline # show environments for pipeline
popper env mypipeline --add ubuntu-xenial,centos-7.2
popper env mypipeline --rm host
:argument Used to pass an argument to Docker through popper.
Can be given multiple times (Ignored for 'host').
An example of usage is as follows:
popper env mypipeline --add debian-9 -arg --runtime=runc -arg --ipc=host
This will add to the environment 'debian-9' the set of
arguments runtime=runc and ipc=host.
"""
config = pu.read_config()
if ls:
try:
response = requests.get("https://hub.docker.com/v2/repositories/"
"falsifiable/popper/tags")
environments = []
for result in response.json()['results']:
environments.append(result['name'])
pu.info('environments:')
pu.print_yaml(environments)
except requests.exceptions.RequestException as e:
click.echo(click.style("Error: " + str(e), fg='red'), err=True)
sys.exit(0)
if not pipeline:
get_pipe = pu.in_pipeline(name=True)
if get_pipe is not None:
pipeline = get_pipe
else:
pu.fail("This is not a pipeline")
if not add and not rm:
if pipeline not in config['pipelines']:
pu.fail("Pipeline '{}' not found in .popper.yml".format(pipeline))
pu.print_yaml(config['pipelines'][pipeline]['envs'], fg='yellow')
sys.exit(0)
envs = config['pipelines'][pipeline]['envs']
args = set(argument)
if add:
elems = add.split(',')
environments = set(elems) - set(envs)
envs.update({env: {'args': []} for env in environments})
for env in elems:
envs[env]['args'] = args
if rm:
for env in rm.split(','):
if env in envs:
envs.pop(env)
else:
pu.warn('Environment {} not found in {}'.format(env, pipeline))
config['pipelines'][pipeline]['envs'] = envs
pu.write_config(config)