def cli(ctx):
"""Resets a popper repository completely, removing all existing
pipelines and folders, leaving behind a newly created .popper.yml file.
Note: It only removes those files inside a pipeline folder that are also
tracked by git. Untracked files will not be deleted.
"""
msg = (
"This will remove all the pipeline files in this "
" project, do you want to continue?"
)
if(not click.confirm(msg, abort=False)):
sys.exit(0)
project_root = pu.get_project_root()
if project_root != os.getcwd():
msg = 'This command can only be executed from the project root folder'
pu.fail(msg)
config = pu.read_config()
for _, p in config['pipelines'].items():
pu.exec_cmd('git rm -r {}'.format(p['path']))
pu.write_config(pu.init_config)
content = pt.ReadMe()
content.init_project()
pu.info("Reset complete", fg="cyan")
def cli(ctx, pipeline):
"""Remove a popper pipeline from the user's repository effectively
to keep the pipelines folder and the .popper.yml files in sync.
"""
project_root = pu.get_project_root()
if pipeline == 'paper':
pipeline_dir = project_root
else:
pipeline_dir = os.path.join(project_root, 'pipelines')
pipeline_path = os.path.join(pipeline_dir, pipeline)
if os.path.isdir(pipeline_path):
shutil.rmtree(pipeline_path)
popper_config = pu.read_config()
del popper_config['pipelines'][pipeline]
pu.info("Pipeline {} removed successfully".format(pipeline),
fg="green")
pu.write_config(popper_config)
else:
pu.fail("Pipeline {} doesn't exists".format(pipeline))
def cli(ctx, pipeline):
"""Remove a popper pipeline from the user's repository effectively
to keep the pipelines folder and the .popper.yml files in sync.
"""
project_root = pu.get_project_root()
pipelines = pu.read_config()['pipelines']
if pipeline in pipelines:
path = pipelines[pipeline]['path']
pipeline_dir = os.path.join(
project_root,
path)
else:
pu.fail("Pipeline '{}' not in this project".format(pipeline))
if os.path.isdir(pipeline_dir):
shutil.rmtree(pipeline_dir)
popper_config = pu.read_config()
del popper_config['pipelines'][pipeline]
pu.info("Pipeline '{}' removed successfully".format(pipeline),
fg="blue")
pu.write_config(popper_config)
else:
pu.fail("Path '{}' is not a folder".format(pipeline))
def cli(ctx):
"""Synchronize your pipelines and popper.yml file if any pipeline or stage
has been deleted.
"""
popper_config = pu.read_config()
project_root = pu.get_project_root()
pipelines = popper_config['pipelines']
# Removing nonexistent pipelines from .popper.yml
for p in list(pipelines):
pipeline = pipelines[p]
pipe_path = os.path.join(project_root, pipeline['path'])
# Checking if the pipeline exists
if os.path.exists(pipe_path):
# Synchronizing stages
stages = [
x[:-3] for x in os.listdir(pipe_path) if x.endswith(".sh")
]
pipelines[p]['stages'] = stages
else:
del pipelines[p]
popper_config['pipelines'] = pipelines
pu.write_config(popper_config)
pu.info(
"\nYour popper.yml file has been updated! Run git diff to see "
"the differences.",
fg="white")
def get_access_token(service):
"""Tries to read the access token from a key file. If not present,
prompts the user for a key and also stores the key in a key file
if the user wishes."""
project_root = pu.get_project_root()
os.chdir(project_root)
try:
with open('.{}.key'.format(service), 'r') as keyfile:
encrypted_access_token = keyfile.read().strip()
passphrase = click.prompt(
'Please enter your passphrase for {}'.format(service),
hide_input=True).encode()
aes = pyaes.AESModeOfOperationCTR(generate_key(passphrase))
try:
access_token = aes.decrypt(encrypted_access_token).decode()
except UnicodeDecodeError:
pu.fail("Invalid passphrase. Please use the same passphrase "
"used at the time of encrypting the access_token.")
except FileNotFoundError:
pu.info('No access token found for {}'.format(service))
access_token = click.prompt(
'Please enter your access token for {}'.format(service))
if click.confirm('Would you like to store this key?'):
passphrase = click.prompt('Enter a strong passphrase',
hide_input=True).encode()
aes = pyaes.AESModeOfOperationCTR(generate_key(passphrase))
encrypted_access_token = aes.encrypt(access_token)
with open('.{}.key'.format(service), 'w') as keyfile:
keyfile.writelines('{}'.format(''.join(
chr(b) for b in encrypted_access_token)))
pu.info('Your key is stored in .{}.key'.format(service))
return access_token
def create_archive(self):
"""Creates an archive of the entire project folder using gzip or zip.
Returns:
Name of the archive file.
"""
cwd = os.getcwd()
project_root = pu.get_project_root()
project_name = os.path.basename(project_root)
os.chdir(project_root)
archive_file = '/tmp/' + project_name + '.tar'
cmd = 'git archive master > ' + archive_file
subprocess.check_output(cmd, shell=True)
if not self.ignore_untracked:
cmd = (
'git ls-files --others --exclude-standard -z | '
'xargs -0 tar rf ' + archive_file
)
subprocess.check_output(cmd, shell=True)
cmd = 'gzip ' + archive_file
subprocess.check_output(cmd, shell=True)
os.chdir(cwd)
return archive_file + '.gz'
def cli(ctx):
"""Synchronize your pipelines and popper.yml file if any pipeline or stage
has been deleted.
"""
pipeline_dir = os.path.join(pu.get_project_root(), 'pipelines')
popper_config = pu.read_config()
pipelines = {}
for pipeline in os.listdir(pipeline_dir):
envs = popper_config['pipelines'][pipeline]['envs']
relative_path = popper_config['pipelines'][pipeline]['path']
defined_stages = popper_config['pipelines'][pipeline]['stages']
existing_stages = []
for stage in defined_stages:
os.chdir(os.path.join(pipeline_dir, pipeline))
if os.path.exists(stage+'.sh') or os.path.exists(stage):
existing_stages.append(stage)
pipelines[pipeline] = {
'envs': envs,
'path': relative_path,
'stages': existing_stages
}
popper_config['pipelines'] = pipelines
pu.write_config(popper_config)
pu.info("\nYour popper.yml file has been updated! Run git diff to see "
"the differences.", fg="white")
def cli(ctx, pipeline):
"""Add a pipeline to your repository from the existing popperized
repositories on github. The pipeline argument is provided as owner/repo/
pipeline. For example, popper add popperized/quiho-popper/single-node
adds the single-node pipeline from the quiho-popper repository.
"""
try:
owner, repo, pipeline_name = pipeline.split('/')
except ValueError:
pu.fail("See popper add --help for more info.")
project_root = pu.get_project_root()
path = os.path.join(project_root, 'pipelines')
if os.path.exists(path):
pass
else:
os.chdir(project_root)
os.mkdir('pipelines')
dirname = pipeline_name
url = ('https://api.github.com/repos/{}/{}/contents/pipelines/{}'
.format(owner, repo, pipeline_name))
repo_config = get_config(owner, repo)
save_directory(path, dirname, url)
path = os.path.join(path, pipeline_name)
update_config(owner, repo, pipeline_name, path, repo_config)
pu.info("Pipeline {} successfully added.".format(pipeline_name)
+ " It can be viewed in the pipelines directory.",
fg="green")
def cli(ctx, pipeline, timeout, skip):
"""Executes a pipeline and reports its status. When PIPELINE is given, it
executes only the pipeline with such a name. If the argument is omitted,
all pipelines are executed in lexicographical order.
"""
cwd = os.getcwd()
pipes = pu.read_config()['pipelines']
project_root = pu.get_project_root()
if pipeline:
if pipeline not in pipes:
pu.fail("Cannot find pipeline {} in .popper.yml".format(pipeline))
status = run_pipeline(project_root, pipes[pipeline], timeout, skip)
else:
if os.path.basename(cwd) in pipes:
# run just the one for CWD
status = run_pipeline(project_root, pipes[os.path.basename(cwd)],
timeout, skip)
else:
# run all
for pipe in pipes:
status = run_pipeline(project_root, pipes[pipe], timeout, skip)
if status == 'FAIL':
break
os.chdir(cwd)
if status == 'FAIL':
pu.fail("Failed to execute pipeline")
def delete_archive(self):
"""Deletes the created archive from the filesystem.
"""
project_root = pu.get_project_root()
archive_file = os.path.basename(project_root) + '.tar.gz'
os.chdir(project_root)
command = 'rm ' + archive_file
subprocess.call(command, shell=True)
def cli(ctx, service, history, inplace):
"""Generates markdown for the badge of a service. Currently available
services are: CloudLab, Chameleon, Google Cloud Engine and Popper.
"""
if history and service:
raise BadArgumentUsage("--history can't be combined with other flags.")
remote_url = pu.get_remote_url()
if not remote_url:
pu.fail("Failed to infer remote URL for git repository.")
org, repo = remote_url.split('/')[-2:]
if history:
baseurl = pu.read_config().get('badge-server-url',
'http://badges.falsifiable.us')
try:
r = requests.get('{}/{}/{}/list'.format(baseurl, org, repo))
if r.json():
pu.print_yaml(r.json())
else:
pu.info("No records to show")
except requests.exceptions.RequestException:
pu.fail("Could not communicate with the badge server")
sys.exit(0)
if not service and inplace:
raise BadArgumentUsage("--inplace must be given with --service")
if service is None:
pu.fail('Please specify a service name.')
if service not in services:
pu.fail('Unknown service {}.'.format(service))
if service == 'popper':
org, repo = remote_url.split('/')[-2:]
markup = '[]({})'.format(
services[service][0], services[service][1].format(org, repo),
services[service][2])
else:
markup = '[]({})'.format(*services[service])
if not inplace:
pu.info(markup)
sys.exit(0)
try:
os.chdir(pu.get_project_root())
with open('README.md', 'r+') as f:
content = f.read()
f.seek(0, 0)
f.write(markup + '\n\n' + content)
except IOError as e:
if e.errno == ENOENT:
pu.fail("README.md does not exist at the root of the project")
def cli(ctx, name, stages, envs, existing, infer_stages):
"""Initializes a repository or a pipeline. Without an argument, this
command initializes a popper repository. If an argument is given, a
pipeline or paper folder is initialized. If the given name is 'paper',
then a 'paper' folder is created. Otherwise, a pipeline named NAME is
created and initialized inside the 'pipelines' folder.
By default, the stages of a pipeline are: setup, run, post-run, validate
and teardown. To override these, the `--stages` flag can be provided, which
expects a comma-separated list of stage names.
The teardown stage is to be provided at the end if the --stages flag is
being used.
If the --existing flag is given, the NAME argument is treated as a path to
a folder, which is assumed to contain bash scripts. --stages must be given.
"""
# check if the the teardown stage is the last stage of the pipeline
if stages and 'teardown' in stages and stages.split(',')[-1] != 'teardown':
raise BadArgumentUsage(
'--stages = Teardown should be the last stage.' +
' Consider renaming it or putting it at the end.')
project_root = pu.get_project_root()
# init repo
if name is None:
initialize_repo(project_root)
return
if not pu.is_popperized():
pu.fail("Repository has not been popperized yet. See 'init --help'")
if isdir(os.path.join(project_root, name)) and existing:
# existing pipeline
abs_path = os.path.join(project_root, name)
relative_path = name
if infer_stages:
stages = ",".join(
map(lambda x: x[:-3], sorted(glob.glob1(abs_path, '*.sh'))))
else:
initialize_existing_pipeline(abs_path, stages, envs)
elif name == 'paper':
# create a paper pipeline
abs_path = os.path.join(project_root, 'paper')
relative_path = os.path.join('paper')
initialize_paper(abs_path, envs)
else:
# new pipeline
abs_path = os.path.join(project_root, 'pipelines', name)
relative_path = os.path.join('pipelines', name)
initialize_new_pipeline(abs_path, stages, envs)
pu.update_config(name, stages, envs, relative_path)
pu.info('Initialized pipeline ' + name, fg='blue', bold=True)
def delete_archive(self):
"""Deletes the created archive from the filesystem.
"""
cwd = os.getcwd()
project_root = pu.get_project_root()
archive_file = '/tmp/' + os.path.basename(project_root) + '.tar.gz'
os.chdir(project_root)
subprocess.call('rm ' + archive_file, shell=True)
os.chdir(cwd)
def cli(ctx, pipeline, folder, branch):
"""Add a pipeline to your repository from the existing popperized
repositories on github. The pipeline argument is provided as owner/repo/
pipeline. For example, 'popper add popperized/quiho-popper/single-node'
adds the 'single-node' pipeline from the 'quiho-popper' repository from the
'popperized' organization.
"""
if len(pipeline.split('/')) != 3:
raise BadArgumentUsage(
"Bad pipeline name. See 'popper add --help' for more info.")
owner, repo, pipe_name = pipeline.split('/')
config = pu.read_config()
if pipe_name in config['pipelines']:
pu.fail("Pipeline {} already in repo.".format(pipe_name))
project_root = pu.get_project_root()
pipelines_dir = os.path.join(project_root, folder)
if not os.path.exists(pipelines_dir):
os.mkdir(pipelines_dir)
gh_url = 'https://github.com/{}/{}/'.format(owner, repo)
gh_url += 'archive/{}.tar.gz'.format(branch)
pu.info("Downloading pipeline {}... ".format(pipe_name))
r = pu.make_gh_request(
gh_url,
msg="Unable to fetch the pipeline. Please check if the name"
" of the pipeline is correct and the internet is connected"
)
# Downloading and extracting the tarfile
with tarfile.open(
mode='r:gz', fileobj=BytesIO(r.content)) as t:
t.extractall()
os.rename('{}-{}/pipelines/{}'.format(
repo, branch, pipe_name), os.path.join(folder, pipe_name))
shutil.rmtree('{}-{}'.format(repo, branch))
pu.info("Updating popper configuration... ")
repo_config = get_config(owner, repo)
config['pipelines'][pipe_name] = repo_config['pipelines'][pipe_name]
config['pipelines'][pipe_name]['path'] = os.path.join(folder, pipe_name)
pu.write_config(config)
pu.info("Pipeline {} has been added successfully.".format(pipe_name),
fg="green")
def upload_snapshot(self):
self.update_metadata()
deposition_id = self.deposition['id']
new_file = self.create_archive()
project_root = pu.get_project_root()
url = '{}/{}/files'.format(self.baseurl, deposition_id)
data = {'filename': new_file}
files = {'file': open(os.path.join(project_root, new_file), 'rb')}
self.delete_archive()
r = requests.post(url, data=data, files=files, params=self.params)
if r.status_code != 201:
pu.fail("Status {}: {}".format(r.status_code, r.json()))
def create_archive(self):
"""Creates an archive of the entire repsitory
using the git archive command.
Returns:
Name of the archive file.
"""
project_root = pu.get_project_root()
project_name = os.path.basename(project_root)
os.chdir(project_root)
archive_file = project_name + '.tar.gz'
command = 'git archive master | gzip > ' + archive_file
subprocess.call(command, shell=True)
return archive_file
def cli(ctx, pipeline, timeout, skip, ignore_errors, output, no_badge_update,
requirement_level):
"""Executes one or more pipelines and reports on their status. When
PIPELINE is given, it executes only the pipeline with that name. If the
argument is omitted, all pipelines are executed in lexicographical order.
"""
project_pipelines = pu.read_config()['pipelines']
if len(project_pipelines) == 0:
pu.info(
"No pipelines defined in .popper.yml. "
"Run popper init --help for more info.",
fg='yellow')
sys.exit(0)
project_root = pu.get_project_root()
cwd = os.getcwd()
pipelines = get_pipelines_to_execute(cwd, pipeline, project_pipelines)
if os.environ.get('CI', False):
pipes_from_log = pipelines_from_commit_message(project_pipelines)
if len(pipes_from_log) != 0:
pu.info("Found 'CI', ignoring PIPELINE argument.")
pipelines = pipes_from_log
pipelines = {
pipe_n: pipe_c
for pipe_n, pipe_c in pipelines.items()
if check_requirements(pipe_n, pipe_c, requirement_level)
}
pipelines = check_skiplist(pipelines, skip)
if not len(pipelines):
pu.info("No pipelines to execute")
sys.exit(0)
status = run_pipelines(pipelines, project_root, timeout, skip,
ignore_errors, output)
os.chdir(cwd)
if os.environ.get('CI', False) and not no_badge_update:
update_badge(status)
if status == 'FAIL':
pu.fail("Failed to execute pipeline")
def cli(ctx, pipeline, folder):
"""Add a pipeline to your repository from the existing popperized
repositories on github. The pipeline argument is provided as owner/repo/
pipeline. For example, 'popper add popperized/quiho-popper/single-node'
adds the 'single-node' pipeline from the 'quiho-popper' repository from the
'popperized' organization.
"""
if len(pipeline.split('/')) != 3:
pu.fail("Bad pipeline name. See 'popper add --help' for more info.")
owner, repo, pipe_name = pipeline.split('/')
config = pu.read_config()
if pipe_name in config['pipelines']:
pu.fail("Pipeline {} already in repo.".format(pipe_name))
project_root = pu.get_project_root()
pipelines_dir = os.path.join(project_root, folder)
if not os.path.exists(pipelines_dir):
os.mkdir(pipelines_dir)
gh_url = 'https://github.com/{}/{}/archive/master.zip'.format(owner, repo)
pu.info("Downloading pipeline {}... ".format(pipe_name))
r = requests.get(gh_url)
if r.status_code != 200:
pu.fail("Unable to fetch the pipeline. Please check if the name" +
" of the pipeline is correct and the internet is connected")
with zipfile.ZipFile(BytesIO(r.content)) as z:
z.extractall()
os.rename('{}-master/pipelines/{}'.format(repo, pipe_name),
os.path.join(folder, pipe_name))
shutil.rmtree('{}-master'.format(repo))
pu.info("Updating popper configuration... ")
repo_config = get_config(owner, repo)
config['pipelines'][pipe_name] = repo_config['pipelines'][pipe_name]
config['pipelines'][pipe_name]['path'] = os.path.join(folder, pipe_name)
pu.write_config(config)
pu.info("Pipeline {} has been added successfully.".format(pipe_name),
fg="green")
def cli(ctx, service):
"""Generates configuration files for distinct CI services.
"""
project_root = pu.get_project_root()
if service not in ci_files:
pu.fail("Unrecognized service " + service)
for ci_file, ci_file_content in pu.get_items(ci_files[service]):
ci_file = os.path.join(project_root, ci_file)
# create parent folder
if not os.path.isdir(os.path.dirname(ci_file)):
os.makedirs(os.path.dirname(ci_file))
# write content
with open(ci_file, 'w') as f:
f.write(ci_file_content)
def cli(ctx, pipeline, timeout, skip, ignore_errors):
"""Executes a pipeline and reports its status. When PIPELINE is given, it
executes only the pipeline with such a name. If the argument is omitted,
all pipelines are executed in lexicographical order. Reports an error if
no pipelines have been configured.
"""
cwd = os.getcwd()
pipes = pu.read_config()['pipelines']
project_root = pu.get_project_root()
time_out = pu.parse_timeout(timeout)
if len(pipes) == 0:
pu.info("No pipelines defined in .popper.yml. "
"Run popper init --help for more info.", fg='yellow')
sys.exit(0)
if pipeline:
if ignore_errors:
pu.warn("--ignore-errors flag is ignored when pipeline "
"argument is provided")
if pipeline not in pipes:
pu.fail("Cannot find pipeline {} in .popper.yml".format(pipeline))
status = run_pipeline(project_root, pipes[pipeline], time_out, skip)
else:
if os.path.basename(cwd) in pipes:
# run just the one for CWD
status = run_pipeline(project_root, pipes[os.path.basename(cwd)],
time_out, skip)
else:
# run all
skip_list = skip.split(',') if skip else []
for pipe in pipes:
if pipe not in skip_list:
status = run_pipeline(
project_root, pipes[pipe], time_out, []
)
if status == 'FAIL' and not ignore_errors:
break
os.chdir(cwd)
if status == 'FAIL':
pu.fail("Failed to execute pipeline")
def cli(ctx):
"""Initializes a repository by creating the .popper.yml file.
"""
project_root = pu.get_project_root()
content = pt.ReadMe()
if pu.is_popperized():
pu.fail('Repository has already been popperized')
return
pu.write_config(pu.init_config)
with open(os.path.join(project_root, '.gitignore'), 'a') as f:
f.write(pu.gitignore_content)
# write README
content.init_project()
pu.info('Popperized repository {}\n'.format(project_root))
def cli(ctx, service, key, no_publish):
"""Creates a archive of the repository on the provided service using an
access token. Reports an error if archive creation is not successful.
Currently supported services are Zenodo.
"""
supported_services = ['zenodo']
if service not in supported_services:
pu.fail("The service {} is not supported. See popper archive "
"--help for more info.".format(service))
project_root = pu.get_project_root()
project_name = os.path.basename(project_root)
if not key:
key = get_access_token(service, project_root)
if service == 'zenodo':
service_url = 'https://zenodo.org/api/deposit/depositions'
params = {'access_token': key}
if no_publish:
archive_file = create_archive(project_root, project_name)
deposition_id = upload_snapshot(service_url, params, archive_file)
delete_archive(project_root, archive_file)
else:
# Get the list of depositions
uploads = requests.get(service_url, params=params)
if uploads.status_code == 200:
if not uploads.json()[0]['submitted']:
deposition_id = uploads.json()[0]['id']
else:
archive_file = create_archive(project_root, project_name)
deposition_id = upload_snapshot(service_url, params,
archive_file)
delete_archive(project_root, archive_file)
metadata_url = service_url + '/{}'.format(deposition_id)
publish_url = add_metadata(metadata_url, params)
doi = publish_snapshot(publish_url, params)
pu.info("Done..!")
def cli(ctx, name, stages, envs, existing):
"""Initializes a repository or a pipeline. Without an argument, this
command initializes a popper repository. If an argument is given, a
pipeline or paper folder is initialized. If the given name is 'paper',
then a 'paper' folder is created. Otherwise, a pipeline named NAME is
created and initialized inside the 'pipelines' folder.
By default, the stages of a pipeline are: setup, run, post-run, validate
and teardown. To override these, the `--stages` flag can be provided, which
expects a comma-separated list of stage names.
If the --existing flag is given, the NAME argument is treated as a path to
a folder, which is assumed to contain bash scripts. --stages must be given.
"""
project_root = pu.get_project_root()
# init repo
if name is None:
initialize_repo(project_root)
return
if not pu.is_popperized():
pu.fail("Repository has not been popperized yet. See 'init --help'")
if isdir(os.path.join(project_root, name)) and existing:
# existing pipeline
abs_path = os.path.join(project_root, name)
relative_path = name
initialize_existing_pipeline(abs_path, stages, envs)
elif name == 'paper':
# create a paper pipeline
abs_path = os.path.join(project_root, 'paper')
relative_path = os.path.join('paper')
initialize_paper(abs_path, envs)
else:
# new pipeline
abs_path = os.path.join(project_root, 'pipelines', name)
relative_path = os.path.join('pipelines', name)
initialize_new_pipeline(abs_path, stages, envs)
pu.update_config(name, stages, envs, relative_path)
pu.info('Initialized pipeline ' + name, fg='blue', bold=True)
def cli(ctx, service, skip, requirement_level):
"""Generates configuration files for distinct CI services.
"""
project_root = pu.get_project_root()
if service not in ci_files:
pu.fail("Unrecognized service " + service)
runargs = '--skip={} '.format(skip) if skip else ''
runargs += '--requirement-level {}'.format(requirement_level)
for ci_file, ci_file_content in pu.get_items(ci_files[service]):
ci_file_content = ci_file_content.format(runargs=runargs)
ci_file = os.path.join(project_root, ci_file)
# create parent folder
if not os.path.isdir(os.path.dirname(ci_file)):
os.makedirs(os.path.dirname(ci_file))
# write content
with open(ci_file, 'w') as f:
f.write(ci_file_content)
def cli(ctx, pipeline):
"""This command is used to remove a popper pipeline from the user's repository
effectively to keep the pipelines folder and the .popper.yml files in sync.
Examples:
popper rm single-node
"""
pipeline_dir = os.path.join(pu.get_project_root(), 'pipelines')
popper_config = pu.read_config()
pipeline_path = os.path.join(pipeline_dir, pipeline)
if os.path.isdir(pipeline_path):
shutil.rmtree(pipeline_path)
popper_config = pu.read_config()
del popper_config['pipelines'][pipeline]
if 'stages' in popper_config:
if pipeline in popper_config['stages']:
del popper_config['stages'][pipeline]
if 'envs' in popper_config:
if pipeline in popper_config['envs']:
del popper_config['envs'][pipeline]
pu.info("Pipeline {} removed successfully".format(pipeline),
fg="green")
pu.write_config(popper_config)
else:
pu.fail("Pipeline {} doesn't exists".format(pipeline))
def cli(ctx):
"""Resets a popper repository completely, removing all existing
pipelines and folders, leaving behind a newly created .popper.yml file.
"""
project_root = pu.get_project_root()
for file_name in os.listdir(project_root):
if file_name in [".git", ".cache"]:
continue
file_path = os.path.join(project_root, file_name)
try:
shutil.rmtree(file_path)
except OSError:
os.remove(file_path)
config = {
'metadata': {
'access_right': "open",
'license': "CC-BY-4.0",
'upload_type': "publication",
'publication_type': "article"
},
'pipelines': {},
'popperized': ["github/popperized"]
}
pu.write_config(config)
with open(os.path.join(project_root, '.gitignore'), 'a') as f:
f.write('.cache\n')
f.write('popper_logs\n')
f.write('popper_status\n')
pu.info("Reset complete", fg="cyan")
def cli(ctx, keywords, skip_update, add, rm, ls, include_readme):
"""Searches for pipelines on GitHub matching the given keyword(s).
The list of repositories or organizations scraped for Popper pipelines is
specified in the 'popperized' list in the .popper.yml file. By default,
https://github.com/popperized is added to the configuration.
If no keywords are specified, a list of all the pipelines from all
organizations (in the .popper.yml file) and repositories will be returned.
Example:
popper search quiho
would result in:
popperized/quiho-popper
To add or remove orgs/repos to/from the 'popperized' ,
use the --add and --rm flags while searching.
popper search --add org/repo
To remove an organization/person do:
popper search --rm org/repo
To view the list repositories that are available to the search command:
popper search --ls
"""
if (rm or add or ls) and (keywords):
raise BadArgumentUsage(
"'add', 'rm' and 'ls' flags cannot be combined with others.")
project_root = pu.get_project_root()
config = pu.read_config()
popperized_list = config['popperized']
if add:
add = 'github/' + add
if add not in popperized_list:
popperized_list.append(add)
config['popperized'] = popperized_list
pu.write_config(config)
sys.exit(0)
if rm:
rm = 'github/' + rm
if rm in popperized_list:
popperized_list.remove(rm)
config['popperized'] = popperized_list
pu.write_config(config)
sys.exit(0)
result = [] # to store the result of the search query as a list
if ls:
for p in popperized_list:
if p.count('/') == 1:
org_name = p.split('/')[1]
org_url = ('https://api.github.com/users/{}/repos')
org_url = org_url.format(org_name)
response = pu.make_gh_request(org_url)
repos = response.json()
temp = [r["full_name"] for r in repos]
result.extend(temp)
else:
result.extend(p[7:])
if len(result) > 0:
pu.info("The list of available poppperized repositories are:\n")
pu.print_yaml(result)
sys.exit()
else:
fail_msg = "There are no popperized repositores available"
"for search. Use the --add flag to add an org/repo."
pu.fail(fail_msg)
sys.exit(0)
search_params = {}
if not keywords: # checks if the query is empty or not
search_params['empty_query'] = True
else:
search_params['empty_query'] = False
cache_dir = os.path.join(project_root, '.cache')
search_params["keywords"] = keywords
search_params["cache_dir"] = cache_dir
search_params["skip_update"] = True if skip_update else False
search_params["in_readme"] = True if include_readme else False
if not os.path.exists(cache_dir):
os.makedirs(cache_dir)
for popperized in popperized_list:
if popperized.count('/') == 1:
# it is an organization
org_name = popperized.split('/')[1]
repos = ""
if not skip_update:
org_url = (
'https://api.github.com/users/{}/repos'.format(org_name))
response = pu.make_gh_request(org_url)
with open(os.path.join(cache_dir, org_name + '_repos.json'),
'w') as f:
json.dump(response.json(), f)
try:
with open(os.path.join(cache_dir, org_name + '_repos.json'),
'r') as f:
repos = json.load(f)
except FileNotFoundError:
pu.fail('No cached metadata has been downloaded')
with click.progressbar(
repos,
show_eta=False,
label='Searching in ' + org_name,
bar_template='[%(bar)s] %(label)s | %(info)s',
show_percent=True) as bar:
for r in bar:
if search_params["empty_query"]:
temp = ' {}/{}'\
.format(org_name, r['name'])
result.append(temp)
elif l_distance(r["name"].lower(), keywords.lower()) < 1:
temp = ' {}/{}' \
.format(org_name, r['name'])
result.append(temp)
else:
search_params["repo_url"] = r["url"]
search_params["uname"] = org_name
result.extend(search_pipeline(search_params))
else:
# it is a repository
user, repo = popperized.split('/')[1:]
repo_url = ('https://api.github.com/repos/{}/{}'.format(
user, repo))
search_params["repo_url"] = repo_url
search_params["uname"] = user
pu.info("Searching in repository : {}".format(repo))
result.extend(search_pipeline(search_params))
if len(result) != 0:
pu.info("\nSearch results:\n", fg="green")
for res in result:
pu.info("> " + res + "\n")
if search_params["in_readme"]:
pu.info("Use popper info command to view the"
" details of a pipeline. See popper info --"
"help")
else:
pu.fail("Unable to find any matching pipelines")
def upload_snapshot(self):
self.update_metadata()
new_file = self.create_archive()
project_root = pu.get_project_root()
file_name = os.path.join(project_root, new_file)
CHUNK_SIZE = 1048576
# Initiate file upload
with open(file_name, 'rb') as stream:
md5 = hashlib.md5()
size = 0
data = stream.read(CHUNK_SIZE)
while data:
size += len(data)
md5.update(data)
data = stream.read(CHUNK_SIZE)
md5, size = md5.hexdigest(), size
data = {
'name': new_file,
'md5': md5,
'size': size
}
url = '{}/{}/files'.format(self.baseurl, self.record_id)
r = requests.post(url, data=json.dumps(data), params=self.params)
# Receive the location and issue a get request
location = r.json()['location']
r = requests.get(location, params=self.params)
# Receive the upload url and issue a get request to the upload url
# to receive the number of file parts
file_info = r.json()
url = file_info['upload_url']
r = requests.get(url, params=self.params)
# Upload all the file parts
parts = r.json()['parts']
with open(file_name, 'rb') as stream:
for part in parts:
upload_data = file_info.copy()
upload_data.update(part)
url = '{upload_url}/{partNo}'.format(**upload_data)
stream.seek(part['startOffset'])
data = stream.read(part['endOffset'] - part['startOffset'] + 1)
r = requests.put(url, data=data, params=self.params)
if r.status_code != 200:
self.delete_archive()
pu.fail(
"Status {}: Could not upload the file. Please"
"try again later.".format(r.status_code)
)
self.delete_archive()
# Complete the file upload
url = '{}/{}/files/{}'.format(
self.baseurl, self.record_id, file_info['id']
)
r = requests.post(url, params=self.params)
if r.status_code != 202:
pu.fail("Status {}: {}".format(r.status_code, r.json()))
def cli(ctx, pipeline_name, skip_update):
"""Searches for the specified pipelines inside the GitHub
organizations specified in the .popper.yml file.
The organization at -> https://github.com/popperized is present
by default.
If no pipeline name is specified by the user, then a list of all the
pipelines from all the listed organizations and their respective
repositories will be returned.
Example:
popper search quiho
would result in:
popperized/quiho-popper
"""
project_root = pu.get_project_root()
empty_query = False
if not pipeline_name: # checks if the query is empty or not
empty_query = True
POPPER_GITHUB_API_TOKEN = ""
# check if the github personal access token has been specified by the user
if 'POPPER_GITHUB_API_TOKEN' in os.environ:
POPPER_GITHUB_API_TOKEN = os.environ['POPPER_GITHUB_API_TOKEN']
config = pu.read_config()
popperized_list = config['popperized']
result = [] # to store the result of the search query as a list
cache_dir = os.path.join(project_root, '.cache')
if not os.path.exists(cache_dir):
os.makedirs(cache_dir)
for popperized in popperized_list:
if popperized.count('/') == 1:
# it is an organization
org_name = popperized.split('/')[1]
repos = ""
if not skip_update:
org_url = ('https://api.github.com/users/{}/repos'
.format(org_name))
headers = {}
if POPPER_GITHUB_API_TOKEN != "":
headers = {
'Authorization': 'token %s' % POPPER_GITHUB_API_TOKEN
}
response = requests.get(org_url, headers=headers)
if response.status_code != 200:
pu.fail("Unable to connect. Please check your network"
" and try again. Response code = {} {}"
.format(response.status_code, org_url))
#
with open(os.path.join(cache_dir, org_name + '_repos.json'),
'w') as f:
json.dump(response.json(), f)
try:
with open(os.path.join(cache_dir, org_name + '_repos.json'),
'r') as f:
repos = json.load(f)
except FileNotFoundError:
pu.fail('No cached metadata has been downloaded')
with click.progressbar(
repos,
show_eta=False,
label='Searching {}'.format(org_name),
bar_template='[%(bar)s] %(label)s | %(info)s',
show_percent=True) as bar:
for r in bar:
if l_distance(r["name"].lower(),
pipeline_name.lower()) < 1:
temp = ' {}/{}' \
.format(org_name, r['name'])
result.append(temp)
else:
result.extend(
search_pipeline(
r["url"],
pipeline_name,
org_name,
empty_query,
POPPER_GITHUB_API_TOKEN,
cache_dir,
skip_update))
else:
# it is a repository
user, repo = popperized.split('/')[1:]
repo_url = ('https://api.github.com/repos/{}/{}'
.format(user, repo))
headers = {}
pu.info("Searching {}".format(repo))
result.extend(
search_pipeline(repo_url, pipeline_name,
user, empty_query, POPPER_GITHUB_API_TOKEN,
cache_dir, skip_update)
)
if len(result) != 0:
pu.info("Search results:\n", fg="green")
pu.print_yaml(result)
else:
pu.fail("Unable to find any matching pipelines")
def cli(ctx, pipeline):
"""Generates a workflow diagram corresponding to a Popper pipeline, in the
.dot format. The string defining the graph is printed to stdout so it can
be piped into other tools. For example, to generate a png file, one can
make use of the graphviz CLI tools:
popper workflow mypipe | dot -T png -o mypipe.png
"""
pipes = pu.read_config()['pipelines']
if pipeline not in pipes:
pu.fail("Cannot find pipeline {} in .popper.yml".format(pipeline))
project_root = pu.get_project_root()
abs_path = os.path.join(project_root, pipes[pipeline]['path'])
transformer = ObtainDotGraph()
parser = Lark(bash_grammar,
parser='lalr',
lexer='contextual',
transformer=transformer)
for stage in pipes[pipeline]['stages']:
transformer.current_stage = stage
stage_file = pu.get_filename(abs_path, stage)
with open(stage_file, 'r') as f:
s = f.read()
parser.parse(s)
cs = transformer.comment_stack
cs = remove_redundant_if(cs)
# print(cs)
print('digraph pipeline {')
curr_node = None
prev_node = None
node_id = 's{}'.format(0)
curr_if_node = None
if_created = False
if_index = None
for i, item in enumerate(cs):
if item == '[wf]#stage#':
prev_item = cs[i - 1]
next_item = cs[i + 1]
label = '"{' + '{} | {}'.format(next_item, prev_item) + '}"'
curr_node = (next_item.replace('-', '_')).replace('.sh', ' ')
# create the stage node
print('{} [{}];'.format(curr_node, 'shape=record, label=' + label))
if prev_node:
print('{} -> {};'.format(prev_node, curr_node))
prev_node = curr_node
continue
# initialize the if-node
elif item == '[wf]#if#':
if_created = False
c = 'condition'
if i > 1 and (not cs[i - 1].startswith('[wf]#')
and '.sh' not in cs[i - 1]):
c += ' : {}'.format(cs[i - 1])
if_index = i - 1
curr_if_node = node_id
# inside if-elif-else construct
elif (item == '[wf]#else#' or item == '[wf]#elif#'
or item == '[wf]#fi#'):
if not cs[i - 1].startswith('[wf]#'):
if not if_created:
if_created, node_id = create_if_node(node_id, c, prev_node)
print('{} [shape=record, label="{}"];'.format(
node_id, cs[i - 1]))
print('{} -> {};'.format(curr_if_node, node_id))
node_id = increment(node_id)
if item == '[wf]#fi':
if_created = False
continue
# inside loop
elif item == '[wf]#done':
c = 'loop'
if not cs[i - 1].startswith('[wf]#') and '.sh' not in cs[i - 1]:
c += ' : {}'.format(cs[i - 1])
print('{} [shape=record,label="{}"];'.format(node_id, c))
print('{} -> {};'.format(prev_node, node_id))
node_id = increment(node_id)
# is a comment outside any control structures
elif not item.startswith('[wf]#') and '.sh' not in item:
if i == len(cs) - 1 and not cs[i - 1] == '[wf]#stage#':
print('{} [shape=record,label="{}"];'.format(node_id, item))
print('{} -> {};'.format(prev_node, node_id))
node_id = increment(node_id)
elif i < len(cs) - 1:
if (not cs[i + 1].startswith('[wf]#')
and not cs[i - 1] == '[wf]#stage#'):
print('{} [shape=record,label="{}"];'.format(
node_id, item))
print('{} -> {};'.format(prev_node, node_id))
node_id = increment(node_id)
print('}')
