def cli(ctx): """Resets a popper repository completely, removing all existing pipelines and folders, leaving behind a newly created .popper.yml file. Note: It only removes those files inside a pipeline folder that are also tracked by git. Untracked files will not be deleted. """ msg = ( "This will remove all the pipeline files in this " " project, do you want to continue?" ) if(not click.confirm(msg, abort=False)): sys.exit(0) project_root = pu.get_project_root() if project_root != os.getcwd(): msg = 'This command can only be executed from the project root folder' pu.fail(msg) config = pu.read_config() for _, p in config['pipelines'].items(): pu.exec_cmd('git rm -r {}'.format(p['path'])) pu.write_config(pu.init_config) content = pt.ReadMe() content.init_project() pu.info("Reset complete", fg="cyan")
def cli(ctx, pipeline): """Remove a popper pipeline from the user's repository effectively to keep the pipelines folder and the .popper.yml files in sync. """ project_root = pu.get_project_root() if pipeline == 'paper': pipeline_dir = project_root else: pipeline_dir = os.path.join(project_root, 'pipelines') pipeline_path = os.path.join(pipeline_dir, pipeline) if os.path.isdir(pipeline_path): shutil.rmtree(pipeline_path) popper_config = pu.read_config() del popper_config['pipelines'][pipeline] pu.info("Pipeline {} removed successfully".format(pipeline), fg="green") pu.write_config(popper_config) else: pu.fail("Pipeline {} doesn't exists".format(pipeline))
def cli(ctx, pipeline): """Remove a popper pipeline from the user's repository effectively to keep the pipelines folder and the .popper.yml files in sync. """ project_root = pu.get_project_root() pipelines = pu.read_config()['pipelines'] if pipeline in pipelines: path = pipelines[pipeline]['path'] pipeline_dir = os.path.join( project_root, path) else: pu.fail("Pipeline '{}' not in this project".format(pipeline)) if os.path.isdir(pipeline_dir): shutil.rmtree(pipeline_dir) popper_config = pu.read_config() del popper_config['pipelines'][pipeline] pu.info("Pipeline '{}' removed successfully".format(pipeline), fg="blue") pu.write_config(popper_config) else: pu.fail("Path '{}' is not a folder".format(pipeline))
def cli(ctx): """Synchronize your pipelines and popper.yml file if any pipeline or stage has been deleted. """ popper_config = pu.read_config() project_root = pu.get_project_root() pipelines = popper_config['pipelines'] # Removing nonexistent pipelines from .popper.yml for p in list(pipelines): pipeline = pipelines[p] pipe_path = os.path.join(project_root, pipeline['path']) # Checking if the pipeline exists if os.path.exists(pipe_path): # Synchronizing stages stages = [ x[:-3] for x in os.listdir(pipe_path) if x.endswith(".sh") ] pipelines[p]['stages'] = stages else: del pipelines[p] popper_config['pipelines'] = pipelines pu.write_config(popper_config) pu.info( "\nYour popper.yml file has been updated! Run git diff to see " "the differences.", fg="white")
def get_access_token(service): """Tries to read the access token from a key file. If not present, prompts the user for a key and also stores the key in a key file if the user wishes.""" project_root = pu.get_project_root() os.chdir(project_root) try: with open('.{}.key'.format(service), 'r') as keyfile: encrypted_access_token = keyfile.read().strip() passphrase = click.prompt( 'Please enter your passphrase for {}'.format(service), hide_input=True).encode() aes = pyaes.AESModeOfOperationCTR(generate_key(passphrase)) try: access_token = aes.decrypt(encrypted_access_token).decode() except UnicodeDecodeError: pu.fail("Invalid passphrase. Please use the same passphrase " "used at the time of encrypting the access_token.") except FileNotFoundError: pu.info('No access token found for {}'.format(service)) access_token = click.prompt( 'Please enter your access token for {}'.format(service)) if click.confirm('Would you like to store this key?'): passphrase = click.prompt('Enter a strong passphrase', hide_input=True).encode() aes = pyaes.AESModeOfOperationCTR(generate_key(passphrase)) encrypted_access_token = aes.encrypt(access_token) with open('.{}.key'.format(service), 'w') as keyfile: keyfile.writelines('{}'.format(''.join( chr(b) for b in encrypted_access_token))) pu.info('Your key is stored in .{}.key'.format(service)) return access_token
def create_archive(self): """Creates an archive of the entire project folder using gzip or zip. Returns: Name of the archive file. """ cwd = os.getcwd() project_root = pu.get_project_root() project_name = os.path.basename(project_root) os.chdir(project_root) archive_file = '/tmp/' + project_name + '.tar' cmd = 'git archive master > ' + archive_file subprocess.check_output(cmd, shell=True) if not self.ignore_untracked: cmd = ( 'git ls-files --others --exclude-standard -z | ' 'xargs -0 tar rf ' + archive_file ) subprocess.check_output(cmd, shell=True) cmd = 'gzip ' + archive_file subprocess.check_output(cmd, shell=True) os.chdir(cwd) return archive_file + '.gz'
def cli(ctx): """Synchronize your pipelines and popper.yml file if any pipeline or stage has been deleted. """ pipeline_dir = os.path.join(pu.get_project_root(), 'pipelines') popper_config = pu.read_config() pipelines = {} for pipeline in os.listdir(pipeline_dir): envs = popper_config['pipelines'][pipeline]['envs'] relative_path = popper_config['pipelines'][pipeline]['path'] defined_stages = popper_config['pipelines'][pipeline]['stages'] existing_stages = [] for stage in defined_stages: os.chdir(os.path.join(pipeline_dir, pipeline)) if os.path.exists(stage+'.sh') or os.path.exists(stage): existing_stages.append(stage) pipelines[pipeline] = { 'envs': envs, 'path': relative_path, 'stages': existing_stages } popper_config['pipelines'] = pipelines pu.write_config(popper_config) pu.info("\nYour popper.yml file has been updated! Run git diff to see " "the differences.", fg="white")
def cli(ctx, pipeline): """Add a pipeline to your repository from the existing popperized repositories on github. The pipeline argument is provided as owner/repo/ pipeline. For example, popper add popperized/quiho-popper/single-node adds the single-node pipeline from the quiho-popper repository. """ try: owner, repo, pipeline_name = pipeline.split('/') except ValueError: pu.fail("See popper add --help for more info.") project_root = pu.get_project_root() path = os.path.join(project_root, 'pipelines') if os.path.exists(path): pass else: os.chdir(project_root) os.mkdir('pipelines') dirname = pipeline_name url = ('https://api.github.com/repos/{}/{}/contents/pipelines/{}' .format(owner, repo, pipeline_name)) repo_config = get_config(owner, repo) save_directory(path, dirname, url) path = os.path.join(path, pipeline_name) update_config(owner, repo, pipeline_name, path, repo_config) pu.info("Pipeline {} successfully added.".format(pipeline_name) + " It can be viewed in the pipelines directory.", fg="green")
def cli(ctx, pipeline, timeout, skip): """Executes a pipeline and reports its status. When PIPELINE is given, it executes only the pipeline with such a name. If the argument is omitted, all pipelines are executed in lexicographical order. """ cwd = os.getcwd() pipes = pu.read_config()['pipelines'] project_root = pu.get_project_root() if pipeline: if pipeline not in pipes: pu.fail("Cannot find pipeline {} in .popper.yml".format(pipeline)) status = run_pipeline(project_root, pipes[pipeline], timeout, skip) else: if os.path.basename(cwd) in pipes: # run just the one for CWD status = run_pipeline(project_root, pipes[os.path.basename(cwd)], timeout, skip) else: # run all for pipe in pipes: status = run_pipeline(project_root, pipes[pipe], timeout, skip) if status == 'FAIL': break os.chdir(cwd) if status == 'FAIL': pu.fail("Failed to execute pipeline")
def delete_archive(self): """Deletes the created archive from the filesystem. """ project_root = pu.get_project_root() archive_file = os.path.basename(project_root) + '.tar.gz' os.chdir(project_root) command = 'rm ' + archive_file subprocess.call(command, shell=True)
def cli(ctx, service, history, inplace): """Generates markdown for the badge of a service. Currently available services are: CloudLab, Chameleon, Google Cloud Engine and Popper. """ if history and service: raise BadArgumentUsage("--history can't be combined with other flags.") remote_url = pu.get_remote_url() if not remote_url: pu.fail("Failed to infer remote URL for git repository.") org, repo = remote_url.split('/')[-2:] if history: baseurl = pu.read_config().get('badge-server-url', 'http://badges.falsifiable.us') try: r = requests.get('{}/{}/{}/list'.format(baseurl, org, repo)) if r.json(): pu.print_yaml(r.json()) else: pu.info("No records to show") except requests.exceptions.RequestException: pu.fail("Could not communicate with the badge server") sys.exit(0) if not service and inplace: raise BadArgumentUsage("--inplace must be given with --service") if service is None: pu.fail('Please specify a service name.') if service not in services: pu.fail('Unknown service {}.'.format(service)) if service == 'popper': org, repo = remote_url.split('/')[-2:] markup = '[![{}]({})]({})'.format( services[service][0], services[service][1].format(org, repo), services[service][2]) else: markup = '[![{}]({})]({})'.format(*services[service]) if not inplace: pu.info(markup) sys.exit(0) try: os.chdir(pu.get_project_root()) with open('README.md', 'r+') as f: content = f.read() f.seek(0, 0) f.write(markup + '\n\n' + content) except IOError as e: if e.errno == ENOENT: pu.fail("README.md does not exist at the root of the project")
def cli(ctx, name, stages, envs, existing, infer_stages): """Initializes a repository or a pipeline. Without an argument, this command initializes a popper repository. If an argument is given, a pipeline or paper folder is initialized. If the given name is 'paper', then a 'paper' folder is created. Otherwise, a pipeline named NAME is created and initialized inside the 'pipelines' folder. By default, the stages of a pipeline are: setup, run, post-run, validate and teardown. To override these, the `--stages` flag can be provided, which expects a comma-separated list of stage names. The teardown stage is to be provided at the end if the --stages flag is being used. If the --existing flag is given, the NAME argument is treated as a path to a folder, which is assumed to contain bash scripts. --stages must be given. """ # check if the the teardown stage is the last stage of the pipeline if stages and 'teardown' in stages and stages.split(',')[-1] != 'teardown': raise BadArgumentUsage( '--stages = Teardown should be the last stage.' + ' Consider renaming it or putting it at the end.') project_root = pu.get_project_root() # init repo if name is None: initialize_repo(project_root) return if not pu.is_popperized(): pu.fail("Repository has not been popperized yet. See 'init --help'") if isdir(os.path.join(project_root, name)) and existing: # existing pipeline abs_path = os.path.join(project_root, name) relative_path = name if infer_stages: stages = ",".join( map(lambda x: x[:-3], sorted(glob.glob1(abs_path, '*.sh')))) else: initialize_existing_pipeline(abs_path, stages, envs) elif name == 'paper': # create a paper pipeline abs_path = os.path.join(project_root, 'paper') relative_path = os.path.join('paper') initialize_paper(abs_path, envs) else: # new pipeline abs_path = os.path.join(project_root, 'pipelines', name) relative_path = os.path.join('pipelines', name) initialize_new_pipeline(abs_path, stages, envs) pu.update_config(name, stages, envs, relative_path) pu.info('Initialized pipeline ' + name, fg='blue', bold=True)
def delete_archive(self): """Deletes the created archive from the filesystem. """ cwd = os.getcwd() project_root = pu.get_project_root() archive_file = '/tmp/' + os.path.basename(project_root) + '.tar.gz' os.chdir(project_root) subprocess.call('rm ' + archive_file, shell=True) os.chdir(cwd)
def cli(ctx, pipeline, folder, branch): """Add a pipeline to your repository from the existing popperized repositories on github. The pipeline argument is provided as owner/repo/ pipeline. For example, 'popper add popperized/quiho-popper/single-node' adds the 'single-node' pipeline from the 'quiho-popper' repository from the 'popperized' organization. """ if len(pipeline.split('/')) != 3: raise BadArgumentUsage( "Bad pipeline name. See 'popper add --help' for more info.") owner, repo, pipe_name = pipeline.split('/') config = pu.read_config() if pipe_name in config['pipelines']: pu.fail("Pipeline {} already in repo.".format(pipe_name)) project_root = pu.get_project_root() pipelines_dir = os.path.join(project_root, folder) if not os.path.exists(pipelines_dir): os.mkdir(pipelines_dir) gh_url = 'https://github.com/{}/{}/'.format(owner, repo) gh_url += 'archive/{}.tar.gz'.format(branch) pu.info("Downloading pipeline {}... ".format(pipe_name)) r = pu.make_gh_request( gh_url, msg="Unable to fetch the pipeline. Please check if the name" " of the pipeline is correct and the internet is connected" ) # Downloading and extracting the tarfile with tarfile.open( mode='r:gz', fileobj=BytesIO(r.content)) as t: t.extractall() os.rename('{}-{}/pipelines/{}'.format( repo, branch, pipe_name), os.path.join(folder, pipe_name)) shutil.rmtree('{}-{}'.format(repo, branch)) pu.info("Updating popper configuration... ") repo_config = get_config(owner, repo) config['pipelines'][pipe_name] = repo_config['pipelines'][pipe_name] config['pipelines'][pipe_name]['path'] = os.path.join(folder, pipe_name) pu.write_config(config) pu.info("Pipeline {} has been added successfully.".format(pipe_name), fg="green")
def upload_snapshot(self): self.update_metadata() deposition_id = self.deposition['id'] new_file = self.create_archive() project_root = pu.get_project_root() url = '{}/{}/files'.format(self.baseurl, deposition_id) data = {'filename': new_file} files = {'file': open(os.path.join(project_root, new_file), 'rb')} self.delete_archive() r = requests.post(url, data=data, files=files, params=self.params) if r.status_code != 201: pu.fail("Status {}: {}".format(r.status_code, r.json()))
def create_archive(self): """Creates an archive of the entire repsitory using the git archive command. Returns: Name of the archive file. """ project_root = pu.get_project_root() project_name = os.path.basename(project_root) os.chdir(project_root) archive_file = project_name + '.tar.gz' command = 'git archive master | gzip > ' + archive_file subprocess.call(command, shell=True) return archive_file
def cli(ctx, pipeline, timeout, skip, ignore_errors, output, no_badge_update, requirement_level): """Executes one or more pipelines and reports on their status. When PIPELINE is given, it executes only the pipeline with that name. If the argument is omitted, all pipelines are executed in lexicographical order. """ project_pipelines = pu.read_config()['pipelines'] if len(project_pipelines) == 0: pu.info( "No pipelines defined in .popper.yml. " "Run popper init --help for more info.", fg='yellow') sys.exit(0) project_root = pu.get_project_root() cwd = os.getcwd() pipelines = get_pipelines_to_execute(cwd, pipeline, project_pipelines) if os.environ.get('CI', False): pipes_from_log = pipelines_from_commit_message(project_pipelines) if len(pipes_from_log) != 0: pu.info("Found 'CI', ignoring PIPELINE argument.") pipelines = pipes_from_log pipelines = { pipe_n: pipe_c for pipe_n, pipe_c in pipelines.items() if check_requirements(pipe_n, pipe_c, requirement_level) } pipelines = check_skiplist(pipelines, skip) if not len(pipelines): pu.info("No pipelines to execute") sys.exit(0) status = run_pipelines(pipelines, project_root, timeout, skip, ignore_errors, output) os.chdir(cwd) if os.environ.get('CI', False) and not no_badge_update: update_badge(status) if status == 'FAIL': pu.fail("Failed to execute pipeline")
def cli(ctx, pipeline, folder): """Add a pipeline to your repository from the existing popperized repositories on github. The pipeline argument is provided as owner/repo/ pipeline. For example, 'popper add popperized/quiho-popper/single-node' adds the 'single-node' pipeline from the 'quiho-popper' repository from the 'popperized' organization. """ if len(pipeline.split('/')) != 3: pu.fail("Bad pipeline name. See 'popper add --help' for more info.") owner, repo, pipe_name = pipeline.split('/') config = pu.read_config() if pipe_name in config['pipelines']: pu.fail("Pipeline {} already in repo.".format(pipe_name)) project_root = pu.get_project_root() pipelines_dir = os.path.join(project_root, folder) if not os.path.exists(pipelines_dir): os.mkdir(pipelines_dir) gh_url = 'https://github.com/{}/{}/archive/master.zip'.format(owner, repo) pu.info("Downloading pipeline {}... ".format(pipe_name)) r = requests.get(gh_url) if r.status_code != 200: pu.fail("Unable to fetch the pipeline. Please check if the name" + " of the pipeline is correct and the internet is connected") with zipfile.ZipFile(BytesIO(r.content)) as z: z.extractall() os.rename('{}-master/pipelines/{}'.format(repo, pipe_name), os.path.join(folder, pipe_name)) shutil.rmtree('{}-master'.format(repo)) pu.info("Updating popper configuration... ") repo_config = get_config(owner, repo) config['pipelines'][pipe_name] = repo_config['pipelines'][pipe_name] config['pipelines'][pipe_name]['path'] = os.path.join(folder, pipe_name) pu.write_config(config) pu.info("Pipeline {} has been added successfully.".format(pipe_name), fg="green")
def cli(ctx, service): """Generates configuration files for distinct CI services. """ project_root = pu.get_project_root() if service not in ci_files: pu.fail("Unrecognized service " + service) for ci_file, ci_file_content in pu.get_items(ci_files[service]): ci_file = os.path.join(project_root, ci_file) # create parent folder if not os.path.isdir(os.path.dirname(ci_file)): os.makedirs(os.path.dirname(ci_file)) # write content with open(ci_file, 'w') as f: f.write(ci_file_content)
def cli(ctx, pipeline, timeout, skip, ignore_errors): """Executes a pipeline and reports its status. When PIPELINE is given, it executes only the pipeline with such a name. If the argument is omitted, all pipelines are executed in lexicographical order. Reports an error if no pipelines have been configured. """ cwd = os.getcwd() pipes = pu.read_config()['pipelines'] project_root = pu.get_project_root() time_out = pu.parse_timeout(timeout) if len(pipes) == 0: pu.info("No pipelines defined in .popper.yml. " "Run popper init --help for more info.", fg='yellow') sys.exit(0) if pipeline: if ignore_errors: pu.warn("--ignore-errors flag is ignored when pipeline " "argument is provided") if pipeline not in pipes: pu.fail("Cannot find pipeline {} in .popper.yml".format(pipeline)) status = run_pipeline(project_root, pipes[pipeline], time_out, skip) else: if os.path.basename(cwd) in pipes: # run just the one for CWD status = run_pipeline(project_root, pipes[os.path.basename(cwd)], time_out, skip) else: # run all skip_list = skip.split(',') if skip else [] for pipe in pipes: if pipe not in skip_list: status = run_pipeline( project_root, pipes[pipe], time_out, [] ) if status == 'FAIL' and not ignore_errors: break os.chdir(cwd) if status == 'FAIL': pu.fail("Failed to execute pipeline")
def cli(ctx): """Initializes a repository by creating the .popper.yml file. """ project_root = pu.get_project_root() content = pt.ReadMe() if pu.is_popperized(): pu.fail('Repository has already been popperized') return pu.write_config(pu.init_config) with open(os.path.join(project_root, '.gitignore'), 'a') as f: f.write(pu.gitignore_content) # write README content.init_project() pu.info('Popperized repository {}\n'.format(project_root))
def cli(ctx, service, key, no_publish): """Creates a archive of the repository on the provided service using an access token. Reports an error if archive creation is not successful. Currently supported services are Zenodo. """ supported_services = ['zenodo'] if service not in supported_services: pu.fail("The service {} is not supported. See popper archive " "--help for more info.".format(service)) project_root = pu.get_project_root() project_name = os.path.basename(project_root) if not key: key = get_access_token(service, project_root) if service == 'zenodo': service_url = 'https://zenodo.org/api/deposit/depositions' params = {'access_token': key} if no_publish: archive_file = create_archive(project_root, project_name) deposition_id = upload_snapshot(service_url, params, archive_file) delete_archive(project_root, archive_file) else: # Get the list of depositions uploads = requests.get(service_url, params=params) if uploads.status_code == 200: if not uploads.json()[0]['submitted']: deposition_id = uploads.json()[0]['id'] else: archive_file = create_archive(project_root, project_name) deposition_id = upload_snapshot(service_url, params, archive_file) delete_archive(project_root, archive_file) metadata_url = service_url + '/{}'.format(deposition_id) publish_url = add_metadata(metadata_url, params) doi = publish_snapshot(publish_url, params) pu.info("Done..!")
def cli(ctx, name, stages, envs, existing): """Initializes a repository or a pipeline. Without an argument, this command initializes a popper repository. If an argument is given, a pipeline or paper folder is initialized. If the given name is 'paper', then a 'paper' folder is created. Otherwise, a pipeline named NAME is created and initialized inside the 'pipelines' folder. By default, the stages of a pipeline are: setup, run, post-run, validate and teardown. To override these, the `--stages` flag can be provided, which expects a comma-separated list of stage names. If the --existing flag is given, the NAME argument is treated as a path to a folder, which is assumed to contain bash scripts. --stages must be given. """ project_root = pu.get_project_root() # init repo if name is None: initialize_repo(project_root) return if not pu.is_popperized(): pu.fail("Repository has not been popperized yet. See 'init --help'") if isdir(os.path.join(project_root, name)) and existing: # existing pipeline abs_path = os.path.join(project_root, name) relative_path = name initialize_existing_pipeline(abs_path, stages, envs) elif name == 'paper': # create a paper pipeline abs_path = os.path.join(project_root, 'paper') relative_path = os.path.join('paper') initialize_paper(abs_path, envs) else: # new pipeline abs_path = os.path.join(project_root, 'pipelines', name) relative_path = os.path.join('pipelines', name) initialize_new_pipeline(abs_path, stages, envs) pu.update_config(name, stages, envs, relative_path) pu.info('Initialized pipeline ' + name, fg='blue', bold=True)
def cli(ctx, service, skip, requirement_level): """Generates configuration files for distinct CI services. """ project_root = pu.get_project_root() if service not in ci_files: pu.fail("Unrecognized service " + service) runargs = '--skip={} '.format(skip) if skip else '' runargs += '--requirement-level {}'.format(requirement_level) for ci_file, ci_file_content in pu.get_items(ci_files[service]): ci_file_content = ci_file_content.format(runargs=runargs) ci_file = os.path.join(project_root, ci_file) # create parent folder if not os.path.isdir(os.path.dirname(ci_file)): os.makedirs(os.path.dirname(ci_file)) # write content with open(ci_file, 'w') as f: f.write(ci_file_content)
def cli(ctx, pipeline): """This command is used to remove a popper pipeline from the user's repository effectively to keep the pipelines folder and the .popper.yml files in sync. Examples: popper rm single-node """ pipeline_dir = os.path.join(pu.get_project_root(), 'pipelines') popper_config = pu.read_config() pipeline_path = os.path.join(pipeline_dir, pipeline) if os.path.isdir(pipeline_path): shutil.rmtree(pipeline_path) popper_config = pu.read_config() del popper_config['pipelines'][pipeline] if 'stages' in popper_config: if pipeline in popper_config['stages']: del popper_config['stages'][pipeline] if 'envs' in popper_config: if pipeline in popper_config['envs']: del popper_config['envs'][pipeline] pu.info("Pipeline {} removed successfully".format(pipeline), fg="green") pu.write_config(popper_config) else: pu.fail("Pipeline {} doesn't exists".format(pipeline))
def cli(ctx): """Resets a popper repository completely, removing all existing pipelines and folders, leaving behind a newly created .popper.yml file. """ project_root = pu.get_project_root() for file_name in os.listdir(project_root): if file_name in [".git", ".cache"]: continue file_path = os.path.join(project_root, file_name) try: shutil.rmtree(file_path) except OSError: os.remove(file_path) config = { 'metadata': { 'access_right': "open", 'license': "CC-BY-4.0", 'upload_type': "publication", 'publication_type': "article" }, 'pipelines': {}, 'popperized': ["github/popperized"] } pu.write_config(config) with open(os.path.join(project_root, '.gitignore'), 'a') as f: f.write('.cache\n') f.write('popper_logs\n') f.write('popper_status\n') pu.info("Reset complete", fg="cyan")
def cli(ctx, keywords, skip_update, add, rm, ls, include_readme): """Searches for pipelines on GitHub matching the given keyword(s). The list of repositories or organizations scraped for Popper pipelines is specified in the 'popperized' list in the .popper.yml file. By default, https://github.com/popperized is added to the configuration. If no keywords are specified, a list of all the pipelines from all organizations (in the .popper.yml file) and repositories will be returned. Example: popper search quiho would result in: popperized/quiho-popper To add or remove orgs/repos to/from the 'popperized' , use the --add and --rm flags while searching. popper search --add org/repo To remove an organization/person do: popper search --rm org/repo To view the list repositories that are available to the search command: popper search --ls """ if (rm or add or ls) and (keywords): raise BadArgumentUsage( "'add', 'rm' and 'ls' flags cannot be combined with others.") project_root = pu.get_project_root() config = pu.read_config() popperized_list = config['popperized'] if add: add = 'github/' + add if add not in popperized_list: popperized_list.append(add) config['popperized'] = popperized_list pu.write_config(config) sys.exit(0) if rm: rm = 'github/' + rm if rm in popperized_list: popperized_list.remove(rm) config['popperized'] = popperized_list pu.write_config(config) sys.exit(0) result = [] # to store the result of the search query as a list if ls: for p in popperized_list: if p.count('/') == 1: org_name = p.split('/')[1] org_url = ('https://api.github.com/users/{}/repos') org_url = org_url.format(org_name) response = pu.make_gh_request(org_url) repos = response.json() temp = [r["full_name"] for r in repos] result.extend(temp) else: result.extend(p[7:]) if len(result) > 0: pu.info("The list of available poppperized repositories are:\n") pu.print_yaml(result) sys.exit() else: fail_msg = "There are no popperized repositores available" "for search. Use the --add flag to add an org/repo." pu.fail(fail_msg) sys.exit(0) search_params = {} if not keywords: # checks if the query is empty or not search_params['empty_query'] = True else: search_params['empty_query'] = False cache_dir = os.path.join(project_root, '.cache') search_params["keywords"] = keywords search_params["cache_dir"] = cache_dir search_params["skip_update"] = True if skip_update else False search_params["in_readme"] = True if include_readme else False if not os.path.exists(cache_dir): os.makedirs(cache_dir) for popperized in popperized_list: if popperized.count('/') == 1: # it is an organization org_name = popperized.split('/')[1] repos = "" if not skip_update: org_url = ( 'https://api.github.com/users/{}/repos'.format(org_name)) response = pu.make_gh_request(org_url) with open(os.path.join(cache_dir, org_name + '_repos.json'), 'w') as f: json.dump(response.json(), f) try: with open(os.path.join(cache_dir, org_name + '_repos.json'), 'r') as f: repos = json.load(f) except FileNotFoundError: pu.fail('No cached metadata has been downloaded') with click.progressbar( repos, show_eta=False, label='Searching in ' + org_name, bar_template='[%(bar)s] %(label)s | %(info)s', show_percent=True) as bar: for r in bar: if search_params["empty_query"]: temp = ' {}/{}'\ .format(org_name, r['name']) result.append(temp) elif l_distance(r["name"].lower(), keywords.lower()) < 1: temp = ' {}/{}' \ .format(org_name, r['name']) result.append(temp) else: search_params["repo_url"] = r["url"] search_params["uname"] = org_name result.extend(search_pipeline(search_params)) else: # it is a repository user, repo = popperized.split('/')[1:] repo_url = ('https://api.github.com/repos/{}/{}'.format( user, repo)) search_params["repo_url"] = repo_url search_params["uname"] = user pu.info("Searching in repository : {}".format(repo)) result.extend(search_pipeline(search_params)) if len(result) != 0: pu.info("\nSearch results:\n", fg="green") for res in result: pu.info("> " + res + "\n") if search_params["in_readme"]: pu.info("Use popper info command to view the" " details of a pipeline. See popper info --" "help") else: pu.fail("Unable to find any matching pipelines")
def upload_snapshot(self): self.update_metadata() new_file = self.create_archive() project_root = pu.get_project_root() file_name = os.path.join(project_root, new_file) CHUNK_SIZE = 1048576 # Initiate file upload with open(file_name, 'rb') as stream: md5 = hashlib.md5() size = 0 data = stream.read(CHUNK_SIZE) while data: size += len(data) md5.update(data) data = stream.read(CHUNK_SIZE) md5, size = md5.hexdigest(), size data = { 'name': new_file, 'md5': md5, 'size': size } url = '{}/{}/files'.format(self.baseurl, self.record_id) r = requests.post(url, data=json.dumps(data), params=self.params) # Receive the location and issue a get request location = r.json()['location'] r = requests.get(location, params=self.params) # Receive the upload url and issue a get request to the upload url # to receive the number of file parts file_info = r.json() url = file_info['upload_url'] r = requests.get(url, params=self.params) # Upload all the file parts parts = r.json()['parts'] with open(file_name, 'rb') as stream: for part in parts: upload_data = file_info.copy() upload_data.update(part) url = '{upload_url}/{partNo}'.format(**upload_data) stream.seek(part['startOffset']) data = stream.read(part['endOffset'] - part['startOffset'] + 1) r = requests.put(url, data=data, params=self.params) if r.status_code != 200: self.delete_archive() pu.fail( "Status {}: Could not upload the file. Please" "try again later.".format(r.status_code) ) self.delete_archive() # Complete the file upload url = '{}/{}/files/{}'.format( self.baseurl, self.record_id, file_info['id'] ) r = requests.post(url, params=self.params) if r.status_code != 202: pu.fail("Status {}: {}".format(r.status_code, r.json()))
def cli(ctx, pipeline_name, skip_update): """Searches for the specified pipelines inside the GitHub organizations specified in the .popper.yml file. The organization at -> https://github.com/popperized is present by default. If no pipeline name is specified by the user, then a list of all the pipelines from all the listed organizations and their respective repositories will be returned. Example: popper search quiho would result in: popperized/quiho-popper """ project_root = pu.get_project_root() empty_query = False if not pipeline_name: # checks if the query is empty or not empty_query = True POPPER_GITHUB_API_TOKEN = "" # check if the github personal access token has been specified by the user if 'POPPER_GITHUB_API_TOKEN' in os.environ: POPPER_GITHUB_API_TOKEN = os.environ['POPPER_GITHUB_API_TOKEN'] config = pu.read_config() popperized_list = config['popperized'] result = [] # to store the result of the search query as a list cache_dir = os.path.join(project_root, '.cache') if not os.path.exists(cache_dir): os.makedirs(cache_dir) for popperized in popperized_list: if popperized.count('/') == 1: # it is an organization org_name = popperized.split('/')[1] repos = "" if not skip_update: org_url = ('https://api.github.com/users/{}/repos' .format(org_name)) headers = {} if POPPER_GITHUB_API_TOKEN != "": headers = { 'Authorization': 'token %s' % POPPER_GITHUB_API_TOKEN } response = requests.get(org_url, headers=headers) if response.status_code != 200: pu.fail("Unable to connect. Please check your network" " and try again. Response code = {} {}" .format(response.status_code, org_url)) # with open(os.path.join(cache_dir, org_name + '_repos.json'), 'w') as f: json.dump(response.json(), f) try: with open(os.path.join(cache_dir, org_name + '_repos.json'), 'r') as f: repos = json.load(f) except FileNotFoundError: pu.fail('No cached metadata has been downloaded') with click.progressbar( repos, show_eta=False, label='Searching {}'.format(org_name), bar_template='[%(bar)s] %(label)s | %(info)s', show_percent=True) as bar: for r in bar: if l_distance(r["name"].lower(), pipeline_name.lower()) < 1: temp = ' {}/{}' \ .format(org_name, r['name']) result.append(temp) else: result.extend( search_pipeline( r["url"], pipeline_name, org_name, empty_query, POPPER_GITHUB_API_TOKEN, cache_dir, skip_update)) else: # it is a repository user, repo = popperized.split('/')[1:] repo_url = ('https://api.github.com/repos/{}/{}' .format(user, repo)) headers = {} pu.info("Searching {}".format(repo)) result.extend( search_pipeline(repo_url, pipeline_name, user, empty_query, POPPER_GITHUB_API_TOKEN, cache_dir, skip_update) ) if len(result) != 0: pu.info("Search results:\n", fg="green") pu.print_yaml(result) else: pu.fail("Unable to find any matching pipelines")
def cli(ctx, pipeline): """Generates a workflow diagram corresponding to a Popper pipeline, in the .dot format. The string defining the graph is printed to stdout so it can be piped into other tools. For example, to generate a png file, one can make use of the graphviz CLI tools: popper workflow mypipe | dot -T png -o mypipe.png """ pipes = pu.read_config()['pipelines'] if pipeline not in pipes: pu.fail("Cannot find pipeline {} in .popper.yml".format(pipeline)) project_root = pu.get_project_root() abs_path = os.path.join(project_root, pipes[pipeline]['path']) transformer = ObtainDotGraph() parser = Lark(bash_grammar, parser='lalr', lexer='contextual', transformer=transformer) for stage in pipes[pipeline]['stages']: transformer.current_stage = stage stage_file = pu.get_filename(abs_path, stage) with open(stage_file, 'r') as f: s = f.read() parser.parse(s) cs = transformer.comment_stack cs = remove_redundant_if(cs) # print(cs) print('digraph pipeline {') curr_node = None prev_node = None node_id = 's{}'.format(0) curr_if_node = None if_created = False if_index = None for i, item in enumerate(cs): if item == '[wf]#stage#': prev_item = cs[i - 1] next_item = cs[i + 1] label = '"{' + '{} | {}'.format(next_item, prev_item) + '}"' curr_node = (next_item.replace('-', '_')).replace('.sh', ' ') # create the stage node print('{} [{}];'.format(curr_node, 'shape=record, label=' + label)) if prev_node: print('{} -> {};'.format(prev_node, curr_node)) prev_node = curr_node continue # initialize the if-node elif item == '[wf]#if#': if_created = False c = 'condition' if i > 1 and (not cs[i - 1].startswith('[wf]#') and '.sh' not in cs[i - 1]): c += ' : {}'.format(cs[i - 1]) if_index = i - 1 curr_if_node = node_id # inside if-elif-else construct elif (item == '[wf]#else#' or item == '[wf]#elif#' or item == '[wf]#fi#'): if not cs[i - 1].startswith('[wf]#'): if not if_created: if_created, node_id = create_if_node(node_id, c, prev_node) print('{} [shape=record, label="{}"];'.format( node_id, cs[i - 1])) print('{} -> {};'.format(curr_if_node, node_id)) node_id = increment(node_id) if item == '[wf]#fi': if_created = False continue # inside loop elif item == '[wf]#done': c = 'loop' if not cs[i - 1].startswith('[wf]#') and '.sh' not in cs[i - 1]: c += ' : {}'.format(cs[i - 1]) print('{} [shape=record,label="{}"];'.format(node_id, c)) print('{} -> {};'.format(prev_node, node_id)) node_id = increment(node_id) # is a comment outside any control structures elif not item.startswith('[wf]#') and '.sh' not in item: if i == len(cs) - 1 and not cs[i - 1] == '[wf]#stage#': print('{} [shape=record,label="{}"];'.format(node_id, item)) print('{} -> {};'.format(prev_node, node_id)) node_id = increment(node_id) elif i < len(cs) - 1: if (not cs[i + 1].startswith('[wf]#') and not cs[i - 1] == '[wf]#stage#'): print('{} [shape=record,label="{}"];'.format( node_id, item)) print('{} -> {};'.format(prev_node, node_id)) node_id = increment(node_id) print('}')