def git_status(component, exclude_components, short, base): # noqa: D301 """Report status of REANA source repositories. \b :param components: The option ``component`` can be repeated. The value may consist of: * (1) standard component name such as 'reana-workflow-controller'; * (2) short component name such as 'r-w-controller'; * (3) special value '.' indicating component of the current working directory; * (4) special value 'CLUSTER' that will expand to cover all REANA cluster components [default]; * (5) special value 'CLIENT' that will expand to cover all REANA client components; * (6) special value 'DEMO' that will expand to include several runable REANA demo examples; * (7) special value 'ALL' that will expand to include all REANA repositories. :param exclude_components: List of components to exclude from command. :param base: Against which git base branch are we working? [default=master] :param verbose: Show git status details? [default=False] :type component: str :type exclude_components: str :type base: str """ if exclude_components: exclude_components = exclude_components.split(",") components = select_components(component, exclude_components) for component in components: current_branch = get_current_branch(get_srcdir(component)) # detect all local and remote branches all_branches = get_all_branches(get_srcdir(component)) # detect branch to compare against if current_branch == base: # base branch branch_to_compare = "upstream/" + base elif current_branch.startswith("pr-"): # other people's PR branch_to_compare = "upstream/" + current_branch.replace( "pr-", "pr/") else: branch_to_compare = "origin/" + current_branch # my PR if "remotes/" + branch_to_compare not in all_branches: branch_to_compare = "origin/" + base # local unpushed branch print_branch_difference_report(component, branch_to_compare, base=base, short=short)
def print_branch_difference_report( component, branch_to_compare, current_branch=None, base=GIT_DEFAULT_BASE_BRANCH, commit=None, short=False, ): """Report to stdout the differences between two branches.""" # detect how far it is ahead/behind from pr/origin/upstream current_branch = current_branch or get_current_branch( get_srcdir(component)) commit = commit or get_current_commit(get_srcdir(component)) report = "" behind, ahead = compare_branches(branch_to_compare, current_branch) if ahead or behind: report += "(" if ahead: report += "{0} AHEAD ".format(ahead) if behind: report += "{0} BEHIND ".format(behind) report += branch_to_compare + ")" # detect rebase needs for local branches and PRs if branch_to_compare != "upstream/{}".format(base): branch_to_compare = "upstream/{}".format(base) behind, ahead = compare_branches(branch_to_compare, current_branch) if behind: report += "(STEMS FROM " if behind: report += "{0} BEHIND ".format(behind) report += branch_to_compare + ")" click.secho("{0}".format(component), nl=False, bold=True) click.secho(" @ ", nl=False, dim=True) if current_branch == base: click.secho("{0}".format(current_branch), nl=False) else: click.secho("{0}".format(current_branch), nl=False, fg="green") if report: click.secho(" {0}".format(report), nl=False, fg="red") click.secho(" @ ", nl=False, dim=True) click.secho("{0}".format(commit)) # optionally, display also short status if short: cmd = "git status --short" run_command(cmd, component, display=False)
def branch_exists(component, branch): """Check whether a branch exists on a given component. :param component: Component in which check whether the branch exists. :param branch: Name of the branch. :return: Whether the branch exists in components git repo. :rtype: bool """ return branch in get_all_branches(get_srcdir(component))
def git_clone(user, component, exclude_components): # noqa: D301 """Clone REANA source repositories from GitHub. If the ``user`` argument is provided, the ``origin`` will be cloned from the user repository on GitHub and the ``upstream`` will be set to ``reanahub`` organisation. Useful for setting up personal REANA development environment, If the ``user`` argument is not provided, the cloning will be done in anonymous manner from ``reanahub`` organisation. Also, the clone will be shallow to save disk space and CPU time. Useful for CI purposes. \b :param user: The GitHub user name. [default=anonymous] :param components: The option ``component`` can be repeated. The value may consist of: * (1) standard component name such as 'reana-workflow-controller'; * (2) short component name such as 'r-w-controller'; * (3) special value '.' indicating component of the current working directory; * (4) special value 'CLUSTER' that will expand to cover all REANA cluster components [default]; * (5) special value 'CLIENT' that will expand to cover all REANA client components; * (6) special value 'DEMO' that will expand to include several runable REANA demo examples; * (7) special value 'ALL' that will expand to include all REANA repositories. :param exclude_components: List of components to exclude from command. :type user: str :type component: str :type exclude_components: str """ if exclude_components: exclude_components = exclude_components.split(",") components = select_components(component, exclude_components) for component in components: os.chdir(get_srcdir()) if os.path.exists("{0}/.git/config".format(component)): msg = "Component seems already cloned. Skipping." display_message(msg, component) elif user == "anonymous": cmd = "git clone https://github.com/reanahub/{0} --depth 1".format( component) run_command(cmd) else: cmd = "git clone [email protected]:{0}/{1}".format(user, component) run_command(cmd) for cmd in [ "git remote add upstream" ' "[email protected]:reanahub/{0}"'.format(component), "git config --add remote.upstream.fetch" ' "+refs/pull/*/head:refs/remotes/upstream/pr/*"', ]: run_command(cmd, component)
def is_component_behind_branch( component, branch_to_compare, current_branch=None, ): """Report to stdout the differences between two branches.""" current_branch = current_branch or get_current_branch( get_srcdir(component)) behind, _ = compare_branches(branch_to_compare, current_branch) return bool(behind)
def helm_upgrade_components(ctx, user, push): # noqa: D301 """Upgrade REANA Helm dependencies. Checks if any docker releases are missed. If yes, exists with the error and lists missing components. If not, updates values.yaml and prefetch-images.sh with new docker images version """ _check_if_missing_docker_releases() new_docker_images = _get_docker_releases(user) values_yaml_abs_path = os.path.join(get_srcdir("reana"), "helm/reana/values.yaml") _upgrade_docker_images(values_yaml_abs_path, new_docker_images) prefetch_script_abs_path = os.path.join( get_srcdir("reana"), "scripts/prefetch-images.sh" ) _upgrade_docker_images(prefetch_script_abs_path, new_docker_images) ctx.invoke(git_diff, component=["reana"]) if push: git_push_to_origin(["reana"])
def _create_commit_or_amend(components): for c in components: commit_cmd = 'git commit -m "installation: bump shared modules"' if amend: commit_cmd = "git commit --amend --no-edit" files_to_commit = ["setup.py"] if os.path.exists(get_srcdir(c) + os.sep + "requirements.txt"): files_to_commit.append("requirements.txt") run_command( f"git add {' '.join(files_to_commit)} && {commit_cmd}", c, )
def release_helm(ctx, user: str, dry_run: bool) -> None: # noqa: D301 """Release REANA as a Helm chart.""" component = "reana" version = get_current_component_version_from_source_files(component) is_chart_releaser_installed = which("cr") github_pages_branch = "gh-pages" package_path = ".cr-release-packages" index_path = ".cr-index" repository = f"https://{user}.github.io/{component}" is_component_releasable(component, exit_code=True, display=True) if not is_chart_releaser_installed: click.secho( "Please install chart-releaser to be able to do a Helm release", fg="red", ) sys.exit(1) if not os.getenv("CR_TOKEN"): click.secho( "Please provide your GitHub token as CR_TOKEN environment variable", fg="red", ) sys.exit(1) current_commit_sha = get_current_commit( get_srcdir(component)).split(" ")[0] for cmd in [ f"rm -rf {package_path}", f"mkdir {package_path}", f"rm -rf {index_path}", f"mkdir {index_path}", f"helm package helm/reana --destination {package_path} --dependency-update", f"cr upload -o {user} -r {component} --release-name-template '{{{{ .Version }}}}' --commit {current_commit_sha}", f"cr index -o {user} -r {component} -c {repository} --release-name-template '{{{{ .Version }}}}'", ]: run_command(cmd, component, dry_run=dry_run) with tempfile.TemporaryDirectory() as gh_pages_worktree: run_command( f"git worktree add '{gh_pages_worktree}' gh-pages && " f"cd {gh_pages_worktree} && " f"cp -f {get_srcdir(component) + os.sep + index_path}/index.yaml {gh_pages_worktree}/index.yaml && " f"git add index.yaml && " f"git commit -m 'index.yaml: {version}' && " f"git push origin {github_pages_branch} && " f"cd - && " f"git worktree remove '{gh_pages_worktree}'", dry_run=dry_run, )
def is_component_python_package(component): """Return whether the component is a Python package. Useful to skip running wide unit test commands for those components that are not concerned. :param component: standard component name :type component: str :return: True/False whether the component is a Python package :rtype: bool """ if os.path.exists(get_srcdir(component) + os.sep + "setup.py"): return True return False
def git_create_release_commit(component, base=GIT_DEFAULT_BASE_BRANCH, next_version=None): """Create a release commit for the given component.""" if "release:" in get_current_commit(get_srcdir(component)): display_message("Nothing to do, last commit is a release commit.", component) return False current_version = get_current_component_version_from_source_files( component) if not current_version and not next_version: display_message("Version cannot be autodiscovered from source files.", component) sys.exit(1) elif not git_is_current_version_tagged(component) and not next_version: display_message( f"Current version ({current_version}) " "not present as a git tag, please release it and add a tag.", component, ) sys.exit(1) next_version, modified_files = bump_component_version( component, current_version, next_version=next_version) if (run_command( "git branch --show-current", component, display=False, return_output=True, ) == base): run_command(f"git checkout -b release-{next_version}", component) if modified_files: run_command(f"git add {' '.join(modified_files)}", component) run_command( f"git commit -m 'release: {next_version}' {'--allow-empty' if not modified_files else ''}", component, ) return True
def get_example_reana_yaml_file_path(example, workflow_engine, compute_backend): """Get absolute path to ``reana.yaml`` file for the specified example, workflow engine and compute backend. :param example: Example where find the ``reana.yaml`` file (one of ``reana.config.REPO_LIST_DEMO``). :param workflow_engine: Workflow engine used by the example (one of ``reana.config.WORKFLOW_ENGINE_LIST_ALL``). :param components: Compute backend used by the example (one of ``reana.config.COMPUTE_BACKEND_LIST_ALL``). :return: Absolute path to ``reana.yaml`` that fulfills the specified characteristics. Empty string otherwise. :type example: str :type workflow_engine: str :type compute_backend: str :rtype: str """ reana_yaml_filename = (EXAMPLE_NON_STANDARD_REANA_YAML_FILENAME.get( example, {}).get(workflow_engine, {}).get(compute_backend, {})) if not reana_yaml_filename: reana_yaml_filename = "reana{workflow_engine}{compute_backend}.yaml".format( workflow_engine="" if workflow_engine == "serial" else "-{}".format(workflow_engine), compute_backend="" if compute_backend == "kubernetes" else "-{}".format(compute_backend), ) reana_yaml_filename_path = get_srcdir( example) + os.sep + reana_yaml_filename try: # check whether example contains recipe for given engine subprocess.check_output( [ "grep", "-q", "type: {}".format(workflow_engine), reana_yaml_filename_path, ], stderr=subprocess.DEVNULL, ) return reana_yaml_filename_path except subprocess.CalledProcessError: return ""
def _update_values_yaml(new_docker_images): """Update all images in ``values.yaml``, skipping the ones up to date.""" values_yaml_relative_path = "helm/reana/values.yaml" values_yaml_abs_path = os.path.join( get_srcdir("reana"), values_yaml_relative_path ) values_yaml = "" with open(values_yaml_abs_path) as f: values_yaml = f.read() for docker_image in new_docker_images: image_name, _ = docker_image.split(":") if image_name in values_yaml: values_yaml = re.sub( f"{image_name}:.*", lambda _: docker_image, values_yaml, count=1 ) with open(values_yaml_abs_path, "w") as f: f.write(values_yaml) display_message( f"{values_yaml_relative_path} successfully updated.", component="reana" )
def run_example(component, workflow_engine, file, timecheck, timeout, parameters, options): # noqa: D301 """Run given REANA example with given workflow engine. \b Example: $ reana-dev run-example -c r-d-r-roofit \b :param component: The option ``component`` can be repeated. The value is the repository name of the example. The special value `DEMO` will run all examples. [default=reana-demo-root6-roofit] :param workflow_engine: The option ``workflow_engine`` can be repeated. The value is the workflow engine to use to run the example. [default=cwl,serial,yadage] :param file: The option ``file`` can be repeated. The value is the expected output file the workflow should produce. [default=plot.png] :param timecheck: Checking frequency in seconds for results. [default=5 (TIMECHECK)] :param timeout: Maximum timeout to wait for results. [default=300 (TIMEOUT)] :param parameters: Additional input parameters to override original ones from reana.yaml. E.g. -p myparam1=myval1 -p myparam2=myval2. :param options: Additional operational options for the workflow execution. E.g. CACHE=off. :type component: str :type workflow_engine: str :type sleep: int """ components = select_components(component) workflow_engines = select_workflow_engines(workflow_engine) reana_yaml = { "cwl": "reana-cwl.yaml", "serial": "reana.yaml", "yadage": "reana-yadage.yaml", } for component in components: for workflow_engine in workflow_engines: workflow_name = construct_workflow_name(component, workflow_engine) # check whether example contains recipe for given engine if not os.path.exists( get_srcdir(component) + os.sep + reana_yaml[workflow_engine]): msg = "Skipping example with workflow engine {0}.".format( workflow_engine) display_message(msg, component) continue # create workflow: for cmd in [ "reana-client create -f {0} -n {1}".format( reana_yaml[workflow_engine], workflow_name), ]: run_command(cmd, component) # upload inputs for cmd in [ "reana-client upload -w {0}".format(workflow_name), ]: run_command(cmd, component) # run workflow input_parameters = " ".join( ["-p " + parameter for parameter in parameters]) operational_options = " ".join( ["-o " + option for option in options]) for cmd in [ "reana-client start -w {0} {1} {2}".format( workflow_name, input_parameters, operational_options), ]: run_command(cmd, component) # verify whether job finished within time limits time_start = time.time() while time.time() - time_start <= timeout: time.sleep(timecheck) cmd = "reana-client status -w {0}".format(workflow_name) status = run_command(cmd, component, return_output=True) click.secho(status) if "finished" in status or "failed" in status or "stopped" in status: break # verify logs message presence for log_message in get_expected_log_messages_for_example( component): cmd = "reana-client logs -w {0} | grep -c '{1}'".format( workflow_name, log_message) run_command(cmd, component) # verify output file presence cmd = "reana-client ls -w {0}".format(workflow_name) listing = run_command(cmd, component, return_output=True) click.secho(listing) expected_files = file or get_expected_output_filenames_for_example( component) for expected_file in expected_files: if expected_file not in listing: click.secho("[ERROR] Expected output file {0} not found. " "Exiting.".format(expected_file)) sys.exit(1) # report that everything was OK run_command("echo OK", component)
def cluster_deploy( namespace, job_mounts, mode, values, exclude_components, admin_email, admin_password, instance_name, ): # noqa: D301 """Deploy REANA cluster. \b Example: $ reana-dev cluster-deploy --mode debug --exclude-components=r-ui --admin-email [email protected] --admin-password mysecretpassword """ def job_mounts_to_config(job_mounts): job_mount_list = [] for mount in job_mounts: job_mount_list.append({ "name": mount["containerPath"].replace("/", "-")[1:], "hostPath": mount["hostPath"], "mountPath": mount["containerPath"], }) job_mount_config = "" if job_mount_list: job_mount_config = json.dumps(job_mount_list) else: job_mount_config = "" return job_mount_config if mode in ( "releasehelm") and values == "helm/configurations/values-dev.yaml": values = "helm/reana/values.yaml" values_dict = {} with open(os.path.join(get_srcdir("reana"), values)) as f: values_dict = yaml.safe_load(f.read()) job_mount_config = job_mounts_to_config(job_mounts) if job_mount_config: values_dict.setdefault("components", {}).setdefault( "reana_workflow_controller", {}).setdefault("environment", {})["REANA_JOB_HOSTPATH_MOUNTS"] = job_mount_config if mode in ("debug"): values_dict.setdefault("debug", {})["enabled"] = True if exclude_components: standard_named_exclude_components = [ find_standard_component_name(c) for c in exclude_components.split(",") ] if "reana-ui" in standard_named_exclude_components: values_dict["components"]["reana_ui"]["enabled"] = False helm_install = "cat <<EOF | helm install reana helm/reana -n {namespace} --create-namespace --wait -f -\n{values}\nEOF".format( namespace=namespace, values=values_dict and yaml.dump(values_dict) or "", ) cmds = [] if mode in ("debug"): cmds.append("reana-dev python-install-eggs") cmds.append("reana-dev git-submodule --update") cmds.extend([ "helm dep update helm/reana", helm_install, "kubectl config set-context --current --namespace={}".format( namespace), os.path.join( get_srcdir("reana"), f"scripts/create-admin-user.sh {namespace} {instance_name} {admin_email} {admin_password}", ), ]) for cmd in cmds: run_command(cmd, "reana")
def is_last_commit_release_commit(package): """Check whether the last commit is a release commit.""" current_commit = get_current_commit(get_srcdir(package)) return current_commit.split()[1] == "release:"