def git_status(component, exclude_components, short, base): # noqa: D301
"""Report status of REANA source repositories.
\b
:param components: The option ``component`` can be repeated. The value may
consist of:
* (1) standard component name such as
'reana-workflow-controller';
* (2) short component name such as 'r-w-controller';
* (3) special value '.' indicating component of the
current working directory;
* (4) special value 'CLUSTER' that will expand to
cover all REANA cluster components [default];
* (5) special value 'CLIENT' that will expand to
cover all REANA client components;
* (6) special value 'DEMO' that will expand
to include several runable REANA demo examples;
* (7) special value 'ALL' that will expand to include
all REANA repositories.
:param exclude_components: List of components to exclude from command.
:param base: Against which git base branch are we working? [default=master]
:param verbose: Show git status details? [default=False]
:type component: str
:type exclude_components: str
:type base: str
"""
if exclude_components:
exclude_components = exclude_components.split(",")
components = select_components(component, exclude_components)
for component in components:
current_branch = get_current_branch(get_srcdir(component))
# detect all local and remote branches
all_branches = get_all_branches(get_srcdir(component))
# detect branch to compare against
if current_branch == base: # base branch
branch_to_compare = "upstream/" + base
elif current_branch.startswith("pr-"): # other people's PR
branch_to_compare = "upstream/" + current_branch.replace(
"pr-", "pr/")
else:
branch_to_compare = "origin/" + current_branch # my PR
if "remotes/" + branch_to_compare not in all_branches:
branch_to_compare = "origin/" + base # local unpushed branch
print_branch_difference_report(component,
branch_to_compare,
base=base,
short=short)
def print_branch_difference_report(
component,
branch_to_compare,
current_branch=None,
base=GIT_DEFAULT_BASE_BRANCH,
commit=None,
short=False,
):
"""Report to stdout the differences between two branches."""
# detect how far it is ahead/behind from pr/origin/upstream
current_branch = current_branch or get_current_branch(
get_srcdir(component))
commit = commit or get_current_commit(get_srcdir(component))
report = ""
behind, ahead = compare_branches(branch_to_compare, current_branch)
if ahead or behind:
report += "("
if ahead:
report += "{0} AHEAD ".format(ahead)
if behind:
report += "{0} BEHIND ".format(behind)
report += branch_to_compare + ")"
# detect rebase needs for local branches and PRs
if branch_to_compare != "upstream/{}".format(base):
branch_to_compare = "upstream/{}".format(base)
behind, ahead = compare_branches(branch_to_compare, current_branch)
if behind:
report += "(STEMS FROM "
if behind:
report += "{0} BEHIND ".format(behind)
report += branch_to_compare + ")"
click.secho("{0}".format(component), nl=False, bold=True)
click.secho(" @ ", nl=False, dim=True)
if current_branch == base:
click.secho("{0}".format(current_branch), nl=False)
else:
click.secho("{0}".format(current_branch), nl=False, fg="green")
if report:
click.secho(" {0}".format(report), nl=False, fg="red")
click.secho(" @ ", nl=False, dim=True)
click.secho("{0}".format(commit))
# optionally, display also short status
if short:
cmd = "git status --short"
run_command(cmd, component, display=False)
def branch_exists(component, branch):
"""Check whether a branch exists on a given component.
:param component: Component in which check whether the branch exists.
:param branch: Name of the branch.
:return: Whether the branch exists in components git repo.
:rtype: bool
"""
return branch in get_all_branches(get_srcdir(component))
def git_clone(user, component, exclude_components): # noqa: D301
"""Clone REANA source repositories from GitHub.
If the ``user`` argument is provided, the ``origin`` will be cloned from
the user repository on GitHub and the ``upstream`` will be set to
``reanahub`` organisation. Useful for setting up personal REANA development
environment,
If the ``user`` argument is not provided, the cloning will be done in
anonymous manner from ``reanahub`` organisation. Also, the clone will be
shallow to save disk space and CPU time. Useful for CI purposes.
\b
:param user: The GitHub user name. [default=anonymous]
:param components: The option ``component`` can be repeated. The value may
consist of:
* (1) standard component name such as
'reana-workflow-controller';
* (2) short component name such as 'r-w-controller';
* (3) special value '.' indicating component of the
current working directory;
* (4) special value 'CLUSTER' that will expand to
cover all REANA cluster components [default];
* (5) special value 'CLIENT' that will expand to
cover all REANA client components;
* (6) special value 'DEMO' that will expand
to include several runable REANA demo examples;
* (7) special value 'ALL' that will expand to include
all REANA repositories.
:param exclude_components: List of components to exclude from command.
:type user: str
:type component: str
:type exclude_components: str
"""
if exclude_components:
exclude_components = exclude_components.split(",")
components = select_components(component, exclude_components)
for component in components:
os.chdir(get_srcdir())
if os.path.exists("{0}/.git/config".format(component)):
msg = "Component seems already cloned. Skipping."
display_message(msg, component)
elif user == "anonymous":
cmd = "git clone https://github.com/reanahub/{0} --depth 1".format(
component)
run_command(cmd)
else:
cmd = "git clone [email protected]:{0}/{1}".format(user, component)
run_command(cmd)
for cmd in [
"git remote add upstream"
' "[email protected]:reanahub/{0}"'.format(component),
"git config --add remote.upstream.fetch"
' "+refs/pull/*/head:refs/remotes/upstream/pr/*"',
]:
run_command(cmd, component)
def is_component_behind_branch(
component,
branch_to_compare,
current_branch=None,
):
"""Report to stdout the differences between two branches."""
current_branch = current_branch or get_current_branch(
get_srcdir(component))
behind, _ = compare_branches(branch_to_compare, current_branch)
return bool(behind)
def helm_upgrade_components(ctx, user, push): # noqa: D301
"""Upgrade REANA Helm dependencies.
Checks if any docker releases are missed.
If yes, exists with the error and lists missing components.
If not, updates values.yaml and prefetch-images.sh with new docker images version
"""
_check_if_missing_docker_releases()
new_docker_images = _get_docker_releases(user)
values_yaml_abs_path = os.path.join(get_srcdir("reana"), "helm/reana/values.yaml")
_upgrade_docker_images(values_yaml_abs_path, new_docker_images)
prefetch_script_abs_path = os.path.join(
get_srcdir("reana"), "scripts/prefetch-images.sh"
)
_upgrade_docker_images(prefetch_script_abs_path, new_docker_images)
ctx.invoke(git_diff, component=["reana"])
if push:
git_push_to_origin(["reana"])
def _create_commit_or_amend(components):
for c in components:
commit_cmd = 'git commit -m "installation: bump shared modules"'
if amend:
commit_cmd = "git commit --amend --no-edit"
files_to_commit = ["setup.py"]
if os.path.exists(get_srcdir(c) + os.sep + "requirements.txt"):
files_to_commit.append("requirements.txt")
run_command(
f"git add {' '.join(files_to_commit)} && {commit_cmd}",
c,
)
def release_helm(ctx, user: str, dry_run: bool) -> None: # noqa: D301
"""Release REANA as a Helm chart."""
component = "reana"
version = get_current_component_version_from_source_files(component)
is_chart_releaser_installed = which("cr")
github_pages_branch = "gh-pages"
package_path = ".cr-release-packages"
index_path = ".cr-index"
repository = f"https://{user}.github.io/{component}"
is_component_releasable(component, exit_code=True, display=True)
if not is_chart_releaser_installed:
click.secho(
"Please install chart-releaser to be able to do a Helm release",
fg="red",
)
sys.exit(1)
if not os.getenv("CR_TOKEN"):
click.secho(
"Please provide your GitHub token as CR_TOKEN environment variable",
fg="red",
)
sys.exit(1)
current_commit_sha = get_current_commit(
get_srcdir(component)).split(" ")[0]
for cmd in [
f"rm -rf {package_path}",
f"mkdir {package_path}",
f"rm -rf {index_path}",
f"mkdir {index_path}",
f"helm package helm/reana --destination {package_path} --dependency-update",
f"cr upload -o {user} -r {component} --release-name-template '{{{{ .Version }}}}' --commit {current_commit_sha}",
f"cr index -o {user} -r {component} -c {repository} --release-name-template '{{{{ .Version }}}}'",
]:
run_command(cmd, component, dry_run=dry_run)
with tempfile.TemporaryDirectory() as gh_pages_worktree:
run_command(
f"git worktree add '{gh_pages_worktree}' gh-pages && "
f"cd {gh_pages_worktree} && "
f"cp -f {get_srcdir(component) + os.sep + index_path}/index.yaml {gh_pages_worktree}/index.yaml && "
f"git add index.yaml && "
f"git commit -m 'index.yaml: {version}' && "
f"git push origin {github_pages_branch} && "
f"cd - && "
f"git worktree remove '{gh_pages_worktree}'",
dry_run=dry_run,
)
def is_component_python_package(component):
"""Return whether the component is a Python package.
Useful to skip running wide unit test commands for those components that
are not concerned.
:param component: standard component name
:type component: str
:return: True/False whether the component is a Python package
:rtype: bool
"""
if os.path.exists(get_srcdir(component) + os.sep + "setup.py"):
return True
return False
def git_create_release_commit(component,
base=GIT_DEFAULT_BASE_BRANCH,
next_version=None):
"""Create a release commit for the given component."""
if "release:" in get_current_commit(get_srcdir(component)):
display_message("Nothing to do, last commit is a release commit.",
component)
return False
current_version = get_current_component_version_from_source_files(
component)
if not current_version and not next_version:
display_message("Version cannot be autodiscovered from source files.",
component)
sys.exit(1)
elif not git_is_current_version_tagged(component) and not next_version:
display_message(
f"Current version ({current_version}) "
"not present as a git tag, please release it and add a tag.",
component,
)
sys.exit(1)
next_version, modified_files = bump_component_version(
component, current_version, next_version=next_version)
if (run_command(
"git branch --show-current",
component,
display=False,
return_output=True,
) == base):
run_command(f"git checkout -b release-{next_version}", component)
if modified_files:
run_command(f"git add {' '.join(modified_files)}", component)
run_command(
f"git commit -m 'release: {next_version}' {'--allow-empty' if not modified_files else ''}",
component,
)
return True
def get_example_reana_yaml_file_path(example, workflow_engine,
compute_backend):
"""Get absolute path to ``reana.yaml`` file for the specified example, workflow engine and compute backend.
:param example: Example where find the ``reana.yaml`` file (one of ``reana.config.REPO_LIST_DEMO``).
:param workflow_engine: Workflow engine used by the example (one of ``reana.config.WORKFLOW_ENGINE_LIST_ALL``).
:param components: Compute backend used by the example (one of ``reana.config.COMPUTE_BACKEND_LIST_ALL``).
:return: Absolute path to ``reana.yaml`` that fulfills the specified characteristics. Empty string otherwise.
:type example: str
:type workflow_engine: str
:type compute_backend: str
:rtype: str
"""
reana_yaml_filename = (EXAMPLE_NON_STANDARD_REANA_YAML_FILENAME.get(
example, {}).get(workflow_engine, {}).get(compute_backend, {}))
if not reana_yaml_filename:
reana_yaml_filename = "reana{workflow_engine}{compute_backend}.yaml".format(
workflow_engine=""
if workflow_engine == "serial" else "-{}".format(workflow_engine),
compute_backend="" if compute_backend == "kubernetes" else
"-{}".format(compute_backend),
)
reana_yaml_filename_path = get_srcdir(
example) + os.sep + reana_yaml_filename
try:
# check whether example contains recipe for given engine
subprocess.check_output(
[
"grep",
"-q",
"type: {}".format(workflow_engine),
reana_yaml_filename_path,
],
stderr=subprocess.DEVNULL,
)
return reana_yaml_filename_path
except subprocess.CalledProcessError:
return ""
def _update_values_yaml(new_docker_images):
"""Update all images in ``values.yaml``, skipping the ones up to date."""
values_yaml_relative_path = "helm/reana/values.yaml"
values_yaml_abs_path = os.path.join(
get_srcdir("reana"), values_yaml_relative_path
)
values_yaml = ""
with open(values_yaml_abs_path) as f:
values_yaml = f.read()
for docker_image in new_docker_images:
image_name, _ = docker_image.split(":")
if image_name in values_yaml:
values_yaml = re.sub(
f"{image_name}:.*", lambda _: docker_image, values_yaml, count=1
)
with open(values_yaml_abs_path, "w") as f:
f.write(values_yaml)
display_message(
f"{values_yaml_relative_path} successfully updated.", component="reana"
)
def run_example(component, workflow_engine, file, timecheck, timeout,
parameters, options): # noqa: D301
"""Run given REANA example with given workflow engine.
\b
Example:
$ reana-dev run-example -c r-d-r-roofit
\b
:param component: The option ``component`` can be repeated. The value is
the repository name of the example. The special value `DEMO`
will run all examples.
[default=reana-demo-root6-roofit]
:param workflow_engine: The option ``workflow_engine`` can be repeated. The
value is the workflow engine to use to run the
example. [default=cwl,serial,yadage]
:param file: The option ``file`` can be repeated. The value is the expected
output file the workflow should produce. [default=plot.png]
:param timecheck: Checking frequency in seconds for results.
[default=5 (TIMECHECK)]
:param timeout: Maximum timeout to wait for results.
[default=300 (TIMEOUT)]
:param parameters: Additional input parameters to override original ones
from reana.yaml.
E.g. -p myparam1=myval1 -p myparam2=myval2.
:param options: Additional operational options for the workflow execution.
E.g. CACHE=off.
:type component: str
:type workflow_engine: str
:type sleep: int
"""
components = select_components(component)
workflow_engines = select_workflow_engines(workflow_engine)
reana_yaml = {
"cwl": "reana-cwl.yaml",
"serial": "reana.yaml",
"yadage": "reana-yadage.yaml",
}
for component in components:
for workflow_engine in workflow_engines:
workflow_name = construct_workflow_name(component, workflow_engine)
# check whether example contains recipe for given engine
if not os.path.exists(
get_srcdir(component) + os.sep +
reana_yaml[workflow_engine]):
msg = "Skipping example with workflow engine {0}.".format(
workflow_engine)
display_message(msg, component)
continue
# create workflow:
for cmd in [
"reana-client create -f {0} -n {1}".format(
reana_yaml[workflow_engine], workflow_name),
]:
run_command(cmd, component)
# upload inputs
for cmd in [
"reana-client upload -w {0}".format(workflow_name),
]:
run_command(cmd, component)
# run workflow
input_parameters = " ".join(
["-p " + parameter for parameter in parameters])
operational_options = " ".join(
["-o " + option for option in options])
for cmd in [
"reana-client start -w {0} {1} {2}".format(
workflow_name, input_parameters, operational_options),
]:
run_command(cmd, component)
# verify whether job finished within time limits
time_start = time.time()
while time.time() - time_start <= timeout:
time.sleep(timecheck)
cmd = "reana-client status -w {0}".format(workflow_name)
status = run_command(cmd, component, return_output=True)
click.secho(status)
if "finished" in status or "failed" in status or "stopped" in status:
break
# verify logs message presence
for log_message in get_expected_log_messages_for_example(
component):
cmd = "reana-client logs -w {0} | grep -c '{1}'".format(
workflow_name, log_message)
run_command(cmd, component)
# verify output file presence
cmd = "reana-client ls -w {0}".format(workflow_name)
listing = run_command(cmd, component, return_output=True)
click.secho(listing)
expected_files = file or get_expected_output_filenames_for_example(
component)
for expected_file in expected_files:
if expected_file not in listing:
click.secho("[ERROR] Expected output file {0} not found. "
"Exiting.".format(expected_file))
sys.exit(1)
# report that everything was OK
run_command("echo OK", component)
def cluster_deploy(
namespace,
job_mounts,
mode,
values,
exclude_components,
admin_email,
admin_password,
instance_name,
): # noqa: D301
"""Deploy REANA cluster.
\b
Example:
$ reana-dev cluster-deploy --mode debug
--exclude-components=r-ui
--admin-email [email protected]
--admin-password mysecretpassword
"""
def job_mounts_to_config(job_mounts):
job_mount_list = []
for mount in job_mounts:
job_mount_list.append({
"name":
mount["containerPath"].replace("/", "-")[1:],
"hostPath":
mount["hostPath"],
"mountPath":
mount["containerPath"],
})
job_mount_config = ""
if job_mount_list:
job_mount_config = json.dumps(job_mount_list)
else:
job_mount_config = ""
return job_mount_config
if mode in (
"releasehelm") and values == "helm/configurations/values-dev.yaml":
values = "helm/reana/values.yaml"
values_dict = {}
with open(os.path.join(get_srcdir("reana"), values)) as f:
values_dict = yaml.safe_load(f.read())
job_mount_config = job_mounts_to_config(job_mounts)
if job_mount_config:
values_dict.setdefault("components", {}).setdefault(
"reana_workflow_controller",
{}).setdefault("environment",
{})["REANA_JOB_HOSTPATH_MOUNTS"] = job_mount_config
if mode in ("debug"):
values_dict.setdefault("debug", {})["enabled"] = True
if exclude_components:
standard_named_exclude_components = [
find_standard_component_name(c)
for c in exclude_components.split(",")
]
if "reana-ui" in standard_named_exclude_components:
values_dict["components"]["reana_ui"]["enabled"] = False
helm_install = "cat <<EOF | helm install reana helm/reana -n {namespace} --create-namespace --wait -f -\n{values}\nEOF".format(
namespace=namespace,
values=values_dict and yaml.dump(values_dict) or "",
)
cmds = []
if mode in ("debug"):
cmds.append("reana-dev python-install-eggs")
cmds.append("reana-dev git-submodule --update")
cmds.extend([
"helm dep update helm/reana",
helm_install,
"kubectl config set-context --current --namespace={}".format(
namespace),
os.path.join(
get_srcdir("reana"),
f"scripts/create-admin-user.sh {namespace} {instance_name} {admin_email} {admin_password}",
),
])
for cmd in cmds:
run_command(cmd, "reana")
def is_last_commit_release_commit(package):
"""Check whether the last commit is a release commit."""
current_commit = get_current_commit(get_srcdir(package))
return current_commit.split()[1] == "release:"
