def git_create_release_commit_command(component, version): # noqa: D301
"""Create a release commit for the specified components.
\b
:param components: The option ``component`` can be repeated. The value may
consist of:
* (1) standard component name such as
'reana-workflow-controller';
* (2) short component name such as 'r-w-controller';
* (3) special value '.' indicating component of the
current working directory;
* (4) special value 'CLUSTER' that will expand to
cover all REANA cluster components [default];
* (5) special value 'CLIENT' that will expand to
cover all REANA client components;
* (6) special value 'DEMO' that will expand
to include several runable REANA demo examples;
* (7) special value 'ALL' that will expand to include
all REANA repositories.
:param version: Manually specifies the version for the component. If not provided,
the last version will be auto-incremented..
:type component: str
:type version: str
"""
components = select_components(component)
for component in components:
if git_create_release_commit(component, next_version=version):
display_message("Release commit created.", component)
def git_checkout_pr(branch, fetch): # noqa: D301
"""Check out local branch corresponding to a component pull request.
The ``-b`` option can be repetitive to check out several pull requests in
several repositories at the same time.
\b
:param branch: The option ``branch`` can be repeated. The value consist of
two strings specifying the component name and the pull
request number. For example, ``-b reana-workflow-controler
72`` will create a local branch called ``pr-72`` in the
reana-workflow-controller source code directory.
:param fetch: Should we fetch latest upstream first? [default=False]
:type branch: list
:type fetch: bool
"""
for cpr in branch:
component, pull_request = cpr
component = select_components([
component,
])[0]
if component in REPO_LIST_ALL:
if fetch:
cmd = "git fetch upstream"
run_command(cmd, component)
cmd = "git checkout -b pr-{0} upstream/pr/{0}".format(pull_request)
run_command(cmd, component)
else:
msg = "Ignoring unknown component."
display_message(msg, component)
def docker_rmi(user, tag, component): # noqa: D301
"""Remove REANA component images.
\b
:param components: The option ``component`` can be repeated. The value may
consist of:
* (1) standard component name such as
'reana-workflow-controller';
* (2) short component name such as 'r-w-controller';
* (3) special value '.' indicating component of the
current working directory;
* (4) special value 'CLUSTER' that will expand to
cover all REANA cluster components [default];
* (5) special value 'CLIENT' that will expand to
cover all REANA client components;
* (6) special value 'DEMO' that will expand
to include several runable REANA demo examples;
* (7) special value 'ALL' that will expand to include
all REANA repositories.
:param user: DockerHub organisation or user name. [default=reanahub]
:param tag: Docker tag to use. [default=latest]
:type component: str
:type user: str
:type tag: str
"""
components = select_components(component)
for component in components:
if is_component_dockerised(component):
cmd = "docker rmi {0}/{1}:{2}".format(user, component, tag)
run_command(cmd, component)
else:
msg = "Ignoring this component that does not contain" " a Dockerfile."
display_message(msg, component)
def is_component_releasable(component, exit_code=False, display=False):
"""Determine whether a component is releasable.
Last commit should be a release commit and the new version should be git tagged.
:param component: Component to determine whether if it is releasable or not.
:param exit_code: Whether the program should exit with error exit code if
the condition is not met.
:param display: Whether error messages providing instructions on how to fix
the problem should be displayed to stdout.
:type component: str
:type exit_code: bool
:type display: bool
:rtype: bool
"""
is_releasable = True
error_message = ""
if not is_last_commit_release_commit(component):
error_message = "The last commit is not a release commit. Please use `reana-dev git-create-release-commit`."
is_releasable = False
if not git_is_current_version_tagged(component):
error_message = (
"The current version is not tagged. Please use `reana-dev git-tag`."
)
is_releasable = False
if error_message and display:
display_message(error_message, component)
if not is_releasable and exit_code:
sys.exit(1)
return is_releasable
def git_create_pr_command(component, exclude_components, base): # noqa: D301
"""Create a GitHub pull request for each selected component.
\b
:param components: The option ``component`` can be repeated. The value may
consist of:
* (1) standard component name such as
'reana-workflow-controller';
* (2) short component name such as 'r-w-controller';
* (3) special value '.' indicating component of the
current working directory;
* (4) special value 'CLUSTER' that will expand to
cover all REANA cluster components [default];
* (5) special value 'CLIENT' that will expand to
cover all REANA client components;
* (6) special value 'DEMO' that will expand
to include several runable REANA demo examples;
* (7) special value 'ALL' that will expand to include
all REANA repositories.
:param exclude_components: List of components to exclude from command.
:param base: Against which git base branch are we working on? [default=master]
:type component: str
:type exclude_components: str
:type base: str
"""
def _git_create_pr(comp):
"""Create a pull request for the provided component."""
for cmd in [
"git push origin HEAD",
"hub pull-request -p --no-edit",
"hub pr list -L 1",
]:
run_command(cmd, component)
if exclude_components:
exclude_components = exclude_components.split(",")
components = select_components(component, exclude_components)
for component in components:
if not is_feature_branch(
component): # replace with is_feature_branch from #371
display_message(
"You are trying to create PR but the current branch is base branch {}, please "
"switch to the wanted feature branch.".format(base),
component,
)
sys.exit(1)
else:
if is_component_behind_branch(component,
"upstream/{}".format(base)):
print_branch_difference_report(component,
"upstream/{}".format(base))
display_message(
"Error: please rebase your feature branch against latest base branch {}."
.format(base),
component,
)
sys.exit(1)
_git_create_pr(component)
def git_clone(user, component, exclude_components): # noqa: D301
"""Clone REANA source repositories from GitHub.
If the ``user`` argument is provided, the ``origin`` will be cloned from
the user repository on GitHub and the ``upstream`` will be set to
``reanahub`` organisation. Useful for setting up personal REANA development
environment,
If the ``user`` argument is not provided, the cloning will be done in
anonymous manner from ``reanahub`` organisation. Also, the clone will be
shallow to save disk space and CPU time. Useful for CI purposes.
\b
:param user: The GitHub user name. [default=anonymous]
:param components: The option ``component`` can be repeated. The value may
consist of:
* (1) standard component name such as
'reana-workflow-controller';
* (2) short component name such as 'r-w-controller';
* (3) special value '.' indicating component of the
current working directory;
* (4) special value 'CLUSTER' that will expand to
cover all REANA cluster components [default];
* (5) special value 'CLIENT' that will expand to
cover all REANA client components;
* (6) special value 'DEMO' that will expand
to include several runable REANA demo examples;
* (7) special value 'ALL' that will expand to include
all REANA repositories.
:param exclude_components: List of components to exclude from command.
:type user: str
:type component: str
:type exclude_components: str
"""
if exclude_components:
exclude_components = exclude_components.split(",")
components = select_components(component, exclude_components)
for component in components:
os.chdir(get_srcdir())
if os.path.exists("{0}/.git/config".format(component)):
msg = "Component seems already cloned. Skipping."
display_message(msg, component)
elif user == "anonymous":
cmd = "git clone https://github.com/reanahub/{0} --depth 1".format(
component)
run_command(cmd)
else:
cmd = "git clone [email protected]:{0}/{1}".format(user, component)
run_command(cmd)
for cmd in [
"git remote add upstream"
' "[email protected]:reanahub/{0}"'.format(component),
"git config --add remote.upstream.fetch"
' "+refs/pull/*/head:refs/remotes/upstream/pr/*"',
]:
run_command(cmd, component)
def cluster_undeploy(): # noqa: D301
"""Undeploy REANA cluster."""
is_deployed = run_command("helm ls", "reana", return_output=True)
if "reana" in is_deployed:
for cmd in [
"helm uninstall reana -n default",
"kubectl get secrets -o custom-columns=':metadata.name' | grep reana | xargs kubectl delete secret",
"docker exec -i -t kind-control-plane sh -c '/bin/rm -rf /var/reana/*'",
]:
run_command(cmd, "reana")
else:
msg = "No REANA cluster to undeploy."
display_message(msg, "reana")
def select_workflow_engines(workflow_engines):
"""Return known workflow engine names that REANA supports.
:param workflow_engines: A list of workflow engine names such as 'cwl'.
:type components: list
:return: Unique workflow engine names.
:rtype: list
"""
output = set([])
for workflow_engine in workflow_engines:
if workflow_engine in WORKFLOW_ENGINE_LIST_ALL:
output.add(workflow_engine)
else:
display_message("Ignoring unknown workflow engine {0}.".format(
workflow_engine))
return list(output)
def _upgrade_docker_images(file_path: str, new_docker_images: List[str]) -> None:
"""Upgrade docker images in the file_path.
Read the content of the provided file_path,
replace docker image strings in the content with the provided new docker images (name + tag),
save the updated content back to the file_path
"""
with open(file_path, "r") as f:
file_content = f.read()
file_content = _replace_docker_images(file_content, new_docker_images)
with open(file_path, "w") as f:
f.write(file_content)
display_message(
f"Docker images in {file_path} successfully updated.", component="reana"
)
def git_merge(branch, base, push): # noqa: D301
"""Merge a component pull request to local base branch.
The ``-b`` option can be repetitive to merge several pull requests in
several repositories at the same time.
\b
:param branch: The option ``branch`` can be repeated. The value consist of
two strings specifying the component name and the pull
request number. For example, ``-b reana-workflow-controler
72`` will merge a local branch called ``pr-72`` from the
reana-workflow-controller to the base branch.
:param base: Against which git base branch are we working on? [default=master]
:param push: Should we push to origin and upstream? [default=False]
:type base: str
:type branch: list
:type push: bool
"""
for cpr in branch:
component, pull_request = cpr
component = select_components([
component,
])[0]
if component in REPO_LIST_ALL:
for cmd in [
"git fetch upstream",
"git diff pr-{0}..upstream/pr/{0} --exit-code".format(
pull_request),
"git checkout {0}".format(base),
"git merge --ff-only upstream/{0}".format(base),
"git merge --ff-only upstream/pr/{0}".format(pull_request),
"git branch -d pr-{0}".format(pull_request),
]:
run_command(cmd, component)
if push:
for cmd in [
"git push origin {0}".format(base),
"git push upstream {0}".format(base),
]:
run_command(cmd, component)
else:
msg = "Ignoring unknown component."
display_message(msg, component)
def kind_load_docker_image(user, component, node,
exclude_components): # noqa: D301
"""Load Docker images to the cluster.
\b
:param user: DockerHub organisation or user name. [default=reanahub]
:param components: The option ``component`` can be repeated. The value may
consist of:
* (1) standard component name such as
'reana-workflow-controller';
* (2) short component name such as 'r-w-controller';
* (3) special value '.' indicating component of the
current working directory;
* (4) special value 'CLUSTER' that will expand to
cover all REANA cluster components [default];
* (5) special value 'CLIENT' that will expand to
cover all REANA client components;
* (6) special value 'DEMO' that will expand
to include several runable REANA demo examples;
* (7) special value 'ALL' that will expand to include
all REANA repositories.
:param exclude_components: List of components to exclude from the build.
:type user: str
:type component: str
:type exclude_components: str
"""
if exclude_components:
exclude_components = exclude_components.split(",")
for component in select_components(component, exclude_components):
if component in DOCKER_PREFETCH_IMAGES:
for image in DOCKER_PREFETCH_IMAGES[component]:
cmd = "kind load docker-image {0}".format(image)
if node:
cmd = f"{cmd} --nodes {','.join(node)}"
run_command(cmd, component)
elif is_component_dockerised(component):
cmd = "kind load docker-image {0}/{1}".format(user, component)
if node:
cmd = f"{cmd} --nodes {','.join(node)}"
run_command(cmd, component)
else:
msg = "Ignoring this component that does not contain" " a Dockerfile."
display_message(msg, component)
def select_compute_backends(compute_backends):
"""Return known compute backends names that REANA supports.
:param workflow_engines: A list of compute backends names such as 'slurmcern'.
:type components: list
:return: Unique compute backend names.
:rtype: list
"""
def _has_cern_secrets():
"""Check whether the test user has the correct CERN secrets?."""
output = subprocess.check_output(["reana-client",
"secrets-list"]).decode("UTF-8")
required_cern_secrets = [
"CERN_KEYTAB",
"CERN_USER",
]
return all(sec in output for sec in required_cern_secrets)
output = set([])
has_selected_cern_compute_backend = False
for compute_backend in compute_backends:
if compute_backend in COMPUTE_BACKEND_LIST_ALL:
if "cern" in compute_backend:
has_selected_cern_compute_backend = True
output.add(compute_backend)
elif compute_backend.upper() == "ALL":
output = COMPUTE_BACKEND_LIST_ALL
has_selected_cern_compute_backend = True
break
else:
display_message("Ignoring unknown compute backend {0}.".format(
compute_backend))
if has_selected_cern_compute_backend and not _has_cern_secrets():
click.secho(
"You are trying to use a CERN compute backend but you don't have "
"the correct secrets setup.\nPlease follow "
"http://docs.reana.io/advanced-usage/access-control/kerberos/#uploading-secrets."
)
sys.exit(1)
return list(output)
def git_upgrade(component, exclude_components, base): # noqa: D301
"""Upgrade REANA local source code repositories and push to GitHub origin.
\b
:param components: The option ``component`` can be repeated. The value may
consist of:
* (1) standard component name such as
'reana-workflow-controller';
* (2) short component name such as 'r-w-controller';
* (3) special value '.' indicating component of the
current working directory;
* (4) special value 'CLUSTER' that will expand to
cover all REANA cluster components [default];
* (5) special value 'CLIENT' that will expand to
cover all REANA client components;
* (6) special value 'DEMO' that will expand
to include several runable REANA demo examples;
* (7) special value 'ALL' that will expand to include
all REANA repositories.
:param exclude_components: List of components to exclude from command.
:param base: Against which git base branch are we working on? [default=master]
:type component: str
:type exclude_components: str
:type base: str
"""
if exclude_components:
exclude_components = exclude_components.split(",")
components = select_components(component, exclude_components)
for component in components:
if not branch_exists(component, base):
display_message("Missing branch {}, skipping.".format(base),
component=component)
continue
for cmd in [
"git fetch upstream",
"git checkout {0}".format(base),
"git merge --ff-only upstream/{0}".format(base),
"git push origin {0}".format(base),
"git checkout -",
]:
run_command(cmd, component)
def cluster_delete(mounts): # noqa: D301
"""Delete REANA cluster.
\b
Example:
$ reana-dev cluster-delete -m /var/reana:/var/reana
"""
cmds = []
# delete cluster
cmds.append("kind delete cluster")
# remove only local paths where cluster path starts with /var/reana for safety
for mount in mounts:
local_path, cluster_node_path = mount.split(":")
if cluster_node_path.startswith("/var/reana"):
cmds.append("sudo rm -rf {}/*".format(local_path))
else:
msg = "Directory {} will not be deleted for safety reasons.".format(
local_path)
display_message(msg, "reana")
# execute commands
for cmd in cmds:
run_command(cmd, "reana")
def _update_values_yaml(new_docker_images):
"""Update all images in ``values.yaml``, skipping the ones up to date."""
values_yaml_relative_path = "helm/reana/values.yaml"
values_yaml_abs_path = os.path.join(
get_srcdir("reana"), values_yaml_relative_path
)
values_yaml = ""
with open(values_yaml_abs_path) as f:
values_yaml = f.read()
for docker_image in new_docker_images:
image_name, _ = docker_image.split(":")
if image_name in values_yaml:
values_yaml = re.sub(
f"{image_name}:.*", lambda _: docker_image, values_yaml, count=1
)
with open(values_yaml_abs_path, "w") as f:
f.write(values_yaml)
display_message(
f"{values_yaml_relative_path} successfully updated.", component="reana"
)
def git_create_release_commit(component,
base=GIT_DEFAULT_BASE_BRANCH,
next_version=None):
"""Create a release commit for the given component."""
if "release:" in get_current_commit(get_srcdir(component)):
display_message("Nothing to do, last commit is a release commit.",
component)
return False
current_version = get_current_component_version_from_source_files(
component)
if not current_version and not next_version:
display_message("Version cannot be autodiscovered from source files.",
component)
sys.exit(1)
elif not git_is_current_version_tagged(component) and not next_version:
display_message(
f"Current version ({current_version}) "
"not present as a git tag, please release it and add a tag.",
component,
)
sys.exit(1)
next_version, modified_files = bump_component_version(
component, current_version, next_version=next_version)
if (run_command(
"git branch --show-current",
component,
display=False,
return_output=True,
) == base):
run_command(f"git checkout -b release-{next_version}", component)
if modified_files:
run_command(f"git add {' '.join(modified_files)}", component)
run_command(
f"git commit -m 'release: {next_version}' {'--allow-empty' if not modified_files else ''}",
component,
)
return True
def run_example(component, workflow_engine, file, timecheck, timeout,
parameters, options): # noqa: D301
"""Run given REANA example with given workflow engine.
\b
Example:
$ reana-dev run-example -c r-d-r-roofit
\b
:param component: The option ``component`` can be repeated. The value is
the repository name of the example. The special value `DEMO`
will run all examples.
[default=reana-demo-root6-roofit]
:param workflow_engine: The option ``workflow_engine`` can be repeated. The
value is the workflow engine to use to run the
example. [default=cwl,serial,yadage]
:param file: The option ``file`` can be repeated. The value is the expected
output file the workflow should produce. [default=plot.png]
:param timecheck: Checking frequency in seconds for results.
[default=5 (TIMECHECK)]
:param timeout: Maximum timeout to wait for results.
[default=300 (TIMEOUT)]
:param parameters: Additional input parameters to override original ones
from reana.yaml.
E.g. -p myparam1=myval1 -p myparam2=myval2.
:param options: Additional operational options for the workflow execution.
E.g. CACHE=off.
:type component: str
:type workflow_engine: str
:type sleep: int
"""
components = select_components(component)
workflow_engines = select_workflow_engines(workflow_engine)
reana_yaml = {
"cwl": "reana-cwl.yaml",
"serial": "reana.yaml",
"yadage": "reana-yadage.yaml",
}
for component in components:
for workflow_engine in workflow_engines:
workflow_name = construct_workflow_name(component, workflow_engine)
# check whether example contains recipe for given engine
if not os.path.exists(
get_srcdir(component) + os.sep +
reana_yaml[workflow_engine]):
msg = "Skipping example with workflow engine {0}.".format(
workflow_engine)
display_message(msg, component)
continue
# create workflow:
for cmd in [
"reana-client create -f {0} -n {1}".format(
reana_yaml[workflow_engine], workflow_name),
]:
run_command(cmd, component)
# upload inputs
for cmd in [
"reana-client upload -w {0}".format(workflow_name),
]:
run_command(cmd, component)
# run workflow
input_parameters = " ".join(
["-p " + parameter for parameter in parameters])
operational_options = " ".join(
["-o " + option for option in options])
for cmd in [
"reana-client start -w {0} {1} {2}".format(
workflow_name, input_parameters, operational_options),
]:
run_command(cmd, component)
# verify whether job finished within time limits
time_start = time.time()
while time.time() - time_start <= timeout:
time.sleep(timecheck)
cmd = "reana-client status -w {0}".format(workflow_name)
status = run_command(cmd, component, return_output=True)
click.secho(status)
if "finished" in status or "failed" in status or "stopped" in status:
break
# verify logs message presence
for log_message in get_expected_log_messages_for_example(
component):
cmd = "reana-client logs -w {0} | grep -c '{1}'".format(
workflow_name, log_message)
run_command(cmd, component)
# verify output file presence
cmd = "reana-client ls -w {0}".format(workflow_name)
listing = run_command(cmd, component, return_output=True)
click.secho(listing)
expected_files = file or get_expected_output_filenames_for_example(
component)
for expected_file in expected_files:
if expected_file not in listing:
click.secho("[ERROR] Expected output file {0} not found. "
"Exiting.".format(expected_file))
sys.exit(1)
# report that everything was OK
run_command("echo OK", component)
def docker_build(
ctx,
user,
tag,
component,
build_arg,
no_cache,
output_component_versions,
quiet,
exclude_components,
): # noqa: D301
"""Build REANA component images.
\b
:param components: The option ``component`` can be repeated. The value may
consist of:
* (1) standard component name such as
'reana-workflow-controller';
* (2) short component name such as 'r-w-controller';
* (3) special value '.' indicating component of the
current working directory;
* (4) special value 'CLUSTER' that will expand to
cover all REANA cluster components [default];
* (5) special value 'CLIENT' that will expand to
cover all REANA client components;
* (6) special value 'DEMO' that will expand
to include several runable REANA demo examples;
* (7) special value 'ALL' that will expand to include
all REANA repositories.
:param exclude_components: List of components to exclude from the build.
:param user: DockerHub organisation or user name. [default=reanahub]
:param tag: Docker image tag to generate. Default 'latest'. Use 'auto' to
generate git-tag-based value such as '0.7.0-alpha.3'.
:param build_arg: Optional docker build argument. (e.g. DEBUG=1)
:param no_cache: Flag instructing to avoid using cache. [default=False]
:param output_component_versions: File where to write the built images
tags. Useful when using `--tag auto` since every REANA component
will have a different tag.
:type component: str
:type exclude_components: str
:type user: str
:type tag: str
:type build_arg: str
:type no_cache: bool
:type output_component_versions: File
:type quiet: bool
"""
if exclude_components:
exclude_components = exclude_components.split(",")
components = select_components(component, exclude_components)
built_components_versions_tags = []
for component in components:
component_tag = tag
if is_component_dockerised(component):
cmd = "docker build"
if tag == "auto":
component_tag = get_docker_tag(component)
for arg in build_arg:
cmd += " --build-arg {0}".format(arg)
if no_cache:
cmd += " --no-cache"
if quiet:
cmd += " --quiet"
component_version_tag = "{0}/{1}:{2}".format(
user, component, component_tag)
cmd += " -t {0} .".format(component_version_tag)
run_command(cmd, component)
built_components_versions_tags.append(component_version_tag)
else:
msg = "Ignoring this component that does not contain" " a Dockerfile."
display_message(msg, component)
if output_component_versions:
output_component_versions.write(
"\n".join(built_components_versions_tags) + "\n")
def run_example(
component,
workflow_engine,
compute_backend,
file,
timecheck,
timeout,
parameters,
options,
submit_only,
check_only,
): # noqa: D301
"""Run given REANA example with given workflow engine.
\b
Example:
$ reana-dev run-example -c r-d-r-roofit
\b
:param component: The option ``component`` can be repeated. The value is
the repository name of the example. The special value `DEMO`
will run all examples.
[default=reana-demo-root6-roofit]
:param workflow_engine: The option ``workflow_engine`` can be repeated. The
value is the workflow engine to use to run the
example. [default=cwl,serial,yadage]
:param compute_backend: The option ``compute_backend`` can be repeated. The
value is the compute backend to use to run the
example. [default=kubernetes,htcondorcern,slurmcern]
:param file: The option ``file`` can be repeated. The value is the expected
output file the workflow should produce. [default=plot.png]
:param timecheck: Checking frequency in seconds for results.
[default=5 (TIMECHECK)]
:param timeout: Maximum timeout to wait for results.
[default=300 (TIMEOUT)]
:param parameters: Additional input parameters to override original ones
from reana.yaml.
E.g. -p myparam1=myval1 -p myparam2=myval2.
:param options: Additional operational options for the workflow execution.
E.g. CACHE=off.
:param submit_only: Do not wait for workflows to finish.
:param check_only: Wait for previously submitted workflows.
:type component: str
:type workflow_engine: list
:type compute_backend: list
:type file: list
:type timecheck: int
:type timeout: int
:type parameters: list
:type options: list
:type submit_only: bool
:type check_only: bool
"""
if submit_only and check_only:
click.secho(
"[ERROR] Options --submit-only and --check-only are mutually exclusive. Choose only one."
)
sys.exit(1)
components = select_components(component)
workflow_engines = select_workflow_engines(workflow_engine)
compute_backends = select_compute_backends(compute_backend)
display_message("Running the following matrix:\n"
"Demos: {}\n"
"Workflow engines: {}\n"
"Compute backends: {}".format(",".join(components),
",".join(workflow_engines),
",".join(compute_backends)))
for component in components:
for workflow_engine in workflow_engines:
for compute_backend in compute_backends:
reana_yaml_file_path = get_example_reana_yaml_file_path(
component, workflow_engine, compute_backend)
if not reana_yaml_file_path:
msg = "Skipping example {0} with workflow engine {1} and compute backend {2}.".format(
component, workflow_engine, compute_backend)
display_message(msg, component)
continue
workflow_name = construct_workflow_name(
component, workflow_engine, compute_backend)
if not check_only:
# create workflow:
for cmd in [
"reana-client create -f {0} -n {1}".format(
reana_yaml_file_path,
workflow_name,
),
]:
run_command(cmd, component)
# upload inputs
for cmd in [
"reana-client upload -w {0}".format(workflow_name),
]:
run_command(cmd, component)
# run workflow
input_parameters = " ".join(
["-p " + parameter for parameter in parameters])
operational_options = " ".join(
["-o " + option for option in options])
for cmd in [
"reana-client start -w {0} {1} {2}".format(
workflow_name, input_parameters,
operational_options),
]:
run_command(cmd, component)
if not submit_only:
# verify whether job finished within time limits
time_start = time.time()
while time.time() - time_start <= timeout:
time.sleep(timecheck)
cmd = "reana-client status -w {0}".format(
workflow_name)
status = run_command(cmd,
component,
return_output=True)
click.secho(status)
if ("finished" in status or "failed" in status
or "stopped" in status):
break
# verify logs message presence
for log_message in get_expected_log_messages_for_example(
component):
cmd = "reana-client logs -w {0} | grep -c '{1}'".format(
workflow_name, log_message)
run_command(cmd, component)
# verify output file presence
cmd = "reana-client ls -w {0}".format(workflow_name)
listing = run_command(cmd, component, return_output=True)
click.secho(listing)
expected_files = file or get_expected_output_filenames_for_example(
component)
for expected_file in expected_files:
if expected_file not in listing:
click.secho(
"[ERROR] Expected output file {0} not found. "
"Exiting.".format(expected_file))
sys.exit(1)
# report that everything was OK
run_command("echo OK", component)
def python_unit_tests(component): # noqa: D301
"""Run Python unit tests in independent environments.
For each component, create a dedicated throw-away virtual environment,
install latest shared modules (reana-commons, reana-db) that are currently
checked-out and run the usual component unit tests. Delete the throw-away
virtual environment afterwards.
\b
:param components: The option ``component`` can be repeated. The value may
consist of:
* (1) standard component name such as
'reana-workflow-controller';
* (2) short component name such as 'r-w-controller';
* (3) special value '.' indicating component of the
current working directory;
* (4) special value 'CLUSTER' that will expand to
cover all REANA cluster components [default];
* (5) special value 'CLIENT' that will expand to
cover all REANA client components;
* (6) special value 'DEMO' that will expand
to include several runable REANA demo examples;
* (7) special value 'ALL' that will expand to include
all REANA repositories.
:type component: str
"""
components = select_components(component)
for component in components:
if component == "reana-job-controller" and platform.system(
) == "Darwin":
msg = ("Ignoring component {} that cannot be tested"
" on a macOS platform yet.".format(component))
display_message(msg, component)
elif is_component_python_package(component):
cmd_activate_venv = "source ~/.virtualenvs/_{}/bin/activate".format(
component)
if does_component_need_db(component):
run_command(
f"docker stop postgres__{component}\n"
f"docker run --rm --name postgres__{component} -p 5432:5432 "
"-e POSTGRES_PASSWORD=mysecretpassword -d postgres:9.6.2")
for cmd in [
"virtualenv ~/.virtualenvs/_{}".format(component),
"{} && which python".format(cmd_activate_venv),
"{} && cd ../pytest-reana && "
" pip install . --upgrade".format(cmd_activate_venv),
"{} && cd ../reana-commons && "
" pip install . --upgrade".format(cmd_activate_venv),
"{} && cd ../reana-db && "
" pip install . --upgrade".format(cmd_activate_venv),
"git clean -d -ff -x",
'{} && pip install ".[tests]" --upgrade'.format(
cmd_activate_venv),
"{} && {} python setup.py test".format(
cmd_activate_venv,
"REANA_SQLALCHEMY_DATABASE_URI=postgresql+psycopg2://postgres:mysecretpassword@localhost/postgres"
if does_component_need_db(component) else "",
),
"rm -rf ~/.virtualenvs/_{}".format(component),
]:
run_command(cmd, component)
if does_component_need_db(component):
run_command(f"docker stop postgres__{component}")
else:
msg = ("Ignoring this component that does not contain"
" a Python setup.py file.")
display_message(msg, component)
def git_log(
range,
component,
exclude_components,
number,
graph,
oneline,
stat,
patch,
all,
paginate,
): # noqa: D301
"""Show commit logs in given component repositories.
\b
:param range: The commit log range to operate on.
:param component: The option ``component`` can be repeated. The value may
consist of:
* (1) standard component name such as
'reana-workflow-controller';
* (2) short component name such as 'r-w-controller';
* (3) special value '.' indicating component of the
current working directory;
* (4) special value 'CLUSTER' that will expand to
cover all REANA cluster components [default];
* (5) special value 'CLIENT' that will expand to
cover all REANA client components;
* (6) special value 'DEMO' that will expand
to include several runable REANA demo examples;
* (7) special value 'ALL' that will expand to include
all REANA repositories.
:param exclude_components: List of components to exclude from command.
:param number: The number of commits to output. [10]
:param graph: Show log graph?
:param oneline: Show one-line format?
:param patch: Show diff patch?
:param all: Show all references?
:param paginate: Paginate output?
:type range: str
:type component: str
:type exclude_components: str
:type number: int
:type graph: bool
:type oneline: bool
:type stat: bool
:type patch: bool
:type all: bool
:type paginate: bool
"""
if exclude_components:
exclude_components = exclude_components.split(",")
components = select_components(component, exclude_components)
for component in components:
if paginate:
cmd = "git --paginate log"
else:
cmd = "git --no-pager log"
if number:
cmd += " -n {}".format(number)
if graph:
cmd += (" --graph --decorate"
' --pretty=format:"%C(blue)%d%Creset'
" %C(yellow)%h%Creset %s, %C(bold green)%an%Creset,"
' %C(green)%cd%Creset" --date=relative')
if oneline or graph or all:
cmd += " --oneline"
if stat:
cmd += " --stat"
if patch:
cmd += " --patch"
if all:
cmd += " --all"
if range:
cmd += " {}".format(range)
msg = cmd[0:cmd.find("--pretty")] + "..."
display_message(msg, component)
run_command(cmd, component, display=False)
