def git_create_release_commit_command(component, version): # noqa: D301 """Create a release commit for the specified components. \b :param components: The option ``component`` can be repeated. The value may consist of: * (1) standard component name such as 'reana-workflow-controller'; * (2) short component name such as 'r-w-controller'; * (3) special value '.' indicating component of the current working directory; * (4) special value 'CLUSTER' that will expand to cover all REANA cluster components [default]; * (5) special value 'CLIENT' that will expand to cover all REANA client components; * (6) special value 'DEMO' that will expand to include several runable REANA demo examples; * (7) special value 'ALL' that will expand to include all REANA repositories. :param version: Manually specifies the version for the component. If not provided, the last version will be auto-incremented.. :type component: str :type version: str """ components = select_components(component) for component in components: if git_create_release_commit(component, next_version=version): display_message("Release commit created.", component)
def git_checkout_pr(branch, fetch): # noqa: D301 """Check out local branch corresponding to a component pull request. The ``-b`` option can be repetitive to check out several pull requests in several repositories at the same time. \b :param branch: The option ``branch`` can be repeated. The value consist of two strings specifying the component name and the pull request number. For example, ``-b reana-workflow-controler 72`` will create a local branch called ``pr-72`` in the reana-workflow-controller source code directory. :param fetch: Should we fetch latest upstream first? [default=False] :type branch: list :type fetch: bool """ for cpr in branch: component, pull_request = cpr component = select_components([ component, ])[0] if component in REPO_LIST_ALL: if fetch: cmd = "git fetch upstream" run_command(cmd, component) cmd = "git checkout -b pr-{0} upstream/pr/{0}".format(pull_request) run_command(cmd, component) else: msg = "Ignoring unknown component." display_message(msg, component)
def docker_rmi(user, tag, component): # noqa: D301 """Remove REANA component images. \b :param components: The option ``component`` can be repeated. The value may consist of: * (1) standard component name such as 'reana-workflow-controller'; * (2) short component name such as 'r-w-controller'; * (3) special value '.' indicating component of the current working directory; * (4) special value 'CLUSTER' that will expand to cover all REANA cluster components [default]; * (5) special value 'CLIENT' that will expand to cover all REANA client components; * (6) special value 'DEMO' that will expand to include several runable REANA demo examples; * (7) special value 'ALL' that will expand to include all REANA repositories. :param user: DockerHub organisation or user name. [default=reanahub] :param tag: Docker tag to use. [default=latest] :type component: str :type user: str :type tag: str """ components = select_components(component) for component in components: if is_component_dockerised(component): cmd = "docker rmi {0}/{1}:{2}".format(user, component, tag) run_command(cmd, component) else: msg = "Ignoring this component that does not contain" " a Dockerfile." display_message(msg, component)
def is_component_releasable(component, exit_code=False, display=False): """Determine whether a component is releasable. Last commit should be a release commit and the new version should be git tagged. :param component: Component to determine whether if it is releasable or not. :param exit_code: Whether the program should exit with error exit code if the condition is not met. :param display: Whether error messages providing instructions on how to fix the problem should be displayed to stdout. :type component: str :type exit_code: bool :type display: bool :rtype: bool """ is_releasable = True error_message = "" if not is_last_commit_release_commit(component): error_message = "The last commit is not a release commit. Please use `reana-dev git-create-release-commit`." is_releasable = False if not git_is_current_version_tagged(component): error_message = ( "The current version is not tagged. Please use `reana-dev git-tag`." ) is_releasable = False if error_message and display: display_message(error_message, component) if not is_releasable and exit_code: sys.exit(1) return is_releasable
def git_create_pr_command(component, exclude_components, base): # noqa: D301 """Create a GitHub pull request for each selected component. \b :param components: The option ``component`` can be repeated. The value may consist of: * (1) standard component name such as 'reana-workflow-controller'; * (2) short component name such as 'r-w-controller'; * (3) special value '.' indicating component of the current working directory; * (4) special value 'CLUSTER' that will expand to cover all REANA cluster components [default]; * (5) special value 'CLIENT' that will expand to cover all REANA client components; * (6) special value 'DEMO' that will expand to include several runable REANA demo examples; * (7) special value 'ALL' that will expand to include all REANA repositories. :param exclude_components: List of components to exclude from command. :param base: Against which git base branch are we working on? [default=master] :type component: str :type exclude_components: str :type base: str """ def _git_create_pr(comp): """Create a pull request for the provided component.""" for cmd in [ "git push origin HEAD", "hub pull-request -p --no-edit", "hub pr list -L 1", ]: run_command(cmd, component) if exclude_components: exclude_components = exclude_components.split(",") components = select_components(component, exclude_components) for component in components: if not is_feature_branch( component): # replace with is_feature_branch from #371 display_message( "You are trying to create PR but the current branch is base branch {}, please " "switch to the wanted feature branch.".format(base), component, ) sys.exit(1) else: if is_component_behind_branch(component, "upstream/{}".format(base)): print_branch_difference_report(component, "upstream/{}".format(base)) display_message( "Error: please rebase your feature branch against latest base branch {}." .format(base), component, ) sys.exit(1) _git_create_pr(component)
def git_clone(user, component, exclude_components): # noqa: D301 """Clone REANA source repositories from GitHub. If the ``user`` argument is provided, the ``origin`` will be cloned from the user repository on GitHub and the ``upstream`` will be set to ``reanahub`` organisation. Useful for setting up personal REANA development environment, If the ``user`` argument is not provided, the cloning will be done in anonymous manner from ``reanahub`` organisation. Also, the clone will be shallow to save disk space and CPU time. Useful for CI purposes. \b :param user: The GitHub user name. [default=anonymous] :param components: The option ``component`` can be repeated. The value may consist of: * (1) standard component name such as 'reana-workflow-controller'; * (2) short component name such as 'r-w-controller'; * (3) special value '.' indicating component of the current working directory; * (4) special value 'CLUSTER' that will expand to cover all REANA cluster components [default]; * (5) special value 'CLIENT' that will expand to cover all REANA client components; * (6) special value 'DEMO' that will expand to include several runable REANA demo examples; * (7) special value 'ALL' that will expand to include all REANA repositories. :param exclude_components: List of components to exclude from command. :type user: str :type component: str :type exclude_components: str """ if exclude_components: exclude_components = exclude_components.split(",") components = select_components(component, exclude_components) for component in components: os.chdir(get_srcdir()) if os.path.exists("{0}/.git/config".format(component)): msg = "Component seems already cloned. Skipping." display_message(msg, component) elif user == "anonymous": cmd = "git clone https://github.com/reanahub/{0} --depth 1".format( component) run_command(cmd) else: cmd = "git clone [email protected]:{0}/{1}".format(user, component) run_command(cmd) for cmd in [ "git remote add upstream" ' "[email protected]:reanahub/{0}"'.format(component), "git config --add remote.upstream.fetch" ' "+refs/pull/*/head:refs/remotes/upstream/pr/*"', ]: run_command(cmd, component)
def cluster_undeploy(): # noqa: D301 """Undeploy REANA cluster.""" is_deployed = run_command("helm ls", "reana", return_output=True) if "reana" in is_deployed: for cmd in [ "helm uninstall reana -n default", "kubectl get secrets -o custom-columns=':metadata.name' | grep reana | xargs kubectl delete secret", "docker exec -i -t kind-control-plane sh -c '/bin/rm -rf /var/reana/*'", ]: run_command(cmd, "reana") else: msg = "No REANA cluster to undeploy." display_message(msg, "reana")
def select_workflow_engines(workflow_engines): """Return known workflow engine names that REANA supports. :param workflow_engines: A list of workflow engine names such as 'cwl'. :type components: list :return: Unique workflow engine names. :rtype: list """ output = set([]) for workflow_engine in workflow_engines: if workflow_engine in WORKFLOW_ENGINE_LIST_ALL: output.add(workflow_engine) else: display_message("Ignoring unknown workflow engine {0}.".format( workflow_engine)) return list(output)
def _upgrade_docker_images(file_path: str, new_docker_images: List[str]) -> None: """Upgrade docker images in the file_path. Read the content of the provided file_path, replace docker image strings in the content with the provided new docker images (name + tag), save the updated content back to the file_path """ with open(file_path, "r") as f: file_content = f.read() file_content = _replace_docker_images(file_content, new_docker_images) with open(file_path, "w") as f: f.write(file_content) display_message( f"Docker images in {file_path} successfully updated.", component="reana" )
def git_merge(branch, base, push): # noqa: D301 """Merge a component pull request to local base branch. The ``-b`` option can be repetitive to merge several pull requests in several repositories at the same time. \b :param branch: The option ``branch`` can be repeated. The value consist of two strings specifying the component name and the pull request number. For example, ``-b reana-workflow-controler 72`` will merge a local branch called ``pr-72`` from the reana-workflow-controller to the base branch. :param base: Against which git base branch are we working on? [default=master] :param push: Should we push to origin and upstream? [default=False] :type base: str :type branch: list :type push: bool """ for cpr in branch: component, pull_request = cpr component = select_components([ component, ])[0] if component in REPO_LIST_ALL: for cmd in [ "git fetch upstream", "git diff pr-{0}..upstream/pr/{0} --exit-code".format( pull_request), "git checkout {0}".format(base), "git merge --ff-only upstream/{0}".format(base), "git merge --ff-only upstream/pr/{0}".format(pull_request), "git branch -d pr-{0}".format(pull_request), ]: run_command(cmd, component) if push: for cmd in [ "git push origin {0}".format(base), "git push upstream {0}".format(base), ]: run_command(cmd, component) else: msg = "Ignoring unknown component." display_message(msg, component)
def kind_load_docker_image(user, component, node, exclude_components): # noqa: D301 """Load Docker images to the cluster. \b :param user: DockerHub organisation or user name. [default=reanahub] :param components: The option ``component`` can be repeated. The value may consist of: * (1) standard component name such as 'reana-workflow-controller'; * (2) short component name such as 'r-w-controller'; * (3) special value '.' indicating component of the current working directory; * (4) special value 'CLUSTER' that will expand to cover all REANA cluster components [default]; * (5) special value 'CLIENT' that will expand to cover all REANA client components; * (6) special value 'DEMO' that will expand to include several runable REANA demo examples; * (7) special value 'ALL' that will expand to include all REANA repositories. :param exclude_components: List of components to exclude from the build. :type user: str :type component: str :type exclude_components: str """ if exclude_components: exclude_components = exclude_components.split(",") for component in select_components(component, exclude_components): if component in DOCKER_PREFETCH_IMAGES: for image in DOCKER_PREFETCH_IMAGES[component]: cmd = "kind load docker-image {0}".format(image) if node: cmd = f"{cmd} --nodes {','.join(node)}" run_command(cmd, component) elif is_component_dockerised(component): cmd = "kind load docker-image {0}/{1}".format(user, component) if node: cmd = f"{cmd} --nodes {','.join(node)}" run_command(cmd, component) else: msg = "Ignoring this component that does not contain" " a Dockerfile." display_message(msg, component)
def select_compute_backends(compute_backends): """Return known compute backends names that REANA supports. :param workflow_engines: A list of compute backends names such as 'slurmcern'. :type components: list :return: Unique compute backend names. :rtype: list """ def _has_cern_secrets(): """Check whether the test user has the correct CERN secrets?.""" output = subprocess.check_output(["reana-client", "secrets-list"]).decode("UTF-8") required_cern_secrets = [ "CERN_KEYTAB", "CERN_USER", ] return all(sec in output for sec in required_cern_secrets) output = set([]) has_selected_cern_compute_backend = False for compute_backend in compute_backends: if compute_backend in COMPUTE_BACKEND_LIST_ALL: if "cern" in compute_backend: has_selected_cern_compute_backend = True output.add(compute_backend) elif compute_backend.upper() == "ALL": output = COMPUTE_BACKEND_LIST_ALL has_selected_cern_compute_backend = True break else: display_message("Ignoring unknown compute backend {0}.".format( compute_backend)) if has_selected_cern_compute_backend and not _has_cern_secrets(): click.secho( "You are trying to use a CERN compute backend but you don't have " "the correct secrets setup.\nPlease follow " "http://docs.reana.io/advanced-usage/access-control/kerberos/#uploading-secrets." ) sys.exit(1) return list(output)
def git_upgrade(component, exclude_components, base): # noqa: D301 """Upgrade REANA local source code repositories and push to GitHub origin. \b :param components: The option ``component`` can be repeated. The value may consist of: * (1) standard component name such as 'reana-workflow-controller'; * (2) short component name such as 'r-w-controller'; * (3) special value '.' indicating component of the current working directory; * (4) special value 'CLUSTER' that will expand to cover all REANA cluster components [default]; * (5) special value 'CLIENT' that will expand to cover all REANA client components; * (6) special value 'DEMO' that will expand to include several runable REANA demo examples; * (7) special value 'ALL' that will expand to include all REANA repositories. :param exclude_components: List of components to exclude from command. :param base: Against which git base branch are we working on? [default=master] :type component: str :type exclude_components: str :type base: str """ if exclude_components: exclude_components = exclude_components.split(",") components = select_components(component, exclude_components) for component in components: if not branch_exists(component, base): display_message("Missing branch {}, skipping.".format(base), component=component) continue for cmd in [ "git fetch upstream", "git checkout {0}".format(base), "git merge --ff-only upstream/{0}".format(base), "git push origin {0}".format(base), "git checkout -", ]: run_command(cmd, component)
def cluster_delete(mounts): # noqa: D301 """Delete REANA cluster. \b Example: $ reana-dev cluster-delete -m /var/reana:/var/reana """ cmds = [] # delete cluster cmds.append("kind delete cluster") # remove only local paths where cluster path starts with /var/reana for safety for mount in mounts: local_path, cluster_node_path = mount.split(":") if cluster_node_path.startswith("/var/reana"): cmds.append("sudo rm -rf {}/*".format(local_path)) else: msg = "Directory {} will not be deleted for safety reasons.".format( local_path) display_message(msg, "reana") # execute commands for cmd in cmds: run_command(cmd, "reana")
def _update_values_yaml(new_docker_images): """Update all images in ``values.yaml``, skipping the ones up to date.""" values_yaml_relative_path = "helm/reana/values.yaml" values_yaml_abs_path = os.path.join( get_srcdir("reana"), values_yaml_relative_path ) values_yaml = "" with open(values_yaml_abs_path) as f: values_yaml = f.read() for docker_image in new_docker_images: image_name, _ = docker_image.split(":") if image_name in values_yaml: values_yaml = re.sub( f"{image_name}:.*", lambda _: docker_image, values_yaml, count=1 ) with open(values_yaml_abs_path, "w") as f: f.write(values_yaml) display_message( f"{values_yaml_relative_path} successfully updated.", component="reana" )
def git_create_release_commit(component, base=GIT_DEFAULT_BASE_BRANCH, next_version=None): """Create a release commit for the given component.""" if "release:" in get_current_commit(get_srcdir(component)): display_message("Nothing to do, last commit is a release commit.", component) return False current_version = get_current_component_version_from_source_files( component) if not current_version and not next_version: display_message("Version cannot be autodiscovered from source files.", component) sys.exit(1) elif not git_is_current_version_tagged(component) and not next_version: display_message( f"Current version ({current_version}) " "not present as a git tag, please release it and add a tag.", component, ) sys.exit(1) next_version, modified_files = bump_component_version( component, current_version, next_version=next_version) if (run_command( "git branch --show-current", component, display=False, return_output=True, ) == base): run_command(f"git checkout -b release-{next_version}", component) if modified_files: run_command(f"git add {' '.join(modified_files)}", component) run_command( f"git commit -m 'release: {next_version}' {'--allow-empty' if not modified_files else ''}", component, ) return True
def run_example(component, workflow_engine, file, timecheck, timeout, parameters, options): # noqa: D301 """Run given REANA example with given workflow engine. \b Example: $ reana-dev run-example -c r-d-r-roofit \b :param component: The option ``component`` can be repeated. The value is the repository name of the example. The special value `DEMO` will run all examples. [default=reana-demo-root6-roofit] :param workflow_engine: The option ``workflow_engine`` can be repeated. The value is the workflow engine to use to run the example. [default=cwl,serial,yadage] :param file: The option ``file`` can be repeated. The value is the expected output file the workflow should produce. [default=plot.png] :param timecheck: Checking frequency in seconds for results. [default=5 (TIMECHECK)] :param timeout: Maximum timeout to wait for results. [default=300 (TIMEOUT)] :param parameters: Additional input parameters to override original ones from reana.yaml. E.g. -p myparam1=myval1 -p myparam2=myval2. :param options: Additional operational options for the workflow execution. E.g. CACHE=off. :type component: str :type workflow_engine: str :type sleep: int """ components = select_components(component) workflow_engines = select_workflow_engines(workflow_engine) reana_yaml = { "cwl": "reana-cwl.yaml", "serial": "reana.yaml", "yadage": "reana-yadage.yaml", } for component in components: for workflow_engine in workflow_engines: workflow_name = construct_workflow_name(component, workflow_engine) # check whether example contains recipe for given engine if not os.path.exists( get_srcdir(component) + os.sep + reana_yaml[workflow_engine]): msg = "Skipping example with workflow engine {0}.".format( workflow_engine) display_message(msg, component) continue # create workflow: for cmd in [ "reana-client create -f {0} -n {1}".format( reana_yaml[workflow_engine], workflow_name), ]: run_command(cmd, component) # upload inputs for cmd in [ "reana-client upload -w {0}".format(workflow_name), ]: run_command(cmd, component) # run workflow input_parameters = " ".join( ["-p " + parameter for parameter in parameters]) operational_options = " ".join( ["-o " + option for option in options]) for cmd in [ "reana-client start -w {0} {1} {2}".format( workflow_name, input_parameters, operational_options), ]: run_command(cmd, component) # verify whether job finished within time limits time_start = time.time() while time.time() - time_start <= timeout: time.sleep(timecheck) cmd = "reana-client status -w {0}".format(workflow_name) status = run_command(cmd, component, return_output=True) click.secho(status) if "finished" in status or "failed" in status or "stopped" in status: break # verify logs message presence for log_message in get_expected_log_messages_for_example( component): cmd = "reana-client logs -w {0} | grep -c '{1}'".format( workflow_name, log_message) run_command(cmd, component) # verify output file presence cmd = "reana-client ls -w {0}".format(workflow_name) listing = run_command(cmd, component, return_output=True) click.secho(listing) expected_files = file or get_expected_output_filenames_for_example( component) for expected_file in expected_files: if expected_file not in listing: click.secho("[ERROR] Expected output file {0} not found. " "Exiting.".format(expected_file)) sys.exit(1) # report that everything was OK run_command("echo OK", component)
def docker_build( ctx, user, tag, component, build_arg, no_cache, output_component_versions, quiet, exclude_components, ): # noqa: D301 """Build REANA component images. \b :param components: The option ``component`` can be repeated. The value may consist of: * (1) standard component name such as 'reana-workflow-controller'; * (2) short component name such as 'r-w-controller'; * (3) special value '.' indicating component of the current working directory; * (4) special value 'CLUSTER' that will expand to cover all REANA cluster components [default]; * (5) special value 'CLIENT' that will expand to cover all REANA client components; * (6) special value 'DEMO' that will expand to include several runable REANA demo examples; * (7) special value 'ALL' that will expand to include all REANA repositories. :param exclude_components: List of components to exclude from the build. :param user: DockerHub organisation or user name. [default=reanahub] :param tag: Docker image tag to generate. Default 'latest'. Use 'auto' to generate git-tag-based value such as '0.7.0-alpha.3'. :param build_arg: Optional docker build argument. (e.g. DEBUG=1) :param no_cache: Flag instructing to avoid using cache. [default=False] :param output_component_versions: File where to write the built images tags. Useful when using `--tag auto` since every REANA component will have a different tag. :type component: str :type exclude_components: str :type user: str :type tag: str :type build_arg: str :type no_cache: bool :type output_component_versions: File :type quiet: bool """ if exclude_components: exclude_components = exclude_components.split(",") components = select_components(component, exclude_components) built_components_versions_tags = [] for component in components: component_tag = tag if is_component_dockerised(component): cmd = "docker build" if tag == "auto": component_tag = get_docker_tag(component) for arg in build_arg: cmd += " --build-arg {0}".format(arg) if no_cache: cmd += " --no-cache" if quiet: cmd += " --quiet" component_version_tag = "{0}/{1}:{2}".format( user, component, component_tag) cmd += " -t {0} .".format(component_version_tag) run_command(cmd, component) built_components_versions_tags.append(component_version_tag) else: msg = "Ignoring this component that does not contain" " a Dockerfile." display_message(msg, component) if output_component_versions: output_component_versions.write( "\n".join(built_components_versions_tags) + "\n")
def run_example( component, workflow_engine, compute_backend, file, timecheck, timeout, parameters, options, submit_only, check_only, ): # noqa: D301 """Run given REANA example with given workflow engine. \b Example: $ reana-dev run-example -c r-d-r-roofit \b :param component: The option ``component`` can be repeated. The value is the repository name of the example. The special value `DEMO` will run all examples. [default=reana-demo-root6-roofit] :param workflow_engine: The option ``workflow_engine`` can be repeated. The value is the workflow engine to use to run the example. [default=cwl,serial,yadage] :param compute_backend: The option ``compute_backend`` can be repeated. The value is the compute backend to use to run the example. [default=kubernetes,htcondorcern,slurmcern] :param file: The option ``file`` can be repeated. The value is the expected output file the workflow should produce. [default=plot.png] :param timecheck: Checking frequency in seconds for results. [default=5 (TIMECHECK)] :param timeout: Maximum timeout to wait for results. [default=300 (TIMEOUT)] :param parameters: Additional input parameters to override original ones from reana.yaml. E.g. -p myparam1=myval1 -p myparam2=myval2. :param options: Additional operational options for the workflow execution. E.g. CACHE=off. :param submit_only: Do not wait for workflows to finish. :param check_only: Wait for previously submitted workflows. :type component: str :type workflow_engine: list :type compute_backend: list :type file: list :type timecheck: int :type timeout: int :type parameters: list :type options: list :type submit_only: bool :type check_only: bool """ if submit_only and check_only: click.secho( "[ERROR] Options --submit-only and --check-only are mutually exclusive. Choose only one." ) sys.exit(1) components = select_components(component) workflow_engines = select_workflow_engines(workflow_engine) compute_backends = select_compute_backends(compute_backend) display_message("Running the following matrix:\n" "Demos: {}\n" "Workflow engines: {}\n" "Compute backends: {}".format(",".join(components), ",".join(workflow_engines), ",".join(compute_backends))) for component in components: for workflow_engine in workflow_engines: for compute_backend in compute_backends: reana_yaml_file_path = get_example_reana_yaml_file_path( component, workflow_engine, compute_backend) if not reana_yaml_file_path: msg = "Skipping example {0} with workflow engine {1} and compute backend {2}.".format( component, workflow_engine, compute_backend) display_message(msg, component) continue workflow_name = construct_workflow_name( component, workflow_engine, compute_backend) if not check_only: # create workflow: for cmd in [ "reana-client create -f {0} -n {1}".format( reana_yaml_file_path, workflow_name, ), ]: run_command(cmd, component) # upload inputs for cmd in [ "reana-client upload -w {0}".format(workflow_name), ]: run_command(cmd, component) # run workflow input_parameters = " ".join( ["-p " + parameter for parameter in parameters]) operational_options = " ".join( ["-o " + option for option in options]) for cmd in [ "reana-client start -w {0} {1} {2}".format( workflow_name, input_parameters, operational_options), ]: run_command(cmd, component) if not submit_only: # verify whether job finished within time limits time_start = time.time() while time.time() - time_start <= timeout: time.sleep(timecheck) cmd = "reana-client status -w {0}".format( workflow_name) status = run_command(cmd, component, return_output=True) click.secho(status) if ("finished" in status or "failed" in status or "stopped" in status): break # verify logs message presence for log_message in get_expected_log_messages_for_example( component): cmd = "reana-client logs -w {0} | grep -c '{1}'".format( workflow_name, log_message) run_command(cmd, component) # verify output file presence cmd = "reana-client ls -w {0}".format(workflow_name) listing = run_command(cmd, component, return_output=True) click.secho(listing) expected_files = file or get_expected_output_filenames_for_example( component) for expected_file in expected_files: if expected_file not in listing: click.secho( "[ERROR] Expected output file {0} not found. " "Exiting.".format(expected_file)) sys.exit(1) # report that everything was OK run_command("echo OK", component)
def python_unit_tests(component): # noqa: D301 """Run Python unit tests in independent environments. For each component, create a dedicated throw-away virtual environment, install latest shared modules (reana-commons, reana-db) that are currently checked-out and run the usual component unit tests. Delete the throw-away virtual environment afterwards. \b :param components: The option ``component`` can be repeated. The value may consist of: * (1) standard component name such as 'reana-workflow-controller'; * (2) short component name such as 'r-w-controller'; * (3) special value '.' indicating component of the current working directory; * (4) special value 'CLUSTER' that will expand to cover all REANA cluster components [default]; * (5) special value 'CLIENT' that will expand to cover all REANA client components; * (6) special value 'DEMO' that will expand to include several runable REANA demo examples; * (7) special value 'ALL' that will expand to include all REANA repositories. :type component: str """ components = select_components(component) for component in components: if component == "reana-job-controller" and platform.system( ) == "Darwin": msg = ("Ignoring component {} that cannot be tested" " on a macOS platform yet.".format(component)) display_message(msg, component) elif is_component_python_package(component): cmd_activate_venv = "source ~/.virtualenvs/_{}/bin/activate".format( component) if does_component_need_db(component): run_command( f"docker stop postgres__{component}\n" f"docker run --rm --name postgres__{component} -p 5432:5432 " "-e POSTGRES_PASSWORD=mysecretpassword -d postgres:9.6.2") for cmd in [ "virtualenv ~/.virtualenvs/_{}".format(component), "{} && which python".format(cmd_activate_venv), "{} && cd ../pytest-reana && " " pip install . --upgrade".format(cmd_activate_venv), "{} && cd ../reana-commons && " " pip install . --upgrade".format(cmd_activate_venv), "{} && cd ../reana-db && " " pip install . --upgrade".format(cmd_activate_venv), "git clean -d -ff -x", '{} && pip install ".[tests]" --upgrade'.format( cmd_activate_venv), "{} && {} python setup.py test".format( cmd_activate_venv, "REANA_SQLALCHEMY_DATABASE_URI=postgresql+psycopg2://postgres:mysecretpassword@localhost/postgres" if does_component_need_db(component) else "", ), "rm -rf ~/.virtualenvs/_{}".format(component), ]: run_command(cmd, component) if does_component_need_db(component): run_command(f"docker stop postgres__{component}") else: msg = ("Ignoring this component that does not contain" " a Python setup.py file.") display_message(msg, component)
def git_log( range, component, exclude_components, number, graph, oneline, stat, patch, all, paginate, ): # noqa: D301 """Show commit logs in given component repositories. \b :param range: The commit log range to operate on. :param component: The option ``component`` can be repeated. The value may consist of: * (1) standard component name such as 'reana-workflow-controller'; * (2) short component name such as 'r-w-controller'; * (3) special value '.' indicating component of the current working directory; * (4) special value 'CLUSTER' that will expand to cover all REANA cluster components [default]; * (5) special value 'CLIENT' that will expand to cover all REANA client components; * (6) special value 'DEMO' that will expand to include several runable REANA demo examples; * (7) special value 'ALL' that will expand to include all REANA repositories. :param exclude_components: List of components to exclude from command. :param number: The number of commits to output. [10] :param graph: Show log graph? :param oneline: Show one-line format? :param patch: Show diff patch? :param all: Show all references? :param paginate: Paginate output? :type range: str :type component: str :type exclude_components: str :type number: int :type graph: bool :type oneline: bool :type stat: bool :type patch: bool :type all: bool :type paginate: bool """ if exclude_components: exclude_components = exclude_components.split(",") components = select_components(component, exclude_components) for component in components: if paginate: cmd = "git --paginate log" else: cmd = "git --no-pager log" if number: cmd += " -n {}".format(number) if graph: cmd += (" --graph --decorate" ' --pretty=format:"%C(blue)%d%Creset' " %C(yellow)%h%Creset %s, %C(bold green)%an%Creset," ' %C(green)%cd%Creset" --date=relative') if oneline or graph or all: cmd += " --oneline" if stat: cmd += " --stat" if patch: cmd += " --patch" if all: cmd += " --all" if range: cmd += " {}".format(range) msg = cmd[0:cmd.find("--pretty")] + "..." display_message(msg, component) run_command(cmd, component, display=False)