def workflow_validate(ctx, file, environments, pull): # noqa: D301
"""Validate workflow specification file.
The `validate` command allows to check syntax and validate the reana.yaml
workflow specification file.
Examples: \n
\t $ reana-client validate -f reana.yaml
"""
logging.debug("command: {}".format(ctx.command_path.replace(" ", ".")))
for p in ctx.params:
logging.debug("{param}: {value}".format(param=p, value=ctx.params[p]))
try:
load_reana_spec(
click.format_filename(file),
skip_validate_environments=not environments,
pull_environment_image=pull,
)
except ValidationError as e:
logging.debug(traceback.format_exc())
logging.debug(str(e))
display_message(
"{0} is not a valid REANA specification:\n{1}".format(
click.format_filename(file), e.message),
msg_type="error",
)
except Exception as e:
logging.debug(traceback.format_exc())
logging.debug(str(e))
display_message(
"Something went wrong when trying to validate {}".format(file),
msg_type="error",
)
def _validate_reana_yaml(reana_yaml):
"""Validate REANA specification file according to jsonschema.
:param reana_yaml: Dictionary which represents REANA specifications file.
:raises ValidationError: Given REANA spec file does not validate against
REANA specification schema.
"""
try:
with open(reana_yaml_schema_file_path, "r") as f:
reana_yaml_schema = json.loads(f.read())
validate(reana_yaml, reana_yaml_schema)
display_message(
"Valid REANA specification file.",
msg_type="success",
indented=True,
)
except IOError as e:
logging.info(
"Something went wrong when reading REANA validation schema from "
"{filepath} : \n"
"{error}".format(filepath=reana_yaml_schema_file_path,
error=e.strerror))
raise e
except ValidationError as e:
logging.info(
"Invalid REANA specification: {error}".format(error=e.message))
raise e
def parse_secret_from_literal(literal):
"""Parse a literal string, into a secret dict.
:param literal: String containg a key and a value. (e.g. 'KEY=VALUE')
:returns secret: Dictionary in the format suitable for sending
via http request.
"""
try:
key, value = literal.split("=", 1)
secret = {
key: {
"value":
base64.b64encode(value.encode("utf-8")).decode("utf-8"),
"type": "env",
}
}
return secret
except ValueError as e:
logging.debug(traceback.format_exc())
logging.debug(str(e))
display_message(
'Option "{0}" is invalid: \n'
'For literal strings use "SECRET_NAME=VALUE" format'.format(
literal),
msg_type="error",
)
def display_messages(self):
"""Display messages in console."""
for msg in self.messages:
display_message(
msg["message"],
msg_type=msg["type"],
indented=True,
)
def version(ctx): # noqa: D301
"""Show version.
The ``version`` command shows REANA client version.
Examples: \n
\t $ reana-client version
"""
display_message(__version__)
def display_error_message(self, compute_backend: str,
step_name: str) -> None:
"""Display validation error message and exit."""
message = (
f'Compute backend "{compute_backend}" found in step "{step_name}" is not supported. '
f'List of supported compute backends: "{", ".join(self.supported_backends)}"'
)
display_message(message, msg_type="error", indented=True)
sys.exit(1)
def requires_environments(ctx, param, value):
"""Require passing ``--environments`` flag."""
if value and not ctx.params.get("environments"):
display_message(
"`{}` flag requires `--environments` flag.".format(param.opts[0]),
msg_type="error",
)
sys.exit(1)
return value
def validate_workflow_name_parameter(
ctx: click.core.Context, _: click.core.Option,
workflow_name: str) -> Union[str, NoReturn]:
"""Validate workflow name parameter."""
try:
return validate_workflow_name(workflow_name)
except ValueError as e:
display_message(str(e), msg_type="error")
sys.exit(1)
def validate(self):
"""Validate REANA workflow parameters."""
try:
self.validate_parameters()
except ParameterValidationError as e:
self.display_messages()
display_message(
str(e), msg_type="error", indented=True,
)
sys.exit(1)
def workflow_uuid_or_name(ctx, param, value):
"""Get UUID of workflow from configuration / cache file based on name."""
if not value:
display_message(
"Workflow name must be provided either with "
"`--workflow` option or with REANA_WORKON "
"environment variable",
msg_type="error",
)
else:
return value
def wrapper(*args, **kwargs):
from reana_client.utils import get_api_url
api_url = get_api_url()
if not api_url:
display_message(
"REANA client is not connected to any REANA cluster.",
msg_type="error",
)
sys.exit(1)
return func(*args, **kwargs)
def validate_input_parameters(live_parameters, original_parameters):
"""Return validated input parameters."""
parsed_input_parameters = dict(live_parameters)
for parameter in parsed_input_parameters.keys():
if parameter not in original_parameters:
display_message(
"Given parameter - {0}, is not in reana.yaml".format(
parameter),
msg_type="error",
)
del live_parameters[parameter]
return live_parameters
def access_token_check(
ctx: click.core.Context,
_: click.core.Option,
access_token: Optional[str],
required: bool,
) -> Union[str, NoReturn]:
"""Check if access token is present."""
if not access_token and required:
display_message(ERROR_MESSAGES["missing_access_token"],
msg_type="error")
ctx.exit(1)
else:
return access_token
def handle_parse_result(self, ctx, opts, args):
"""Overwritten click method."""
argument_present = self.name in opts
other_argument_present = self.not_required_if in opts
if not argument_present and not other_argument_present:
display_message(
"At least one of the options: `{}` or `{}` "
"is required\n".format(self.name, self.not_required_if) +
ctx.get_help(),
msg_type="error",
)
sys.exit(1)
return super(NotRequiredIf, self).handle_parse_result(ctx, opts, args)
def key_value_to_dict(ctx, param, value):
"""Convert tuple params to dictionary. e.g `(foo=bar)` to `{'foo': 'bar'}`.
:param options: A tuple with CLI operational options.
:returns: A dictionary representation of the given options.
"""
try:
return dict(op.split("=") for op in value)
except ValueError:
display_message(
'Input parameter "{0}" is not valid. '
'It must follow format "param=value".'.format(" ".join(value)),
msg_type="error",
),
sys.exit(1)
def delete_files(ctx, workflow, filenames, access_token): # noqa: D301
"""Delete files from workspace.
The ``rm`` command allow to delete files and directories from workspace.
Note that you can use glob to remove similar files.
Examples:\n
\t $ reana-client rm -w myanalysis.42 data/mydata.csv \n
\t $ reana-client rm -w myanalysis.42 'data/*root*'
""" # noqa: W605
from reana_client.api.client import delete_file
logging.debug("command: {}".format(ctx.command_path.replace(" ", ".")))
for p in ctx.params:
logging.debug("{param}: {value}".format(param=p, value=ctx.params[p]))
if workflow:
for filename in filenames:
try:
response = delete_file(workflow, filename, access_token)
freed_space = 0
for file_ in response["deleted"]:
freed_space += response["deleted"][file_]["size"]
display_message(f"File {file_} was successfully deleted.",
msg_type="success")
for file_ in response["failed"]:
display_message(
"Something went wrong while deleting {}.\n"
"{}".format(file_, response["failed"][file_]["error"]),
msg_type="error",
)
if freed_space:
display_message(f"{freed_space} bytes freed up.",
msg_type="success")
except FileDeletionError as e:
display_message(str(e), msg_type="error")
if "invoked_by_subcommand" in ctx.parent.__dict__:
sys.exit(1)
except Exception as e:
logging.debug(traceback.format_exc())
logging.debug(str(e))
display_message(
"Something went wrong while deleting {}".format(filename),
msg_type="error",
)
if "invoked_by_subcommand" in ctx.parent.__dict__:
sys.exit(1)
def workflow_create(ctx, file, name, skip_validation, access_token): # noqa: D301
"""Create a new workflow.
The ``create`` command allows to create a new workflow from reana.yaml
specifications file. The file is expected to be located in the current
working directory, or supplied via command-line -f option, see examples
below.
Examples: \n
\t $ reana-client create\n
\t $ reana-client create -w myanalysis\n
\t $ reana-client create -w myanalysis -f myreana.yaml\n
"""
from reana_client.api.client import create_workflow
from reana_client.utils import get_api_url
logging.debug("command: {}".format(ctx.command_path.replace(" ", ".")))
for p in ctx.params:
logging.debug("{param}: {value}".format(param=p, value=ctx.params[p]))
# Check that name is not an UUIDv4.
# Otherwise it would mess up `--workflow` flag usage because no distinction
# could be made between the name and actual UUID of workflow.
if is_uuid_v4(name):
display_message("Workflow name cannot be a valid UUIDv4", msg_type="error")
try:
reana_specification = load_reana_spec(
click.format_filename(file),
access_token=access_token,
skip_validation=skip_validation,
server_capabilities=True,
)
logging.info("Connecting to {0}".format(get_api_url()))
response = create_workflow(reana_specification, name, access_token)
click.echo(click.style(response["workflow_name"], fg="green"))
# check if command is called from wrapper command
if "invoked_by_subcommand" in ctx.parent.__dict__:
ctx.parent.workflow_name = response["workflow_name"]
except Exception as e:
logging.debug(traceback.format_exc())
logging.debug(str(e))
display_message(
"Cannot create workflow {}: \n{}".format(name, str(e)), msg_type="error"
)
if "invoked_by_subcommand" in ctx.parent.__dict__:
sys.exit(1)
def workflow_disk_usage(ctx, workflow, access_token, summarize, filters,
human_readable_or_raw): # noqa: D301
"""Get workspace disk usage.
The ``du`` command allows to chech the disk usage of given workspace.
Examples: \n
\t $ reana-client du -w myanalysis.42 -s \n
\t $ reana-client du -w myanalysis.42 -s --human-readable \n
\t $ reana-client du -w myanalysis.42 --filter name=data/
"""
from reana_client.api.client import get_workflow_disk_usage
logging.debug("command: {}".format(ctx.command_path.replace(" ", ".")))
for p in ctx.params:
logging.debug("{param}: {value}".format(param=p, value=ctx.params[p]))
search_filter = None
headers = ["size", "name"]
if filters:
_, search_filter = parse_filter_parameters(filters, headers)
if workflow:
try:
parameters = {"summarize": summarize, "search": search_filter}
response = get_workflow_disk_usage(workflow, parameters,
access_token)
if not response["disk_usage_info"]:
display_message("No files matching filter criteria.",
msg_type="error")
sys.exit(1)
data = []
for disk_usage_info in response["disk_usage_info"]:
if not disk_usage_info["name"].startswith(FILES_BLACKLIST):
data.append([
disk_usage_info["size"][human_readable_or_raw],
".{}".format(disk_usage_info["name"]),
])
click_table_printer(headers, [], data)
except Exception as e:
logging.debug(traceback.format_exc())
logging.debug(str(e))
display_message(
"Disk usage could not be retrieved: \n{}".format(e),
msg_type="error",
)
def validate_environment(self):
"""Validate environments in REANA CWL workflow."""
try:
import cwl_utils.parser_v1_0 as cwl_parser
from cwl_utils.docker_extract import traverse
except ImportError as e:
display_message(
"Cannot validate environment. Please install reana-client on Python 3+ to enable environment validation for CWL workflows.",
msg_type="error",
indented=True,
)
raise e
top = cwl_parser.load_document(self.workflow_file)
for image in traverse(top):
self._validate_environment_image(image)
def workflow_delete(ctx, workflow, all_runs, workspace, access_token): # noqa: D301
"""Delete a workflow.
The ``delete`` command allows to remove workflow runs from the database and
the workspace. By default, the command removes the workflow and all its
cached information and hides the workflow from the workflow list. Note that
workflow workspace will still be accessible until you use
``--include-workspace`` flag. Note also that you can remove all past runs of
a workflow by specifying ``--include-all-runs`` flag.
Example: \n
\t $ reana-client delete -w myanalysis.42 \n
\t $ reana-client delete -w myanalysis.42 --include-all-runs \n
\t $ reana-client delete -w myanalysis.42 --include-workspace
"""
from reana_client.api.client import delete_workflow
from reana_client.utils import get_api_url
logging.debug("command: {}".format(ctx.command_path.replace(" ", ".")))
for p in ctx.params:
logging.debug("{param}: {value}".format(param=p, value=ctx.params[p]))
if workflow:
try:
logging.info("Connecting to {0}".format(get_api_url()))
delete_workflow(workflow, all_runs, workspace, access_token)
if all_runs:
message = "All workflows named '{}' have been deleted.".format(
workflow.split(".")[0]
)
else:
message = get_workflow_status_change_msg(workflow, "deleted")
display_message(message, msg_type="success")
except Exception as e:
logging.debug(traceback.format_exc())
logging.debug(str(e))
display_message(
"Cannot delete workflow {} \n{}".format(workflow, str(e)),
msg_type="error",
)
def ping(ctx, access_token): # noqa: D301
"""Check connection to REANA server.
The ``ping`` command allows to test connection to REANA server.
Examples: \n
\t $ reana-client ping
"""
try:
from reana_client.api.client import ping as rs_ping
from reana_client.utils import get_api_url
logging.info("Connecting to {0}".format(get_api_url()))
response = rs_ping(access_token)
msg_color = "red" if response.get("error") else "green"
click.secho(
"REANA server: {0}\n"
"REANA server version: {1}\n"
"REANA client version: {2}\n"
"Authenticated as: {3} <{4}>\n"
"Status: {5}".format(
get_api_url(),
response.get("reana_server_version", ""),
__version__,
response.get("full_name", ""),
response.get("email"),
response.get("status"),
),
fg=msg_color,
)
logging.debug("Server response:\n{}".format(response))
except Exception as e:
logging.debug(traceback.format_exc())
logging.debug(str(e))
display_message(
"Could not connect to the selected REANA cluster "
"server at {0}:\n{1}".format(get_api_url(), e),
msg_type="error",
)
ctx.exit(1)
def _validate_server_capabilities(reana_yaml: Dict, access_token: str) -> None:
"""Validate server capabilities in REANA specification file.
:param reana_yaml: dictionary which represents REANA specification file.
:param access_token: access token of the current user.
"""
from reana_client.api.client import info
info_response = info(access_token)
display_message(
"Verifying compute backends in REANA specification file...",
msg_type="info",
)
supported_backends = info_response.get("compute_backends", {}).get("value")
validate_compute_backends(reana_yaml, supported_backends)
root_path = reana_yaml.get("workspace", {}).get("root_path")
available_workspaces = info_response.get("workspaces_available",
{}).get("value")
_validate_workspace(root_path, available_workspaces)
def workflow_close_interactive_session(workflow, access_token): # noqa: D301
"""Close an interactive session.
The ``close`` command allows to shut down any interactive sessions that you
may have running. You would typically use this command after you finished
exploring data in the Jupyter notebook and after you have transferred any
code created in your interactive session.
Examples:\n
\t $ reana-client close -w myanalysis.42
"""
from reana_client.api.client import close_interactive_session
if workflow:
try:
logging.info("Closing an interactive session on {}".format(workflow))
close_interactive_session(workflow, access_token)
display_message(
"Interactive session for workflow {}"
" was successfully closed".format(workflow),
msg_type="success",
)
except Exception as e:
logging.debug(traceback.format_exc())
logging.debug(str(e))
display_message(
"Interactive session could not be closed: \n{}".format(str(e)),
msg_type="error",
)
else:
display_message("Cannot find workflow {} ".format(workflow), msg_type="error")
def workflow_open_interactive_session(
ctx, workflow, interactive_session_type, image, access_token
): # noqa: D301
"""Open an interactive session inside the workspace.
The ``open`` command allows to open interactive session processes on top of
the workflow workspace, such as Jupyter notebooks. This is useful to
quickly inspect and analyse the produced files while the workflow is stlil
running.
Examples:\n
\t $ reana-client open -w myanalysis.42 jupyter
"""
from reana_client.api.client import open_interactive_session
if workflow:
try:
logging.info("Opening an interactive session on {}".format(workflow))
interactive_session_configuration = {
"image": image or None,
}
path = open_interactive_session(
workflow,
access_token,
interactive_session_type,
interactive_session_configuration,
)
display_message(
"Interactive session opened successfully", msg_type="success"
)
click.secho(
format_session_uri(
reana_server_url=ctx.obj.reana_server_url,
path=path,
access_token=access_token,
),
fg="green",
)
display_message(
"It could take several minutes to start the interactive session."
)
except Exception as e:
logging.debug(traceback.format_exc())
logging.debug(str(e))
display_message(
"Interactive session could not be opened: \n{}".format(str(e)),
msg_type="error",
)
else:
display_message("Cannot find workflow {}".format(workflow), msg_type="error")
def _validate_workspace(root_path: str,
available_workspaces: Optional[List[str]]) -> None:
"""Validate workspace in REANA specification file.
:param root_path: workspace root path to be validated.
:param available_workspaces: a list of the available workspaces.
:raises ValidationError: Given workspace in REANA spec file does not validate against
allowed workspaces.
"""
if root_path:
display_message(
"Verifying workspace in REANA specification file...",
msg_type="info",
)
try:
validate_workspace(root_path, available_workspaces)
display_message(
"Workflow workspace appears valid.",
msg_type="success",
indented=True,
)
except REANAValidationError as e:
display_message(e.message, msg_type="error")
sys.exit(1)
def secrets_delete(secrets, access_token): # noqa: D301
"""Delete user secrets by name.
Examples: \n
\t $ reana-client secrets-delete PASSWORD
"""
from reana_client.api.client import delete_secrets
try:
deleted_secrets = delete_secrets(secrets, access_token)
except REANASecretDoesNotExist as e:
logging.debug(str(e), exc_info=True)
display_message(
"Secrets {} do not exist. Nothing was deleted".format(
e.missing_secrets_list),
msg_type="error",
)
except Exception as e:
logging.debug(str(e), exc_info=True)
display_message(
"Something went wrong while deleting secrets",
msg_type="error",
)
else:
display_message(
"Secrets {} were successfully deleted.".format(
", ".join(deleted_secrets)),
msg_type="success",
)
def info(ctx, access_token: str, output_format: str): # noqa: D301
"""List cluster general information.
The ``info`` command lists general information about the cluster.
Lists all the available workspaces. It also returns the default workspace
defined by the admin.
Examples: \n
\t $ reana-client info
"""
try:
from reana_client.api.client import info
response = info(access_token)
if output_format == JSON:
display_message(json.dumps(response))
else:
for item in response.values():
value = item.get("value")
value = ", ".join(value) if isinstance(value, list) else value
display_message(f"{item.get('title')}: {value}")
except Exception as e:
logging.debug(traceback.format_exc())
logging.debug(str(e))
display_message("Could not list cluster info:\n{0}".format(e), msg_type="error")
ctx.exit(1)
def get_workflow_root():
"""Return the current workflow root directory."""
reana_yaml = get_reana_yaml_file_path()
workflow_root = os.getcwd()
while True:
file_list = os.listdir(workflow_root)
parent_dir = os.path.dirname(workflow_root)
if reana_yaml in file_list:
break
else:
if workflow_root == parent_dir:
display_message(
"Not a workflow directory (or any of the parent directories).\n"
"Please upload from inside the directory containing "
"the reana.yaml file of your workflow.",
msg_type="error",
)
sys.exit(1)
else:
workflow_root = parent_dir
workflow_root += "/"
return workflow_root
def validate_compute_backends(reana_yaml: Dict,
supported_backends: Optional[List[str]]) -> None:
"""Validate compute backends in REANA specification file according to workflow type.
:param reana_yaml: dictionary which represents REANA specification file.
:param supported_backends: a list of the supported compute backends.
"""
def build_validator(workflow: Dict) -> None:
workflow_type = workflow["type"]
if workflow_type == "serial":
workflow_steps = workflow["specification"]["steps"]
return SerialComputeBackendValidator(
workflow_steps=workflow_steps,
supported_backends=supported_backends)
if workflow_type == "yadage":
workflow_steps = workflow["specification"]["stages"]
return YadageComputeBackendValidator(
workflow_steps=workflow_steps,
supported_backends=supported_backends)
if workflow_type == "cwl":
workflow_steps = workflow.get("specification",
{}).get("$graph", workflow)
return CWLComputeBackendValidator(
workflow_steps=workflow_steps,
supported_backends=supported_backends)
if workflow_type == "snakemake":
workflow_steps = workflow["specification"]["steps"]
return SnakemakeComputeBackendValidator(
workflow_steps=workflow_steps,
supported_backends=supported_backends)
workflow = reana_yaml["workflow"]
validator = build_validator(workflow)
validator.validate()
display_message(
"Workflow compute backends appear to be valid.",
msg_type="success",
indented=True,
)
def secrets_list(access_token): # noqa: D301
"""List user secrets.
Examples: \n
\t $ reana-client secrets-list
"""
from reana_client.api.client import list_secrets
try:
secrets = list_secrets(access_token)
headers = ["name", "type"]
data = []
for secret_ in secrets:
data.append(list(map(str, [secret_["name"], secret_["type"]])))
click_table_printer(headers, headers, data)
except Exception as e:
logging.debug(str(e), exc_info=True)
display_message(
"Something went wrong while listing secrets",
msg_type="error",
)
