def workflow_logs(ctx, workflow, access_token, json_format): # noqa: D301
"""Get workflow logs.
The `logs` command allows to retrieve logs of running workflow. Note that
only finished steps of the workflow are returned, the logs of the currently
processed step is not returned until it is finished.
Examples: \n
\t $ reana-client logs -w myanalysis.42
"""
from reana_client.api.client import get_workflow_logs
logging.debug('command: {}'.format(ctx.command_path.replace(" ", ".")))
for p in ctx.params:
logging.debug('{param}: {value}'.format(param=p, value=ctx.params[p]))
if workflow:
try:
response = get_workflow_logs(workflow, access_token)
workflow_logs = json.loads(response['logs'])
if json_format:
click.echo(json.dumps(workflow_logs, indent=2))
sys.exit(0)
if workflow_logs.get('workflow_logs', None):
click.secho('workflow engine logs'.upper(), fg='green')
click.echo(workflow_logs['workflow_logs'])
first = True
for job_id, job_logs in workflow_logs['job_logs'].items():
if job_logs:
if first:
click.echo('\n')
click.secho('job logs'.upper(), fg='green')
first = False
click.secho('job id: {}'.format(job_id), fg='green')
for key, value in job_logs.items():
click.echo('{}: {}'.format(key.upper(), value))
if workflow_logs.get('engine_specific', None):
click.echo('\n')
click.secho('engine internal logs'.upper(), fg='green')
click.secho(workflow_logs['engine_specific'])
except Exception as e:
logging.debug(traceback.format_exc())
logging.debug(str(e))
click.echo(click.style(
'Workflow status could not be retrieved: \n{}'.format(str(e)),
fg='red'),
err=True)
click.echo(response)
except Exception as e:
logging.debug(traceback.format_exc())
logging.debug(str(e))
click.echo(click.style(
'Workflow logs could not be retrieved: \n{}'.format(str(e)),
fg='red'),
err=True)
def get_reana_workflow_logs(workflow_id, workflow=None):
"""Get the logs of a workflow."""
rec_uuid = resolve_uuid(workflow_id)
token = get_reana_token(rec_uuid)
resp = get_workflow_logs(workflow_id, token)
logs = resp.get('logs', '')
resp.update({'rec_uuid': rec_uuid})
logs_serialized = ReanaWorkflowLogsSchema().dump(resp).data
update_workflow(workflow_id, 'logs', logs_serialized)
return jsonify(logs_serialized)
def workflow_logs(ctx, workflow, access_token, json_format):
"""Get workflow logs."""
logging.debug('command: {}'.format(ctx.command_path.replace(" ", ".")))
for p in ctx.params:
logging.debug('{param}: {value}'.format(param=p, value=ctx.params[p]))
if workflow:
try:
response = get_workflow_logs(workflow, access_token)
workflow_logs = json.loads(response['logs'])
if json_format:
click.echo(json.dumps(workflow_logs, indent=2))
sys.exit(0)
if workflow_logs.get('workflow_logs', None):
click.secho('workflow engine logs'.upper(), fg='green')
click.echo(workflow_logs['workflow_logs'])
first = True
for job_id, job_logs in workflow_logs['job_logs'].items():
if job_logs:
if first:
click.echo('\n')
click.secho('job logs'.upper(), fg='green')
first = False
click.secho('job id: {}'.format(job_id), fg='green')
click.echo(job_logs)
if workflow_logs.get('engine_specific', None):
click.echo('\n')
click.secho('engine internal logs'.upper(), fg='green')
click.secho(workflow_logs['engine_specific'])
except Exception as e:
logging.debug(traceback.format_exc())
logging.debug(str(e))
click.echo(click.style(
'Workflow status could not be retrieved: \n{}'.format(str(e)),
fg='red'),
err=True)
click.echo(response)
except Exception as e:
logging.debug(traceback.format_exc())
logging.debug(str(e))
click.echo(click.style(
'Workflow logs could not be retrieved: \n{}'.format(str(e)),
fg='red'),
err=True)
def workflow_logs(ctx,
workflow,
access_token,
json_format,
steps=None): # noqa: D301
"""Get workflow logs.
The `logs` command allows to retrieve logs of running workflow. Note that
only finished steps of the workflow are returned, the logs of the currently
processed step is not returned until it is finished.
Examples: \n
\t $ reana-client logs -w myanalysis.42
\t $ reana-client logs -w myanalysis.42 -s 1st_step
"""
from reana_client.api.client import get_workflow_logs
logging.debug('command: {}'.format(ctx.command_path.replace(" ", ".")))
for p in ctx.params:
logging.debug('{param}: {value}'.format(param=p, value=ctx.params[p]))
if workflow:
try:
if steps:
steps = list(set(steps))
response = get_workflow_logs(workflow, access_token, steps)
workflow_logs = json.loads(response['logs'])
if json_format:
click.echo(json.dumps(workflow_logs, indent=2))
sys.exit(0)
else:
from reana_client.cli.utils import output_user_friendly_logs
output_user_friendly_logs(workflow_logs, steps)
except Exception as e:
logging.debug(traceback.format_exc())
logging.debug(str(e))
click.echo(click.style(
'Cannot retrieve the logs of a workflow {}: \n{}'.format(
workflow, str(e)),
fg='red'),
err=True)
def get_analysis_outputs(workflow_id=None):
"""Start outputs of an analysis workflow."""
token = current_app.config.get('REANA_ACCESS_TOKEN')
response = get_workflow_logs(workflow_id, token)
return jsonify(response)
def cwl_runner(ctx, quiet, outdir, basedir, processfile, jobfile,
access_token):
"""Run CWL files in a standard format <workflow.cwl> <job.json>."""
from reana_client.utils import get_api_url
from reana_client.api.client import (create_workflow, get_workflow_logs,
start_workflow, upload_file)
logging.basicConfig(format='[%(levelname)s] %(message)s',
stream=sys.stderr,
level=logging.INFO if quiet else logging.DEBUG)
try:
basedir = basedir or os.path.abspath(os.path.dirname(processfile))
if processfile:
with open(jobfile) as f:
reana_spec = {
"workflow": {
"type": "cwl"
},
"inputs": {
"parameters": {
"input": yaml.load(f, Loader=yaml.FullLoader)
}
}
}
reana_spec['workflow']['spec'] = load_workflow_spec(
reana_spec['workflow']['type'],
processfile,
)
else:
with open(jobfile) as f:
job = yaml.load(f, Loader=yaml.FullLoader)
reana_spec = {
"workflow": {
"type": "cwl"
},
"parameters": {
"input": ""
}
}
reana_spec['workflow']['spec'] = load_workflow_spec(
reana_spec['workflow']['type'], job['cwl:tool'])
del job['cwl:tool']
reana_spec['inputs']['parameters'] = {'input': job}
reana_spec['workflow']['spec'] = replace_location_in_cwl_spec(
reana_spec['workflow']['spec'])
logging.info('Connecting to {0}'.format(get_api_url()))
response = create_workflow(reana_spec, 'cwl-test', access_token)
logging.error(response)
workflow_name = response['workflow_name']
workflow_id = response['workflow_id']
logging.info('Workflow {0}/{1} has been created.'.format(
workflow_name, workflow_id))
file_dependencies_list = []
for cwlobj in [processfile, jobfile]:
file_dependencies_list.append(
get_file_dependencies_obj(cwlobj, basedir))
files_to_upload = findfiles(file_dependencies_list)
for cwl_file_object in files_to_upload:
file_path = cwl_file_object.get('location')
abs_file_path = os.path.join(basedir, file_path)
with open(abs_file_path, 'r') as f:
upload_file(workflow_id, f, file_path, access_token)
logging.error('File {} uploaded.'.format(file_path))
response = start_workflow(workflow_id, access_token,
reana_spec['inputs']['parameters'])
logging.error(response)
first_logs = ""
while True:
sleep(1)
logging.error('Polling workflow logs')
response = get_workflow_logs(workflow_id, access_token)
logs = response['logs']
if logs != first_logs:
logging.error(logs[len(first_logs):])
first_logs = logs
if "Final process status" in logs or \
"Traceback (most recent call last)" in logs:
# click.echo(response['status'])
break
try:
out = re.search("success{[\S\s]*",
logs).group().replace("success", "")
import ast
import json
json_output = json.dumps(ast.literal_eval(str(out)))
except AttributeError:
logging.error("Workflow execution failed")
sys.exit(1)
except Exception as e:
logging.error(traceback.format_exc())
sys.exit(1)
sys.stdout.write(json_output)
sys.stdout.write("\n")
sys.stdout.flush()
except HTTPServerError as e:
logging.error(traceback.print_exc())
logging.error(e)
except Exception as e:
logging.error(traceback.print_exc())
def workflow_logs(
ctx,
workflow,
access_token,
json_format,
steps=None,
filters=None,
page=None,
size=None,
): # noqa: D301
"""Get workflow logs.
The ``logs`` command allows to retrieve logs of running workflow. Note that
only finished steps of the workflow are returned, the logs of the currently
processed step is not returned until it is finished.
Examples: \n
\t $ reana-client logs -w myanalysis.42
\t $ reana-client logs -w myanalysis.42 -s 1st_step
"""
from reana_client.api.client import get_workflow_logs
available_filters = {
"step": "job_name",
"compute_backend": "compute_backend",
"docker_img": "docker_img",
"status": "status",
}
steps = []
chosen_filters = dict()
logging.debug("command: {}".format(ctx.command_path.replace(" ", ".")))
for p in ctx.params:
logging.debug("{param}: {value}".format(param=p, value=ctx.params[p]))
if workflow:
if filters:
try:
for f in filters:
key, value = f.split("=")
if key not in available_filters:
display_message(
"Filter '{}' is not valid.\n"
"Available filters are '{}'.".format(
key, "' '".join(sorted(available_filters.keys())),
),
msg_type="error",
)
sys.exit(1)
elif key == "step":
steps.append(value)
else:
# Case insensitive for compute backends
if (
key == "compute_backend"
and value.lower() in REANA_COMPUTE_BACKENDS
):
value = REANA_COMPUTE_BACKENDS[value.lower()]
elif key == "status" and value not in RUN_STATUSES:
display_message(
"Input status value {} is not valid. ".format(value),
msg_type="error",
),
sys.exit(1)
chosen_filters[key] = value
except Exception as e:
logging.debug(traceback.format_exc())
logging.debug(str(e))
display_message(
"Please provide complete --filter name=value pairs, "
"for example --filter status=running.\n"
"Available filters are '{}'.".format(
"' '".join(sorted(available_filters.keys()))
),
msg_type="error",
)
sys.exit(1)
try:
response = get_workflow_logs(
workflow,
access_token,
steps=None if not steps else list(set(steps)),
page=page,
size=size,
)
workflow_logs = json.loads(response["logs"])
if filters:
for key, value in chosen_filters.items():
unwanted_steps = [
k
for k, v in workflow_logs["job_logs"].items()
if v[available_filters[key]] != value
]
for job_id in unwanted_steps:
del workflow_logs["job_logs"][job_id]
if json_format:
display_message(json.dumps(workflow_logs, indent=2))
sys.exit(0)
else:
from reana_client.cli.utils import output_user_friendly_logs
output_user_friendly_logs(
workflow_logs, None if not steps else list(set(steps))
)
except Exception as e:
logging.debug(traceback.format_exc())
logging.debug(str(e))
display_message(
"Cannot retrieve the logs of a workflow {}: \n"
"{}".format(workflow, str(e)),
msg_type="error",
)
def cwl_runner(ctx, quiet, outdir, basedir, processfile, jobfile,
access_token):
"""Run CWL files in a standard format <workflow.cwl> <job.json>."""
import json
from reana_client.utils import get_api_url
from reana_client.api.client import (
create_workflow,
get_workflow_logs,
start_workflow,
upload_file,
)
logging.basicConfig(
format="[%(levelname)s] %(message)s",
stream=sys.stderr,
level=logging.INFO if quiet else logging.DEBUG,
)
try:
basedir = basedir or os.path.abspath(os.path.dirname(processfile))
reana_spec = {"workflow": {"type": "cwl"}}
job = {}
if jobfile:
with open(jobfile) as f:
job = yaml.load(f, Loader=yaml.FullLoader)
if processfile:
reana_spec["inputs"] = {"parameters": job}
reana_spec["workflow"]["specification"] = load_workflow_spec(
reana_spec["workflow"]["type"], processfile)
reana_spec["workflow"]["specification"] = replace_location_in_cwl_spec(
reana_spec["workflow"]["specification"])
logging.info("Connecting to {0}".format(get_api_url()))
reana_specification = json.loads(json.dumps(reana_spec,
sort_keys=True))
response = create_workflow(reana_specification, "cwl-test",
access_token)
logging.error(response)
workflow_name = response["workflow_name"]
workflow_id = response["workflow_id"]
logging.info("Workflow {0}/{1} has been created.".format(
workflow_name, workflow_id))
file_dependencies_list = []
for cwlobj in [processfile, jobfile]:
if not cwlobj:
continue
file_dependencies_obj = get_file_dependencies_obj(cwlobj, basedir)
file_dependencies_list.append(file_dependencies_obj)
files_to_upload = findfiles(file_dependencies_list)
upload_files(files_to_upload, basedir, workflow_id, access_token)
response = start_workflow(workflow_id, access_token,
reana_spec["inputs"]["parameters"])
logging.error(response)
first_logs = ""
while True:
sleep(1)
logging.error("Polling workflow logs")
response = get_workflow_logs(workflow_id, access_token)
logs = response["logs"]
if logs != first_logs:
logging.error(logs[len(first_logs):])
first_logs = logs
if ("Final process status" in logs
or "Traceback (most recent call last)" in logs):
# click.echo(response['status'])
break
try:
import ast
out = (re.search(r"FinalOutput[\s\S]*?FinalOutput",
logs).group().replace("FinalOutput", ""))
json_output = out.encode("utf8").decode("unicode_escape")
except AttributeError:
logging.error("Workflow execution failed")
sys.exit(1)
except Exception:
logging.error(traceback.format_exc())
sys.exit(1)
sys.stdout.write(json_output)
sys.stdout.write("\n")
sys.stdout.flush()
except HTTPServerError as e:
logging.error(traceback.print_exc())
logging.error(e)
except Exception:
logging.error(traceback.print_exc())
def get_analysis_outputs(workflow_id=None):
"""Starts outputs of an analysis workflow."""
token = current_app.config.get('REANA_ACCESS_TOKEN')
response = get_workflow_logs(workflow_id, token)
return jsonify(response)
def get_analysis_outputs(workflow_id=None):
"""Start outputs of an analysis workflow."""
uuid = ReanaJob.get_record_from_workflow_id(workflow_id)
token = get_token(uuid)
response = get_workflow_logs(workflow_id, token)
return jsonify(response)