def create_reana_workflow():
"""Create a reana workflow by json."""
_args = request.get_json()
# try fetch the deposit with the provided PID
try:
resolver = Resolver(pid_type='depid',
object_type='rec',
getter=lambda x: x)
deposit, rec_uuid = resolver.resolve(_args.get('pid'))
except PIDDoesNotExistError:
abort(
404, "You tried to create a workflow and connect"
" it with a non-existing record")
# if record exist check if the user has 'deposit-update' rights
with UpdateDepositPermission(deposit).require(403):
token = get_reana_token(rec_uuid)
name = _args.get('workflow_name')
workflow_name = generate_slug(2)
workflow_json = _args.get('workflow_json')
try:
resp = create_workflow(workflow_json, workflow_name, token)
except ValidationError as e:
return jsonify({'message': e.message}), 400
except Exception:
return jsonify({
'message':
'An exception has occured while creating '
'the workflow in REANA.'
}), 400
# create a workflow dict, which can be used to populate
# the db, but also used in the serializer
_workflow = {
'service': 'reana',
'user_id': current_user.id,
'name': name,
'workflow_name': workflow_name,
'name_run': resp['workflow_name'],
'workflow_id': resp['workflow_id'],
'rec_uuid': str(rec_uuid),
'depid': _args.get('pid'),
'status': 'created',
'workflow_json': workflow_json,
}
# TOFIX: check for integrity errors
workflow = ReanaWorkflow(**_workflow)
db.session.add(workflow)
db.session.commit()
workflow_serialized = ReanaWorkflowSchema().dump(_workflow).data
return jsonify(workflow_serialized)
def workflow_create(ctx, file, name, skip_validation,
access_token): # noqa: D301
"""Create a new workflow.
The `create` command allows to create a new workflow from reana.yaml
specifications file. The file is expected to be located in the current
working directory, or supplied via command-line -f option, see examples
below.
Examples: \n
\t $ reana-client create\n
\t $ reana-client create -w myanalysis\n
\t $ reana-client create -w myanalysis -f myreana.yaml\n
"""
logging.debug('command: {}'.format(ctx.command_path.replace(" ", ".")))
for p in ctx.params:
logging.debug('{param}: {value}'.format(param=p, value=ctx.params[p]))
# Check that name is not an UUIDv4.
# Otherwise it would mess up `--workflow` flag usage because no distinction
# could be made between the name and actual UUID of workflow.
if is_uuid_v4(name):
click.echo(click.style('Workflow name cannot be a valid UUIDv4',
fg='red'),
err=True)
if not access_token:
click.echo(click.style(ERROR_MESSAGES['missing_access_token'],
fg='red'),
err=True)
sys.exit(1)
try:
reana_specification = load_reana_spec(click.format_filename(file),
skip_validation)
logging.info('Connecting to {0}'.format(
current_rs_api_client.swagger_spec.api_url))
response = create_workflow(reana_specification, name, access_token)
click.echo(click.style(response['workflow_name'], fg='green'))
# check if command is called from wrapper command
if 'invoked_by_subcommand' in ctx.parent.__dict__:
ctx.parent.workflow_name = response['workflow_name']
except Exception as e:
logging.debug(traceback.format_exc())
logging.debug(str(e))
click.echo(click.style('Workflow could not be created: \n{}'.format(
str(e)),
fg='red'),
err=True)
if 'invoked_by_subcommand' in ctx.parent.__dict__:
sys.exit(1)
def workflow_create(ctx, file, name, skip_validation,
access_token): # noqa: D301
"""Create a new workflow.
The `create` command allows to create a new workflow from reana.yaml
specifications file. The file is expected to be located in the current
working directory, or supplied via command-line -f option, see examples
below.
Examples: \n
\t $ reana-client create\n
\t $ reana-client create -w myanalysis\n
\t $ reana-client create -w myanalysis -f myreana.yaml\n
"""
from reana_client.api.client import create_workflow
from reana_client.utils import get_api_url
logging.debug("command: {}".format(ctx.command_path.replace(" ", ".")))
for p in ctx.params:
logging.debug("{param}: {value}".format(param=p, value=ctx.params[p]))
# Check that name is not an UUIDv4.
# Otherwise it would mess up `--workflow` flag usage because no distinction
# could be made between the name and actual UUID of workflow.
if is_uuid_v4(name):
click.echo(click.style("Workflow name cannot be a valid UUIDv4",
fg="red"),
err=True)
try:
reana_specification = load_reana_spec(click.format_filename(file),
skip_validation)
logging.info("Connecting to {0}".format(get_api_url()))
response = create_workflow(reana_specification, name, access_token)
click.echo(click.style(response["workflow_name"], fg="green"))
# check if command is called from wrapper command
if "invoked_by_subcommand" in ctx.parent.__dict__:
ctx.parent.workflow_name = response["workflow_name"]
except Exception as e:
logging.debug(traceback.format_exc())
logging.debug(str(e))
click.echo(
click.style("Cannot create workflow {}: \n{}".format(name, str(e)),
fg="red"),
err=True,
)
if "invoked_by_subcommand" in ctx.parent.__dict__:
sys.exit(1)
def workflow_create(ctx, file, name, skip_validation, access_token):
"""Create a REANA compatible workflow from REANA spec file."""
logging.debug('command: {}'.format(ctx.command_path.replace(" ", ".")))
for p in ctx.params:
logging.debug('{param}: {value}'.format(param=p, value=ctx.params[p]))
# Check that name is not an UUIDv4.
# Otherwise it would mess up `--workflow` flag usage because no distinction
# could be made between the name and actual UUID of workflow.
if is_uuid_v4(name):
click.echo(
click.style('Workflow name cannot be a valid UUIDv4', fg='red'),
err=True)
if not access_token:
click.echo(
click.style(ERROR_MESSAGES['missing_access_token'],
fg='red'), err=True)
sys.exit(1)
try:
reana_specification = load_reana_spec(click.format_filename(file),
skip_validation)
logging.info('Connecting to {0}'.format(
current_rs_api_client.swagger_spec.api_url))
response = create_workflow(reana_specification,
name,
access_token)
click.echo(click.style(response['workflow_name'], fg='green'))
# check if command is called from wrapper command
if 'invoked_by_subcommand' in ctx.parent.__dict__:
ctx.parent.workflow_name = response['workflow_name']
except Exception as e:
logging.debug(traceback.format_exc())
logging.debug(str(e))
click.echo(
click.style('Workflow could not be created: \n{}'
.format(str(e)), fg='red'),
err=True)
if 'invoked_by_subcommand' in ctx.parent.__dict__:
sys.exit(1)
def cwl_runner(ctx, quiet, outdir, basedir, processfile, jobfile,
access_token):
"""Run CWL files in a standard format <workflow.cwl> <job.json>."""
from reana_client.utils import get_api_url
from reana_client.api.client import (create_workflow, get_workflow_logs,
start_workflow, upload_file)
logging.basicConfig(format='[%(levelname)s] %(message)s',
stream=sys.stderr,
level=logging.INFO if quiet else logging.DEBUG)
try:
basedir = basedir or os.path.abspath(os.path.dirname(processfile))
if processfile:
with open(jobfile) as f:
reana_spec = {
"workflow": {
"type": "cwl"
},
"inputs": {
"parameters": {
"input": yaml.load(f, Loader=yaml.FullLoader)
}
}
}
reana_spec['workflow']['spec'] = load_workflow_spec(
reana_spec['workflow']['type'],
processfile,
)
else:
with open(jobfile) as f:
job = yaml.load(f, Loader=yaml.FullLoader)
reana_spec = {
"workflow": {
"type": "cwl"
},
"parameters": {
"input": ""
}
}
reana_spec['workflow']['spec'] = load_workflow_spec(
reana_spec['workflow']['type'], job['cwl:tool'])
del job['cwl:tool']
reana_spec['inputs']['parameters'] = {'input': job}
reana_spec['workflow']['spec'] = replace_location_in_cwl_spec(
reana_spec['workflow']['spec'])
logging.info('Connecting to {0}'.format(get_api_url()))
response = create_workflow(reana_spec, 'cwl-test', access_token)
logging.error(response)
workflow_name = response['workflow_name']
workflow_id = response['workflow_id']
logging.info('Workflow {0}/{1} has been created.'.format(
workflow_name, workflow_id))
file_dependencies_list = []
for cwlobj in [processfile, jobfile]:
file_dependencies_list.append(
get_file_dependencies_obj(cwlobj, basedir))
files_to_upload = findfiles(file_dependencies_list)
for cwl_file_object in files_to_upload:
file_path = cwl_file_object.get('location')
abs_file_path = os.path.join(basedir, file_path)
with open(abs_file_path, 'r') as f:
upload_file(workflow_id, f, file_path, access_token)
logging.error('File {} uploaded.'.format(file_path))
response = start_workflow(workflow_id, access_token,
reana_spec['inputs']['parameters'])
logging.error(response)
first_logs = ""
while True:
sleep(1)
logging.error('Polling workflow logs')
response = get_workflow_logs(workflow_id, access_token)
logs = response['logs']
if logs != first_logs:
logging.error(logs[len(first_logs):])
first_logs = logs
if "Final process status" in logs or \
"Traceback (most recent call last)" in logs:
# click.echo(response['status'])
break
try:
out = re.search("success{[\S\s]*",
logs).group().replace("success", "")
import ast
import json
json_output = json.dumps(ast.literal_eval(str(out)))
except AttributeError:
logging.error("Workflow execution failed")
sys.exit(1)
except Exception as e:
logging.error(traceback.format_exc())
sys.exit(1)
sys.stdout.write(json_output)
sys.stdout.write("\n")
sys.stdout.flush()
except HTTPServerError as e:
logging.error(traceback.print_exc())
logging.error(e)
except Exception as e:
logging.error(traceback.print_exc())
def _create_workflow(workflow: str, file: str) -> None:
reana_specification = _load_reana_specification(file)
create_workflow(reana_specification, workflow, REANA_ACCESS_TOKEN)
def cwl_runner(ctx, quiet, outdir, basedir, processfile, jobfile,
access_token):
"""Run CWL files in a standard format <workflow.cwl> <job.json>."""
import json
from reana_client.utils import get_api_url
from reana_client.api.client import (
create_workflow,
get_workflow_logs,
start_workflow,
upload_file,
)
logging.basicConfig(
format="[%(levelname)s] %(message)s",
stream=sys.stderr,
level=logging.INFO if quiet else logging.DEBUG,
)
try:
basedir = basedir or os.path.abspath(os.path.dirname(processfile))
reana_spec = {"workflow": {"type": "cwl"}}
job = {}
if jobfile:
with open(jobfile) as f:
job = yaml.load(f, Loader=yaml.FullLoader)
if processfile:
reana_spec["inputs"] = {"parameters": job}
reana_spec["workflow"]["specification"] = load_workflow_spec(
reana_spec["workflow"]["type"], processfile)
reana_spec["workflow"]["specification"] = replace_location_in_cwl_spec(
reana_spec["workflow"]["specification"])
logging.info("Connecting to {0}".format(get_api_url()))
reana_specification = json.loads(json.dumps(reana_spec,
sort_keys=True))
response = create_workflow(reana_specification, "cwl-test",
access_token)
logging.error(response)
workflow_name = response["workflow_name"]
workflow_id = response["workflow_id"]
logging.info("Workflow {0}/{1} has been created.".format(
workflow_name, workflow_id))
file_dependencies_list = []
for cwlobj in [processfile, jobfile]:
if not cwlobj:
continue
file_dependencies_obj = get_file_dependencies_obj(cwlobj, basedir)
file_dependencies_list.append(file_dependencies_obj)
files_to_upload = findfiles(file_dependencies_list)
upload_files(files_to_upload, basedir, workflow_id, access_token)
response = start_workflow(workflow_id, access_token,
reana_spec["inputs"]["parameters"])
logging.error(response)
first_logs = ""
while True:
sleep(1)
logging.error("Polling workflow logs")
response = get_workflow_logs(workflow_id, access_token)
logs = response["logs"]
if logs != first_logs:
logging.error(logs[len(first_logs):])
first_logs = logs
if ("Final process status" in logs
or "Traceback (most recent call last)" in logs):
# click.echo(response['status'])
break
try:
import ast
out = (re.search(r"FinalOutput[\s\S]*?FinalOutput",
logs).group().replace("FinalOutput", ""))
json_output = out.encode("utf8").decode("unicode_escape")
except AttributeError:
logging.error("Workflow execution failed")
sys.exit(1)
except Exception:
logging.error(traceback.format_exc())
sys.exit(1)
sys.stdout.write(json_output)
sys.stdout.write("\n")
sys.stdout.flush()
except HTTPServerError as e:
logging.error(traceback.print_exc())
logging.error(e)
except Exception:
logging.error(traceback.print_exc())
