def create_reana_workflow(): """Create a reana workflow by json.""" _args = request.get_json() # try fetch the deposit with the provided PID try: resolver = Resolver(pid_type='depid', object_type='rec', getter=lambda x: x) deposit, rec_uuid = resolver.resolve(_args.get('pid')) except PIDDoesNotExistError: abort( 404, "You tried to create a workflow and connect" " it with a non-existing record") # if record exist check if the user has 'deposit-update' rights with UpdateDepositPermission(deposit).require(403): token = get_reana_token(rec_uuid) name = _args.get('workflow_name') workflow_name = generate_slug(2) workflow_json = _args.get('workflow_json') try: resp = create_workflow(workflow_json, workflow_name, token) except ValidationError as e: return jsonify({'message': e.message}), 400 except Exception: return jsonify({ 'message': 'An exception has occured while creating ' 'the workflow in REANA.' }), 400 # create a workflow dict, which can be used to populate # the db, but also used in the serializer _workflow = { 'service': 'reana', 'user_id': current_user.id, 'name': name, 'workflow_name': workflow_name, 'name_run': resp['workflow_name'], 'workflow_id': resp['workflow_id'], 'rec_uuid': str(rec_uuid), 'depid': _args.get('pid'), 'status': 'created', 'workflow_json': workflow_json, } # TOFIX: check for integrity errors workflow = ReanaWorkflow(**_workflow) db.session.add(workflow) db.session.commit() workflow_serialized = ReanaWorkflowSchema().dump(_workflow).data return jsonify(workflow_serialized)
def workflow_create(ctx, file, name, skip_validation, access_token): # noqa: D301 """Create a new workflow. The `create` command allows to create a new workflow from reana.yaml specifications file. The file is expected to be located in the current working directory, or supplied via command-line -f option, see examples below. Examples: \n \t $ reana-client create\n \t $ reana-client create -w myanalysis\n \t $ reana-client create -w myanalysis -f myreana.yaml\n """ logging.debug('command: {}'.format(ctx.command_path.replace(" ", "."))) for p in ctx.params: logging.debug('{param}: {value}'.format(param=p, value=ctx.params[p])) # Check that name is not an UUIDv4. # Otherwise it would mess up `--workflow` flag usage because no distinction # could be made between the name and actual UUID of workflow. if is_uuid_v4(name): click.echo(click.style('Workflow name cannot be a valid UUIDv4', fg='red'), err=True) if not access_token: click.echo(click.style(ERROR_MESSAGES['missing_access_token'], fg='red'), err=True) sys.exit(1) try: reana_specification = load_reana_spec(click.format_filename(file), skip_validation) logging.info('Connecting to {0}'.format( current_rs_api_client.swagger_spec.api_url)) response = create_workflow(reana_specification, name, access_token) click.echo(click.style(response['workflow_name'], fg='green')) # check if command is called from wrapper command if 'invoked_by_subcommand' in ctx.parent.__dict__: ctx.parent.workflow_name = response['workflow_name'] except Exception as e: logging.debug(traceback.format_exc()) logging.debug(str(e)) click.echo(click.style('Workflow could not be created: \n{}'.format( str(e)), fg='red'), err=True) if 'invoked_by_subcommand' in ctx.parent.__dict__: sys.exit(1)
def workflow_create(ctx, file, name, skip_validation, access_token): # noqa: D301 """Create a new workflow. The `create` command allows to create a new workflow from reana.yaml specifications file. The file is expected to be located in the current working directory, or supplied via command-line -f option, see examples below. Examples: \n \t $ reana-client create\n \t $ reana-client create -w myanalysis\n \t $ reana-client create -w myanalysis -f myreana.yaml\n """ from reana_client.api.client import create_workflow from reana_client.utils import get_api_url logging.debug("command: {}".format(ctx.command_path.replace(" ", "."))) for p in ctx.params: logging.debug("{param}: {value}".format(param=p, value=ctx.params[p])) # Check that name is not an UUIDv4. # Otherwise it would mess up `--workflow` flag usage because no distinction # could be made between the name and actual UUID of workflow. if is_uuid_v4(name): click.echo(click.style("Workflow name cannot be a valid UUIDv4", fg="red"), err=True) try: reana_specification = load_reana_spec(click.format_filename(file), skip_validation) logging.info("Connecting to {0}".format(get_api_url())) response = create_workflow(reana_specification, name, access_token) click.echo(click.style(response["workflow_name"], fg="green")) # check if command is called from wrapper command if "invoked_by_subcommand" in ctx.parent.__dict__: ctx.parent.workflow_name = response["workflow_name"] except Exception as e: logging.debug(traceback.format_exc()) logging.debug(str(e)) click.echo( click.style("Cannot create workflow {}: \n{}".format(name, str(e)), fg="red"), err=True, ) if "invoked_by_subcommand" in ctx.parent.__dict__: sys.exit(1)
def workflow_create(ctx, file, name, skip_validation, access_token): """Create a REANA compatible workflow from REANA spec file.""" logging.debug('command: {}'.format(ctx.command_path.replace(" ", "."))) for p in ctx.params: logging.debug('{param}: {value}'.format(param=p, value=ctx.params[p])) # Check that name is not an UUIDv4. # Otherwise it would mess up `--workflow` flag usage because no distinction # could be made between the name and actual UUID of workflow. if is_uuid_v4(name): click.echo( click.style('Workflow name cannot be a valid UUIDv4', fg='red'), err=True) if not access_token: click.echo( click.style(ERROR_MESSAGES['missing_access_token'], fg='red'), err=True) sys.exit(1) try: reana_specification = load_reana_spec(click.format_filename(file), skip_validation) logging.info('Connecting to {0}'.format( current_rs_api_client.swagger_spec.api_url)) response = create_workflow(reana_specification, name, access_token) click.echo(click.style(response['workflow_name'], fg='green')) # check if command is called from wrapper command if 'invoked_by_subcommand' in ctx.parent.__dict__: ctx.parent.workflow_name = response['workflow_name'] except Exception as e: logging.debug(traceback.format_exc()) logging.debug(str(e)) click.echo( click.style('Workflow could not be created: \n{}' .format(str(e)), fg='red'), err=True) if 'invoked_by_subcommand' in ctx.parent.__dict__: sys.exit(1)
def cwl_runner(ctx, quiet, outdir, basedir, processfile, jobfile, access_token): """Run CWL files in a standard format <workflow.cwl> <job.json>.""" from reana_client.utils import get_api_url from reana_client.api.client import (create_workflow, get_workflow_logs, start_workflow, upload_file) logging.basicConfig(format='[%(levelname)s] %(message)s', stream=sys.stderr, level=logging.INFO if quiet else logging.DEBUG) try: basedir = basedir or os.path.abspath(os.path.dirname(processfile)) if processfile: with open(jobfile) as f: reana_spec = { "workflow": { "type": "cwl" }, "inputs": { "parameters": { "input": yaml.load(f, Loader=yaml.FullLoader) } } } reana_spec['workflow']['spec'] = load_workflow_spec( reana_spec['workflow']['type'], processfile, ) else: with open(jobfile) as f: job = yaml.load(f, Loader=yaml.FullLoader) reana_spec = { "workflow": { "type": "cwl" }, "parameters": { "input": "" } } reana_spec['workflow']['spec'] = load_workflow_spec( reana_spec['workflow']['type'], job['cwl:tool']) del job['cwl:tool'] reana_spec['inputs']['parameters'] = {'input': job} reana_spec['workflow']['spec'] = replace_location_in_cwl_spec( reana_spec['workflow']['spec']) logging.info('Connecting to {0}'.format(get_api_url())) response = create_workflow(reana_spec, 'cwl-test', access_token) logging.error(response) workflow_name = response['workflow_name'] workflow_id = response['workflow_id'] logging.info('Workflow {0}/{1} has been created.'.format( workflow_name, workflow_id)) file_dependencies_list = [] for cwlobj in [processfile, jobfile]: file_dependencies_list.append( get_file_dependencies_obj(cwlobj, basedir)) files_to_upload = findfiles(file_dependencies_list) for cwl_file_object in files_to_upload: file_path = cwl_file_object.get('location') abs_file_path = os.path.join(basedir, file_path) with open(abs_file_path, 'r') as f: upload_file(workflow_id, f, file_path, access_token) logging.error('File {} uploaded.'.format(file_path)) response = start_workflow(workflow_id, access_token, reana_spec['inputs']['parameters']) logging.error(response) first_logs = "" while True: sleep(1) logging.error('Polling workflow logs') response = get_workflow_logs(workflow_id, access_token) logs = response['logs'] if logs != first_logs: logging.error(logs[len(first_logs):]) first_logs = logs if "Final process status" in logs or \ "Traceback (most recent call last)" in logs: # click.echo(response['status']) break try: out = re.search("success{[\S\s]*", logs).group().replace("success", "") import ast import json json_output = json.dumps(ast.literal_eval(str(out))) except AttributeError: logging.error("Workflow execution failed") sys.exit(1) except Exception as e: logging.error(traceback.format_exc()) sys.exit(1) sys.stdout.write(json_output) sys.stdout.write("\n") sys.stdout.flush() except HTTPServerError as e: logging.error(traceback.print_exc()) logging.error(e) except Exception as e: logging.error(traceback.print_exc())
def _create_workflow(workflow: str, file: str) -> None: reana_specification = _load_reana_specification(file) create_workflow(reana_specification, workflow, REANA_ACCESS_TOKEN)
def cwl_runner(ctx, quiet, outdir, basedir, processfile, jobfile, access_token): """Run CWL files in a standard format <workflow.cwl> <job.json>.""" import json from reana_client.utils import get_api_url from reana_client.api.client import ( create_workflow, get_workflow_logs, start_workflow, upload_file, ) logging.basicConfig( format="[%(levelname)s] %(message)s", stream=sys.stderr, level=logging.INFO if quiet else logging.DEBUG, ) try: basedir = basedir or os.path.abspath(os.path.dirname(processfile)) reana_spec = {"workflow": {"type": "cwl"}} job = {} if jobfile: with open(jobfile) as f: job = yaml.load(f, Loader=yaml.FullLoader) if processfile: reana_spec["inputs"] = {"parameters": job} reana_spec["workflow"]["specification"] = load_workflow_spec( reana_spec["workflow"]["type"], processfile) reana_spec["workflow"]["specification"] = replace_location_in_cwl_spec( reana_spec["workflow"]["specification"]) logging.info("Connecting to {0}".format(get_api_url())) reana_specification = json.loads(json.dumps(reana_spec, sort_keys=True)) response = create_workflow(reana_specification, "cwl-test", access_token) logging.error(response) workflow_name = response["workflow_name"] workflow_id = response["workflow_id"] logging.info("Workflow {0}/{1} has been created.".format( workflow_name, workflow_id)) file_dependencies_list = [] for cwlobj in [processfile, jobfile]: if not cwlobj: continue file_dependencies_obj = get_file_dependencies_obj(cwlobj, basedir) file_dependencies_list.append(file_dependencies_obj) files_to_upload = findfiles(file_dependencies_list) upload_files(files_to_upload, basedir, workflow_id, access_token) response = start_workflow(workflow_id, access_token, reana_spec["inputs"]["parameters"]) logging.error(response) first_logs = "" while True: sleep(1) logging.error("Polling workflow logs") response = get_workflow_logs(workflow_id, access_token) logs = response["logs"] if logs != first_logs: logging.error(logs[len(first_logs):]) first_logs = logs if ("Final process status" in logs or "Traceback (most recent call last)" in logs): # click.echo(response['status']) break try: import ast out = (re.search(r"FinalOutput[\s\S]*?FinalOutput", logs).group().replace("FinalOutput", "")) json_output = out.encode("utf8").decode("unicode_escape") except AttributeError: logging.error("Workflow execution failed") sys.exit(1) except Exception: logging.error(traceback.format_exc()) sys.exit(1) sys.stdout.write(json_output) sys.stdout.write("\n") sys.stdout.flush() except HTTPServerError as e: logging.error(traceback.print_exc()) logging.error(e) except Exception: logging.error(traceback.print_exc())