def cwl_runner(ctx, quiet, outdir, basedir, processfile, jobfile,
access_token):
"""Run CWL files in a standard format <workflow.cwl> <job.json>."""
import json
from reana_client.utils import get_api_url
from reana_client.api.client import (
create_workflow,
get_workflow_logs,
start_workflow,
upload_file,
)
logging.basicConfig(
format="[%(levelname)s] %(message)s",
stream=sys.stderr,
level=logging.INFO if quiet else logging.DEBUG,
)
try:
basedir = basedir or os.path.abspath(os.path.dirname(processfile))
reana_spec = {"workflow": {"type": "cwl"}}
job = {}
if jobfile:
with open(jobfile) as f:
job = yaml.load(f, Loader=yaml.FullLoader)
if processfile:
reana_spec["inputs"] = {"parameters": job}
reana_spec["workflow"]["specification"] = load_workflow_spec(
reana_spec["workflow"]["type"], processfile)
reana_spec["workflow"]["specification"] = replace_location_in_cwl_spec(
reana_spec["workflow"]["specification"])
logging.info("Connecting to {0}".format(get_api_url()))
reana_specification = json.loads(json.dumps(reana_spec,
sort_keys=True))
response = create_workflow(reana_specification, "cwl-test",
access_token)
logging.error(response)
workflow_name = response["workflow_name"]
workflow_id = response["workflow_id"]
logging.info("Workflow {0}/{1} has been created.".format(
workflow_name, workflow_id))
file_dependencies_list = []
for cwlobj in [processfile, jobfile]:
if not cwlobj:
continue
file_dependencies_obj = get_file_dependencies_obj(cwlobj, basedir)
file_dependencies_list.append(file_dependencies_obj)
files_to_upload = findfiles(file_dependencies_list)
upload_files(files_to_upload, basedir, workflow_id, access_token)
response = start_workflow(workflow_id, access_token,
reana_spec["inputs"]["parameters"])
logging.error(response)
first_logs = ""
while True:
sleep(1)
logging.error("Polling workflow logs")
response = get_workflow_logs(workflow_id, access_token)
logs = response["logs"]
if logs != first_logs:
logging.error(logs[len(first_logs):])
first_logs = logs
if ("Final process status" in logs
or "Traceback (most recent call last)" in logs):
# click.echo(response['status'])
break
try:
import ast
out = (re.search(r"FinalOutput[\s\S]*?FinalOutput",
logs).group().replace("FinalOutput", ""))
json_output = out.encode("utf8").decode("unicode_escape")
except AttributeError:
logging.error("Workflow execution failed")
sys.exit(1)
except Exception:
logging.error(traceback.format_exc())
sys.exit(1)
sys.stdout.write(json_output)
sys.stdout.write("\n")
sys.stdout.flush()
except HTTPServerError as e:
logging.error(traceback.print_exc())
logging.error(e)
except Exception:
logging.error(traceback.print_exc())
def cwl_runner(ctx, quiet, outdir, basedir, processfile, jobfile,
access_token):
"""Run CWL files in a standard format <workflow.cwl> <job.json>."""
from reana_client.utils import get_api_url
from reana_client.api.client import (create_workflow, get_workflow_logs,
start_workflow, upload_file)
logging.basicConfig(format='[%(levelname)s] %(message)s',
stream=sys.stderr,
level=logging.INFO if quiet else logging.DEBUG)
try:
basedir = basedir or os.path.abspath(os.path.dirname(processfile))
if processfile:
with open(jobfile) as f:
reana_spec = {
"workflow": {
"type": "cwl"
},
"inputs": {
"parameters": {
"input": yaml.load(f, Loader=yaml.FullLoader)
}
}
}
reana_spec['workflow']['spec'] = load_workflow_spec(
reana_spec['workflow']['type'],
processfile,
)
else:
with open(jobfile) as f:
job = yaml.load(f, Loader=yaml.FullLoader)
reana_spec = {
"workflow": {
"type": "cwl"
},
"parameters": {
"input": ""
}
}
reana_spec['workflow']['spec'] = load_workflow_spec(
reana_spec['workflow']['type'], job['cwl:tool'])
del job['cwl:tool']
reana_spec['inputs']['parameters'] = {'input': job}
reana_spec['workflow']['spec'] = replace_location_in_cwl_spec(
reana_spec['workflow']['spec'])
logging.info('Connecting to {0}'.format(get_api_url()))
response = create_workflow(reana_spec, 'cwl-test', access_token)
logging.error(response)
workflow_name = response['workflow_name']
workflow_id = response['workflow_id']
logging.info('Workflow {0}/{1} has been created.'.format(
workflow_name, workflow_id))
file_dependencies_list = []
for cwlobj in [processfile, jobfile]:
file_dependencies_list.append(
get_file_dependencies_obj(cwlobj, basedir))
files_to_upload = findfiles(file_dependencies_list)
for cwl_file_object in files_to_upload:
file_path = cwl_file_object.get('location')
abs_file_path = os.path.join(basedir, file_path)
with open(abs_file_path, 'r') as f:
upload_file(workflow_id, f, file_path, access_token)
logging.error('File {} uploaded.'.format(file_path))
response = start_workflow(workflow_id, access_token,
reana_spec['inputs']['parameters'])
logging.error(response)
first_logs = ""
while True:
sleep(1)
logging.error('Polling workflow logs')
response = get_workflow_logs(workflow_id, access_token)
logs = response['logs']
if logs != first_logs:
logging.error(logs[len(first_logs):])
first_logs = logs
if "Final process status" in logs or \
"Traceback (most recent call last)" in logs:
# click.echo(response['status'])
break
try:
out = re.search("success{[\S\s]*",
logs).group().replace("success", "")
import ast
import json
json_output = json.dumps(ast.literal_eval(str(out)))
except AttributeError:
logging.error("Workflow execution failed")
sys.exit(1)
except Exception as e:
logging.error(traceback.format_exc())
sys.exit(1)
sys.stdout.write(json_output)
sys.stdout.write("\n")
sys.stdout.flush()
except HTTPServerError as e:
logging.error(traceback.print_exc())
logging.error(e)
except Exception as e:
logging.error(traceback.print_exc())
def cwl_runner(client, quiet, outdir, processfile, jobfile):
"""Run CWL files in a standard format <workflow.cwl> <job.json>."""
server_url = os.environ.get('REANA_SERVER_URL', 'http://reana.cern.ch')
logging.info(
'REANA Server URL ($REANA_SERVER_URL) is: {}'.format(server_url))
client = Client(server_url)
logging.basicConfig(format='[%(levelname)s] %(message)s',
stream=sys.stderr,
level=logging.INFO if quiet else logging.DEBUG)
try:
if processfile:
with open(jobfile) as f:
reana_spec = {
"workflow": {
"type": "cwl"
},
"inputs": {
"parameters": {
"input": yaml.load(f)
}
}
}
reana_spec['workflow']['spec'] = load_workflow_spec(
reana_spec['workflow']['type'],
processfile,
)
else:
with open(jobfile) as f:
job = yaml.load(f)
reana_spec = {
"workflow": {
"type": "cwl"
},
"parameters": {
"input": ""
}
}
reana_spec['workflow']['spec'] = load_workflow_spec(
reana_spec['workflow']['type'], job['cwl:tool'])
del job['cwl:tool']
reana_spec['inputs']['parameters'] = {'input': job}
reana_spec['workflow']['spec'] = replace_location_in_cwl_spec(
reana_spec['workflow']['spec'])
logging.info('Connecting to {0}'.format(client.server_url))
response = client.create_workflow(default_user, default_organization,
reana_spec, 'cwl-runner')
logging.error(response)
workflow_id = response['workflow_id']
upload_files_from_cwl_spec(client, reana_spec['workflow']['spec'],
processfile, workflow_id)
if reana_spec['inputs']['parameters']['input']:
upload_files(client, reana_spec['inputs']['parameters']['input'],
jobfile, workflow_id)
response = client.start_analysis(default_user, default_organization,
workflow_id)
logging.error(response)
first_logs = ""
while True:
sleep(1)
logging.error('Polling workflow logs')
response = client.get_workflow_logs(default_user,
default_organization,
workflow_id)
logs = response['logs']
if logs != first_logs:
logging.error(logs[len(first_logs):])
first_logs = logs
if "Final process status" in logs or \
"Traceback (most recent call last)" in logs:
# click.echo(response['status'])
break
try:
out = re.search("success{[\S\s]*",
logs).group().replace("success", "")
except AttributeError:
logging.error("Workflow execution failed")
sys.exit(1)
sys.stdout.write(out)
sys.stdout.write("\n")
sys.stdout.flush()
except HTTPServerError as e:
logging.error(traceback.print_exc())
logging.error(e)
except Exception as e:
logging.error(traceback.print_exc())
