def upload_reana_workflow_files(workflow_id, workflow=None, deposit=None):
"""Upload files to a workflow."""
_args = request.get_json()
files = _args.get('files_to_upload')
rec_uuid = resolve_uuid(workflow_id)
token = get_reana_token(rec_uuid)
errors = []
successful = []
for _f in files:
file_path = deposit.files[_f['path']].obj.file.uri
try:
with open(file_path, 'rb') as content:
upload_file(workflow_id, content, _f['new_path'], token)
successful.append('{} saved as {}'.format(
_f['path'], _f['new_path']))
except (IOError, FileUploadError):
errors.append(_f['path'])
return jsonify({
'workflow_id': workflow_id,
'successful': successful,
'errors': errors
}), 200
def upload_files(files, basedir, workflow_id, access_token):
"""Upload file or directory to REANA server."""
from reana_client.api.client import upload_file
for cwl_file_object in files:
file_path = cwl_file_object.get("location")
abs_file_path = os.path.join(basedir, file_path)
if os.path.isdir(abs_file_path):
for root, dirs, files in os.walk(abs_file_path, topdown=False):
for next_path in files + dirs:
location = os.path.join(root, next_path).replace(
basedir + "/", "")
upload_files(
[{
"location": location
}],
basedir,
workflow_id,
access_token,
)
else:
with open(abs_file_path, "r") as f:
upload_file(workflow_id, f, file_path, access_token)
logging.error("File {} uploaded.".format(file_path))
def cwl_runner(ctx, quiet, outdir, basedir, processfile, jobfile,
access_token):
"""Run CWL files in a standard format <workflow.cwl> <job.json>."""
from reana_client.utils import get_api_url
from reana_client.api.client import (create_workflow, get_workflow_logs,
start_workflow, upload_file)
logging.basicConfig(format='[%(levelname)s] %(message)s',
stream=sys.stderr,
level=logging.INFO if quiet else logging.DEBUG)
try:
basedir = basedir or os.path.abspath(os.path.dirname(processfile))
if processfile:
with open(jobfile) as f:
reana_spec = {
"workflow": {
"type": "cwl"
},
"inputs": {
"parameters": {
"input": yaml.load(f, Loader=yaml.FullLoader)
}
}
}
reana_spec['workflow']['spec'] = load_workflow_spec(
reana_spec['workflow']['type'],
processfile,
)
else:
with open(jobfile) as f:
job = yaml.load(f, Loader=yaml.FullLoader)
reana_spec = {
"workflow": {
"type": "cwl"
},
"parameters": {
"input": ""
}
}
reana_spec['workflow']['spec'] = load_workflow_spec(
reana_spec['workflow']['type'], job['cwl:tool'])
del job['cwl:tool']
reana_spec['inputs']['parameters'] = {'input': job}
reana_spec['workflow']['spec'] = replace_location_in_cwl_spec(
reana_spec['workflow']['spec'])
logging.info('Connecting to {0}'.format(get_api_url()))
response = create_workflow(reana_spec, 'cwl-test', access_token)
logging.error(response)
workflow_name = response['workflow_name']
workflow_id = response['workflow_id']
logging.info('Workflow {0}/{1} has been created.'.format(
workflow_name, workflow_id))
file_dependencies_list = []
for cwlobj in [processfile, jobfile]:
file_dependencies_list.append(
get_file_dependencies_obj(cwlobj, basedir))
files_to_upload = findfiles(file_dependencies_list)
for cwl_file_object in files_to_upload:
file_path = cwl_file_object.get('location')
abs_file_path = os.path.join(basedir, file_path)
with open(abs_file_path, 'r') as f:
upload_file(workflow_id, f, file_path, access_token)
logging.error('File {} uploaded.'.format(file_path))
response = start_workflow(workflow_id, access_token,
reana_spec['inputs']['parameters'])
logging.error(response)
first_logs = ""
while True:
sleep(1)
logging.error('Polling workflow logs')
response = get_workflow_logs(workflow_id, access_token)
logs = response['logs']
if logs != first_logs:
logging.error(logs[len(first_logs):])
first_logs = logs
if "Final process status" in logs or \
"Traceback (most recent call last)" in logs:
# click.echo(response['status'])
break
try:
out = re.search("success{[\S\s]*",
logs).group().replace("success", "")
import ast
import json
json_output = json.dumps(ast.literal_eval(str(out)))
except AttributeError:
logging.error("Workflow execution failed")
sys.exit(1)
except Exception as e:
logging.error(traceback.format_exc())
sys.exit(1)
sys.stdout.write(json_output)
sys.stdout.write("\n")
sys.stdout.flush()
except HTTPServerError as e:
logging.error(traceback.print_exc())
logging.error(e)
except Exception as e:
logging.error(traceback.print_exc())