def handler(event, context):
'''
this is to check if the task run is done:
http://docs.sevenbridges.com/reference#get-task-execution-details
'''
# used to automatically determine the environment
tibanna_settings = event.get('_tibanna', {})
tibanna = Tibanna(**tibanna_settings)
sbg = sbg_utils.create_sbg_workflow(token=tibanna.sbg_keys, **event.get('workflow'))
ff_meta = ff_utils.create_ffmeta(sbg, **event.get('ff_meta'))
# check status of workflow, error if not done
status = sbg.check_task()
LOG.info("status of sbg task is %s" % status)
if not status['status'] in ['DONE', 'COMPLETED', 'FAILED']:
data = {'workflow': sbg.as_dict(),
'status': status}
raise sbg_utils.SBGStillRunningException('Task not finished => %s' % data)
if status['status'] == 'FAILED':
ff_meta.run_status = 'error'
ff_meta.description = 'SBG task %s reported FAILED status' % sbg.task_id
ff_meta.post(key=tibanna.ff_keys)
# TODO: handle only specific errors so this can terminate the rest of the workflow
return {'workflow': sbg.as_dict(),
'run_response': status,
'ff_meta': ff_meta.as_dict(),
'pf_meta': event.get('pf_meta'),
'_tibanna': tibanna.as_dict(),
}
def handler(event, context):
'''
this is to run the actual task
'''
# used to automatically determine the environment
tibanna_settings = event.get('_tibanna', {})
tibanna = utils.Tibanna(**tibanna_settings)
sbg = sbg_utils.create_sbg_workflow(token=tibanna.sbg_keys, **event.get('workflow'))
ff_meta = ff_utils.create_ffmeta(sbg, **event.get('ff_meta'))
# TODO: check a whole bunch of stuff here maybe...
if ff_meta.run_status == "output_file_transfer_finished":
ff_meta.run_status = 'complete'
else:
ff_meta.run_status = 'error'
ff_meta.description = ('set to error because status comming into finalize was not' +
' output_file_transfer_finished as expected')
# make all the file export meta-data stuff here
# TODO: fix ff_meta mapping issue
ff_meta.post(key=tibanna.ff_keys)
return {'workflow': sbg.as_dict(),
'ff_meta': ff_meta.as_dict(),
"_tibanna": tibanna.as_dict(),
}
def test_sbg_workflow_as_dict_clears_secrets(workflow_event_data):
wf = workflow_event_data['workflow']
sbg = sbg_utils.create_sbg_workflow(**wf)
sbg_dict = sbg.as_dict()
assert not sbg_dict.get('header')
assert not sbg_dict.get('token')
assert sbg_dict.get('output_volume_id') == wf['output_volume_id']
def handler(event, context):
'''
this is to check if the task run is done:
http://docs.sevenbridges.com/reference#get-task-execution-details
consider this format:
{'data': event.get('workflow'), # actully in reverse
'data_name': 'workflow', # how to get the data
'conversion_routine': 'function to run, taking workflow as argument'
}
then put / patch data_name with data
#
'''
# get data
sbg = sbg_utils.create_sbg_workflow(**event.get('workflow'))
# run_response = event.get('run_response')
ff_meta = sbg_utils.create_ffmeta(sbg, **event.get('ff_meta'))
key = event.get('ff_keys')
ff_keys = sbg_utils.get_access_keys() if not key else key
workflow_post_resp = ff_meta.post(key=ff_keys)
return {
"workflow": sbg.as_dict(),
"res": workflow_post_resp,
"ff_meta": ff_meta.as_dict()
}
def md5_sbg_wfrun(sbg_keys):
try:
return sbg_utils.create_sbg_workflow('md5', token=sbg_keys)
except:
print(
"generally this test fails if you haven't set aws keys in your terminal"
)
def test_create_ff_meta_base_sbg_data(json_request):
app_name = json_request['app_name']
sbg = sbg_utils.create_sbg_workflow(app_name)
parameters, input_files = sbg_utils.to_sbg_workflow_args(json_request['parameters'])
ff_meta = sbg_utils.create_ffmeta(sbg, json_request['workflow_uuid'],
input_files, parameters)
assert ff_meta.title.startswith(app_name)
assert ff_meta.input_files == input_files
assert ff_meta.parameters == parameters
def handler(event, context):
# get data
input_file_args = event.get('input_file_args')
parameter_dict = event.get('parameter_dict')
# used to automatically determine the environment
tibanna_settings = event.get('_tibanna', {})
tibanna = utils.Tibanna(**tibanna_settings)
sbg = sbg_utils.create_sbg_workflow(token=tibanna.sbg_keys, **event.get('workflow'))
ff_meta = ff_utils.create_ffmeta(sbg, **event.get('ff_meta'))
# import_ids = sbg.import_id_list
pf_meta = event.get('pf_meta')
# create Task for workflow run later
task_name = sbg.app_name
if tibanna.settings and tibanna.settings.get('run_name'):
task_name = tibanna.settings.get('run_name')
task_input = sbg_utils.SBGTaskInput(task_name,
project=sbg.project_id,
app=sbg.project_id + '/' + sbg.app_name,
inputs=parameter_dict)
for input_file in input_file_args:
for import_id in input_file.get('import_ids', []):
# this will handle checking if the import / file is on sbg, otherwise
# it will throw an error
res = sbg.check_import(import_id)
# No idea why, but sometimes it comes back without
# results as a sub object
results = res.get('result', res)
sbg_file_name = results.get('name')
sbg_file_id = results.get('id')
arg_name = input_file.get('workflow_argument_name')
arg_uuids = input_file.get('uuid')
# we need to know if this is a list so we can build proper task inputs for sbg
is_list = isinstance(arg_uuids, (list, tuple))
task_input.add_inputfile(sbg_file_name, sbg_file_id, arg_name, is_list)
sbg.task_input = task_input
# ff_meta.input_files.append({'workflow_argument_name': arg_name, 'value': arg_uuid})
# Soo: This information was alreadyin ff_meta that was passed into this function.
# make all the file export meta-data stuff here
# TODO: fix ff_meta bugs with input / output files
ff_meta.post(key=tibanna.ff_keys)
return {'workflow': sbg.as_dict(),
'ff_meta': ff_meta.as_dict(),
'pf_meta': pf_meta,
"_tibanna": tibanna.as_dict(),
}
def test_create_sbg_workflow_from_event_parameter(workflow_event_data):
wf = workflow_event_data['workflow']
sbg = sbg_utils.create_sbg_workflow(**wf)
assert sbg.export_id_list == wf['export_id_list']
assert sbg.volume_list == wf['volume_list']
assert sbg.import_id_list == wf['import_id_list']
assert sbg.project_id == wf['project_id']
assert sbg.app_name == wf['app_name']
assert sbg.header == wf['header']
assert sbg.task_input.__dict__ == wf['task_input']
assert sbg.output_volume_id == wf['output_volume_id']
def test_create_ff_meta_pulls_data_from_sbg_object(workflow_event_data, json_request):
sbg = sbg_utils.create_sbg_workflow(**workflow_event_data['workflow'])
parameters, input_files = sbg_utils.to_sbg_workflow_args(json_request['parameters'])
ff_meta = sbg_utils.create_ffmeta(sbg, json_request['workflow_uuid'],
input_files, parameters)
assert ff_meta
assert ff_meta.title.startswith(sbg.app_name)
assert ff_meta.input_files == input_files
assert ff_meta.parameters == parameters
assert ff_meta.sbg_import_ids == sbg.import_id_list
vols_in_test_data = [item['name'] for item in sbg.volume_list]
for vol in ff_meta.sbg_mounted_volume_ids:
assert vol in vols_in_test_data
assert ff_meta.sbg_task_id == sbg.task_id
def handler(event, context):
'''
this is to check if the task run is done:
http://docs.sevenbridges.com/reference#get-task-execution-details
'''
# get data
sbg = sbg_utils.create_sbg_workflow(**event.get('workflow'))
ff_meta = sbg_utils.create_ffmeta(sbg, **event.get('ff_meta'))
pf_meta = event.get('pf_meta')
for idx, export in enumerate(sbg.export_report):
filename = export['filename']
export_id = export['export_id']
export_res = sbg.check_export(export_id)
status = export_res.get('state')
sbg.export_report[idx]['status'] = status
if status == 'COMPLETED':
patch_meta = OUTFILE_UPDATERS[sbg.app_name]('uploaded', sbg,
ff_meta)
if pf_meta:
pf_meta = update_processed_file_metadata('uploaded', pf_meta)
elif status in ['PENDING', 'RUNNING']:
patch_meta = OUTFILE_UPDATERS[sbg.app_name]('uploading', sbg,
ff_meta)
raise Exception("Export of file %s is still running" % filename)
elif status in ['FAILED']:
patch_meta = OUTFILE_UPDATERS[sbg.app_name]('upload failed', sbg,
ff_meta)
raise Exception("Failed to export file %s \n sbg result: %s" %
(filename, export_res))
# if we got all the exports let's go ahead and update our ff_metadata object
ff_meta.run_status = "output_file_transfer_finished"
# allow for a simple way for updater to add appropriate meta_data
if patch_meta:
ff_meta.__dict__.update(patch_meta)
# make all the file export meta-data stuff here
# TODO: fix bugs with ff_meta mapping for output and input file
ff_meta.post(key=sbg_utils.get_access_keys())
return {
'workflow': sbg.as_dict(),
'ff_meta': ff_meta.as_dict(),
'pf_meta': pf_meta
}
def handler(event, context):
'''
this is to check if the task run is done:
http://docs.sevenbridges.com/reference#get-task-execution-details
'''
# get data
sbg = sbg_utils.create_sbg_workflow(**event.get('workflow'))
# check status of workflow, error if not done
status = sbg.check_task()
if not status['status'] in ['DONE', 'COMPLETED', 'FAILED']:
data = {'workflow': sbg.as_dict(),
'status': status}
raise Exception('Task not finished => %s' % data)
return {'workflow': sbg.as_dict(),
'run_response': status,
'ff_meta': event.get('ff_meta')
}
def handler(event, context):
'''
this is to check if the task run is done:
http://docs.sevenbridges.com/reference#get-task-execution-details
'''
# get data
sbg = sbg_utils.create_sbg_workflow(**event.get('workflow'))
ff_meta = sbg_utils.create_ffmeta(sbg, **event.get('ff_meta'))
pf_meta = event.get('pf_meta')
for idx, export in enumerate(sbg.export_report):
filename = export['filename']
export_id = export['export_id']
export_res = sbg.check_export(export_id)
status = export_res.get('state')
sbg.export_report[idx]['status'] = status
if status == 'COMPLETED':
patch_meta = OUTFILE_UPDATERS[sbg.app_name]('uploaded', sbg, ff_meta)
if pf_meta:
pf_meta = update_processed_file_metadata('uploaded', pf_meta)
elif status in ['PENDING', 'RUNNING']:
patch_meta = OUTFILE_UPDATERS[sbg.app_name]('uploading', sbg, ff_meta)
raise Exception("Export of file %s is still running" % filename)
elif status in ['FAILED']:
patch_meta = OUTFILE_UPDATERS[sbg.app_name]('upload failed', sbg, ff_meta)
raise Exception("Failed to export file %s \n sbg result: %s" % (filename, export_res))
# if we got all the exports let's go ahead and update our ff_metadata object
ff_meta.run_status = "output_file_transfer_finished"
# allow for a simple way for updater to add appropriate meta_data
if patch_meta:
ff_meta.__dict__.update(patch_meta)
# make all the file export meta-data stuff here
# TODO: fix bugs with ff_meta mapping for output and input file
ff_meta.post(key=sbg_utils.get_access_keys())
return {'workflow': sbg.as_dict(),
'ff_meta': ff_meta.as_dict(),
'pf_meta': pf_meta
}
def handler(event, context):
'''
this is to run the actual task
'''
# used to automatically determine the environment
tibanna_settings = event.get('_tibanna', {})
tibanna = utils.Tibanna(**tibanna_settings)
sbg = sbg_utils.create_sbg_workflow(token=tibanna.sbg_keys,
**event.get('workflow'))
pf_meta = event.get('pf_meta')
# create task on SBG
LOG.info("sbg task input is %s" % sbg.task_input.__dict__)
create_resp = sbg.create_task(sbg.task_input)
LOG.info("create task response is %s" % create_resp)
if create_resp['status'] != 'DRAFT':
raise Exception("Failed to create draft task with input %s" %
sbg.task_input.__dict__)
run_response = sbg.run_task()
LOG.info("run task respons is %s" % run_response)
if run_response.get('status', 0) == 400:
raise Exception(
"Failed to create task with input %s\n detailed info is %s" %
(sbg.task_input.__dict__, run_response))
ff_meta = ff_utils.create_ffmeta(sbg, **event.get('ff_meta'))
ff_meta.run_status = 'running'
ff_meta.sbg_task_id = run_response.get('id')
# make all the file export meta-data stuff here
# TODO: fix ff_meta mapping issue
ff_meta.post(key=tibanna.ff_keys)
return {
'workflow': sbg.as_dict(),
'run_response': run_response,
'ff_meta': ff_meta.as_dict(),
'pf_meta': pf_meta,
"_tibanna": tibanna.as_dict(),
}
def handler(event, context):
# get data
input_file_args = event.get('input_file_args')
parameter_dict = event.get('parameter_dict')
sbg = sbg_utils.create_sbg_workflow(**event.get('workflow'))
ff_meta = sbg_utils.create_ffmeta(sbg, **event.get('ff_meta'))
import_ids = sbg.import_id_list
# create Task for workflow run later
task_input = sbg_utils.SBGTaskInput(sbg.app_name,
project=sbg.project_id,
inputs=parameter_dict)
for idx, import_id in enumerate(import_ids):
data = json.dumps({"import_id": import_id})
# TODO: Let this be a funciton of SBGWorkflowRun
# Soo: We already had this function in SBGWorkflowRun. What happened to it?
res = _api._get("/storage/imports/" + import_id, data).json()
if res.get('state') != 'COMPLETED':
raise Exception("file still uploading")
else:
# No idea why, but sometimes it comes back without
# results as a sub object
results = res.get('result', res)
sbg_file_name = results.get('name')
sbg_file_id = results.get('id')
arg_name = input_file_args[idx].get('workflow_argument_name')
# arg_uuid = input_file_args[idx].get('uuid')
task_input.add_inputfile(sbg_file_name, sbg_file_id, arg_name)
sbg.task_input = task_input
# ff_meta.input_files.append({'workflow_argument_name': arg_name, 'value': arg_uuid})
# Soo: This information was alreadyin ff_meta that was passed into this function.
# make all the file export meta-data stuff here
# TODO: fix ff_meta bugs with input / output files
ff_meta.post(key=sbg_utils.get_access_keys())
return {'workflow': sbg.as_dict(),
'ff_meta': ff_meta.as_dict()
}
def handler(event, context):
# get data
input_file_args = event.get('input_file_args')
parameter_dict = event.get('parameter_dict')
sbg = sbg_utils.create_sbg_workflow(**event.get('workflow'))
ff_meta = sbg_utils.create_ffmeta(sbg, **event.get('ff_meta'))
import_ids = sbg.import_id_list
# create Task for workflow run later
task_input = sbg_utils.SBGTaskInput(sbg.app_name,
project=sbg.project_id,
inputs=parameter_dict)
for idx, import_id in enumerate(import_ids):
data = json.dumps({"import_id": import_id})
# TODO: Let this be a funciton of SBGWorkflowRun
# Soo: We already had this function in SBGWorkflowRun. What happened to it?
res = _api._get("/storage/imports/" + import_id, data).json()
if res.get('state') != 'COMPLETED':
raise Exception("file still uploading")
else:
# No idea why, but sometimes it comes back without
# results as a sub object
results = res.get('result', res)
sbg_file_name = results.get('name')
sbg_file_id = results.get('id')
arg_name = input_file_args[idx].get('workflow_argument_name')
# arg_uuid = input_file_args[idx].get('uuid')
task_input.add_inputfile(sbg_file_name, sbg_file_id, arg_name)
sbg.task_input = task_input
# ff_meta.input_files.append({'workflow_argument_name': arg_name, 'value': arg_uuid})
# Soo: This information was alreadyin ff_meta that was passed into this function.
# make all the file export meta-data stuff here
# TODO: fix ff_meta bugs with input / output files
ff_meta.post(key=sbg_utils.get_access_keys())
return {'workflow': sbg.as_dict(), 'ff_meta': ff_meta.as_dict()}
def handler(event, context):
'''
this is to check if the task run is done:
http://docs.sevenbridges.com/reference#get-task-execution-details
consider this format:
{'data': event.get('workflow'), # actully in reverse
'data_name': 'workflow', # how to get the data
'conversion_routine': 'function to run, taking workflow as argument'
}
then put / patch data_name with data
#
'''
# get data
sbg = sbg_utils.create_sbg_workflow(**event.get('workflow'))
# run_response = event.get('run_response')
ff_meta = sbg_utils.create_ffmeta(sbg, **event.get('ff_meta'))
key = event.get('ff_keys')
ff_keys = sbg_utils.get_access_keys() if not key else key
workflow_post_resp = ff_meta.post(key=ff_keys)
return{"workflow": sbg.as_dict(),
"res": workflow_post_resp,
"ff_meta": ff_meta.as_dict()}
def handler(event, context):
'''
export output files from sbg to our s3
'''
# get data
# used to automatically determine the environment
tibanna_settings = event.get('_tibanna', {})
tibanna = Tibanna(**tibanna_settings)
sbg = sbg_utils.create_sbg_workflow(token=tibanna.sbg_keys,
**event.get('workflow'))
run_response = event.get('run_response')
ff_meta = event.get('ff_meta')
uuid = ff_meta['uuid']
pf_meta = event.get('pf_meta')
if run_response in ['FAILED', 'ABORTED']:
raise Exception("workflow run failed or aborted")
sbg.export_all_output_files(run_response, ff_meta, base_dir=uuid)
# creating after we export will add output file info to ff_meta
ff_meta = ff_utils.create_ffmeta(sbg, **event.get('ff_meta'))
ff_meta.run_status = "output_files_transferring"
ff_meta.post(key=tibanna.ff_keys)
for pf in pf_meta:
pf['status'] = "uploading"
# we still need a code for patching.
return {
'workflow': sbg.as_dict(),
'ff_meta': ff_meta.as_dict(),
# 'pf_meta': [pf.as_dict() for pf in pf_meta]
'pf_meta': pf_meta,
'_tibanna': tibanna.as_dict()
}
def handler(event, context):
'''
this is generic function to run sbg workflow
based on the data passed in
workflow_uuid : for now, pass this on. Later we can add a code to automatically retrieve this from app_name.
Note multiple workflow_uuids can be available for an app_name
(different versions of the same app could have a different uuid)
'''
# get incomming data
input_file_list = event.get('input_files')
app_name = event.get('app_name')
parameter_dict = event.get('parameters')
workflow_uuid = event.get('workflow_uuid')
output_bucket = event.get('output_bucket')
tibanna_settings = event.get('_tibanna', {})
# if they don't pass in env guess it from output_bucket
env = tibanna_settings.get('env', '-'.join(output_bucket.split('-')[1:-1]))
# tibanna provides access to keys based on env and stuff like that
tibanna = Tibanna(env,
s3_keys=event.get('s3_keys'),
ff_keys=event.get('ff_keys'),
settings=tibanna_settings)
LOG.info("input data is %s" % event)
# represents the SBG info we need
sbg = sbg_utils.create_sbg_workflow(app_name, tibanna.sbg_keys)
LOG.info("sbg is %s" % sbg.__dict__)
# represents the workflow metadata to be stored in fourfront
parameters, _ = sbg_utils.to_sbg_workflow_args(parameter_dict,
vals_as_string=True)
# get argument format & type info from workflow
workflow_info = ff_utils.get_metadata(workflow_uuid, key=tibanna.ff_keys)
LOG.info("workflow info %s" % workflow_info)
if 'error' in workflow_info.get('@type', []):
raise Exception("FATAL, can't lookupt workflow info for % fourfront" %
workflow_uuid)
# This dictionary has a key 'arguments' with a value
# { 'workflow_argument_name': ..., 'argument_type': ..., 'argument_format': ... }
# get format-extension map
try:
fp_schema = ff_utils.get_metadata("profiles/file_processed.json",
key=tibanna.ff_keys)
fe_map = fp_schema.get('file_format_file_extension')
except Exception as e:
LOG.error(
"Can't get format-extension map from file_processed schema. %s\n" %
e)
# processed file metadata
output_files = []
try:
if 'arguments' in workflow_info:
pf_meta = []
for arg in workflow_info.get('arguments'):
if (arg.get('argument_type') in [
'Output processed file', 'Output report file',
'Output QC file'
]):
of = dict()
of['workflow_argument_name'] = arg.get(
'workflow_argument_name')
of['type'] = arg.get('argument_type')
if 'argument_format' in arg:
# These are not processed files but report or QC files.
pf = ff_utils.ProcessedFileMetadata(
file_format=arg.get('argument_format'))
try:
resp = pf.post(
key=tibanna.ff_keys
) # actually post processed file metadata here
resp = resp.get('@graph')[0]
of['upload_key'] = resp.get('upload_key')
of['value'] = resp.get('uuid')
except Exception as e:
LOG.error(
"Failed to post Processed file metadata. %s\n"
% e)
LOG.error("resp" + str(resp) + "\n")
raise e
of['format'] = arg.get('argument_format')
of['extension'] = fe_map.get(
arg.get('argument_format'))
pf_meta.append(pf)
output_files.append(of)
except Exception as e:
LOG.error("output_files = " + str(output_files) + "\n")
LOG.error("Can't prepare output_files information. %s\n" % e)
raise e
# create the ff_meta output info
input_files = []
for input_file in input_file_list:
for idx, uuid in enumerate(ensure_list(input_file['uuid'])):
input_files.append({
'workflow_argument_name':
input_file['workflow_argument_name'],
'value':
uuid,
'ordinal':
idx + 1
})
LOG.info("input_files is %s" % input_files)
ff_meta = ff_utils.create_ffmeta(sbg,
workflow_uuid,
input_files,
parameters,
run_url=tibanna.settings.get('url', ''),
output_files=output_files)
LOG.info("ff_meta is %s" % ff_meta.__dict__)
# store metadata so we know the run has started
ff_meta.post(key=tibanna.ff_keys)
# mount all input files to sbg this will also update sbg to store the import_ids
for infile in input_file_list:
imps = mount_on_sbg(infile, tibanna.s3_keys, sbg)
infile['import_ids'] = imps
# create a link to the output directory as well
if output_bucket:
sbg_volume = sbg_utils.create_sbg_volume_details()
res = sbg.create_volumes(sbg_volume,
output_bucket,
public_key=tibanna.s3_keys['key'],
secret_key=tibanna.s3_keys['secret'])
vol_id = res.get('id')
if not vol_id:
# we got an error
raise Exception("Unable to mount output volume, error is %s " %
res)
sbg.output_volume_id = vol_id
# let's not pass keys in plain text parameters
return {
"input_file_args": input_file_list,
"workflow": sbg.as_dict(),
"ff_meta": ff_meta.as_dict(),
'pf_meta': [meta.as_dict() for meta in pf_meta],
"_tibanna": tibanna.as_dict(),
"parameter_dict": parameter_dict
}
def handler(event, context):
'''
this is generic function to run sbg workflow
based on the data passed in
workflow_uuid : for now, pass this on. Later we can add a code to automatically retrieve this from app_name.
Note multiple workflow_uuids can be available for an app_name
(different versions of the same app could have a different uuid)
'''
print("hello?") # Soo
# get incomming data
input_file_list = event.get('input_files')
app_name = event.get('app_name').encode('utf8')
parameter_dict = event.get('parameters')
workflow_uuid = event.get('workflow_uuid').encode('utf8')
output_bucket = event.get('output_bucket')
tibanna = event.get('_tibanna', {})
# get necessary tokens
s3_keys = event.get('s3_keys')
if not s3_keys:
s3_keys = sbg_utils.get_s3_keys()
ff_keys = event.get('ff_keys')
if not ff_keys:
ff_keys = sbg_utils.get_access_keys()
# represents the SBG info we need
sbg = sbg_utils.create_sbg_workflow(app_name)
# represents the workflow metadata to be stored in fourfront
parameters, _ = sbg_utils.to_sbg_workflow_args(parameter_dict, vals_as_string=True)
# get argument format & type info from workflow
workflow_info = sbg_utils.get_metadata(workflow_uuid, key=ff_keys)
arginfo = dict()
try:
for arg in workflow_info.get('arguments'):
if arg['argument_type'] in ['Output processed file','Output report file','Output QC file']:
argname = arg['workflow_argument_name']
argformat = arg['argument_format'] if arg.has_key('argument_format') else ''
arginfo.update({ argname: {'format': argformat,
'type': arg['argument_type']} })
except Exception as e:
print("Make sure your workflow metadata has 'argument' field. %s\n" % e)
raise e
# create the ff_meta output info
input_files = [{'workflow_argument_name': fil['workflow_argument_name'],
'value': fil['uuid']} for fil in input_file_list]
# create empty output file info
output_files = [{'workflow_argument_name': argname,
'type': arginfo[argname]['type'],
'extension': '', # processed file extention: later get it from file_processed schema
'format': arginfo[argname]['format']} for argname in arginfo.keys()]
ff_meta = sbg_utils.create_ffmeta(sbg, workflow_uuid, input_files, parameters,
run_url=tibanna.get('url', ''), output_files=output_files)
# store metadata so we know the run has started
ff_meta.post(key=ff_keys)
# mount all input files to sbg this will also update sbg to store the import_ids
_ = [mount_on_sbg(infile, s3_keys, sbg) for infile in input_file_list]
# create a link to the output directory as well
if output_bucket:
sbg_volume = sbg_utils.create_sbg_volume_details()
res = sbg.create_volumes(sbg_volume, output_bucket,
public_key=s3_keys['key'],
secret_key=s3_keys['secret'])
vol_id = res.get('id')
if not vol_id:
# we got an error
raise Exception("Unable to mount output volume, error is %s " % res)
sbg.output_volume_id = vol_id
# let's not pass keys in plain text parameters
return {"input_file_args": input_file_list,
"workflow": sbg.as_dict(),
"ff_meta": ff_meta.as_dict(),
"parameter_dict": parameter_dict}
'''
def md5_sbg_wfrun(sbg_keys):
try:
return sbg_utils.create_sbg_workflow('md5', token=sbg_keys)
except:
print("generally this test fails if you haven't set aws keys in your terminal")
def handler(event, context):
'''
this is generic function to run sbg workflow
based on the data passed in
workflow_uuid : for now, pass this on. Later we can add a code to automatically retrieve this from app_name.
Note multiple workflow_uuids can be available for an app_name
(different versions of the same app could have a different uuid)
'''
print("hello?") # Soo
# get incomming data
input_file_list = event.get('input_files')
app_name = event.get('app_name').encode('utf8')
parameter_dict = event.get('parameters')
workflow_uuid = event.get('workflow_uuid').encode('utf8')
output_bucket = event.get('output_bucket')
tibanna = event.get('_tibanna', {})
# get necessary tokens
s3_keys = event.get('s3_keys')
if not s3_keys:
s3_keys = sbg_utils.get_s3_keys()
ff_keys = event.get('ff_keys')
if not ff_keys:
ff_keys = sbg_utils.get_access_keys()
# represents the SBG info we need
sbg = sbg_utils.create_sbg_workflow(app_name)
# represents the workflow metadata to be stored in fourfront
parameters, _ = sbg_utils.to_sbg_workflow_args(parameter_dict,
vals_as_string=True)
# get argument format & type info from workflow
workflow_info = sbg_utils.get_metadata(workflow_uuid, key=ff_keys)
arginfo = dict()
try:
for arg in workflow_info.get('arguments'):
if arg['argument_type'] in [
'Output processed file', 'Output report file',
'Output QC file'
]:
argname = arg['workflow_argument_name']
argformat = arg['argument_format'] if arg.has_key(
'argument_format') else ''
arginfo.update({
argname: {
'format': argformat,
'type': arg['argument_type']
}
})
except Exception as e:
print("Make sure your workflow metadata has 'argument' field. %s\n" %
e)
raise e
# create the ff_meta output info
input_files = [{
'workflow_argument_name': fil['workflow_argument_name'],
'value': fil['uuid']
} for fil in input_file_list]
# create empty output file info
output_files = [
{
'workflow_argument_name': argname,
'type': arginfo[argname]['type'],
'extension':
'', # processed file extention: later get it from file_processed schema
'format': arginfo[argname]['format']
} for argname in arginfo.keys()
]
ff_meta = sbg_utils.create_ffmeta(sbg,
workflow_uuid,
input_files,
parameters,
run_url=tibanna.get('url', ''),
output_files=output_files)
# store metadata so we know the run has started
ff_meta.post(key=ff_keys)
# mount all input files to sbg this will also update sbg to store the import_ids
_ = [mount_on_sbg(infile, s3_keys, sbg) for infile in input_file_list]
# create a link to the output directory as well
if output_bucket:
sbg_volume = sbg_utils.create_sbg_volume_details()
res = sbg.create_volumes(sbg_volume,
output_bucket,
public_key=s3_keys['key'],
secret_key=s3_keys['secret'])
vol_id = res.get('id')
if not vol_id:
# we got an error
raise Exception("Unable to mount output volume, error is %s " %
res)
sbg.output_volume_id = vol_id
# let's not pass keys in plain text parameters
return {
"input_file_args": input_file_list,
"workflow": sbg.as_dict(),
"ff_meta": ff_meta.as_dict(),
"parameter_dict": parameter_dict
}
'''
def handler(event, context):
'''
this is to check if the task run is done:
http://docs.sevenbridges.com/reference#get-task-execution-details
'''
# get data
# used to automatically determine the environment
tibanna_settings = event.get('_tibanna', {})
tibanna = utils.Tibanna(**tibanna_settings)
sbg = sbg_utils.create_sbg_workflow(token=tibanna.sbg_keys,
**event.get('workflow'))
ff_meta = ff_utils.create_ffmeta(sbg, **event.get('ff_meta'))
pf_meta = event.get('pf_meta')
# ensure this bad boy is always initialized
patch_meta = False
if len(sbg.export_report) != len(ff_meta.output_files):
ff_meta.run_status = 'error'
ff_meta.description = "no files output"
ff_meta.post(key=tibanna.ff_keys)
raise Exception(
"Failing the workflow because sbg outputed files = %d and ffmeta = %d"
% (len(sbg.export_report), len(ff_meta.output_files)))
for idx, export in enumerate(sbg.export_report):
upload_key = export['upload_key']
export_id = export['export_id']
export_res = sbg.check_export(export_id)
print("export res is %s", export_res)
status = export_res.get('state')
sbg.export_report[idx]['status'] = status
if status == 'COMPLETED':
patch_meta = OUTFILE_UPDATERS[sbg.app_name]('uploaded', sbg,
ff_meta, tibanna)
if pf_meta:
pf_meta = update_processed_file_metadata(
'uploaded', pf_meta, tibanna)
elif status in ['PENDING', 'RUNNING']:
patch_meta = OUTFILE_UPDATERS[sbg.app_name]('uploading', sbg,
ff_meta, tibanna)
raise sbg_utils.SBGStillRunningException(
"Export of file %s is still running" % upload_key)
elif status in ['FAILED']:
patch_meta = OUTFILE_UPDATERS[sbg.app_name]('upload failed', sbg,
ff_meta, tibanna)
ff_meta.run_status = 'error'
ff_meta.post(key=tibanna.ff_keys)
raise Exception("Failed to export file %s \n sbg result: %s" %
(upload_key, export_res))
# if we got all the exports let's go ahead and update our ff_metadata object
ff_meta.run_status = "output_file_transfer_finished"
# allow for a simple way for updater to add appropriate meta_data
if patch_meta:
ff_meta.__dict__.update(patch_meta)
# make all the file export meta-data stuff here
# TODO: fix bugs with ff_meta mapping for output and input file
ff_meta.post(key=tibanna.ff_keys)
return {
'workflow': sbg.as_dict(),
'ff_meta': ff_meta.as_dict(),
'pf_meta': pf_meta,
'_tibanna': tibanna.as_dict()
}
