def handler(event, context):
'''
this is to check if the task run is done:
http://docs.sevenbridges.com/reference#get-task-execution-details
consider this format:
{'data': event.get('workflow'), # actully in reverse
'data_name': 'workflow', # how to get the data
'conversion_routine': 'function to run, taking workflow as argument'
}
then put / patch data_name with data
#
'''
# get data
sbg = sbg_utils.create_sbg_workflow(**event.get('workflow'))
# run_response = event.get('run_response')
ff_meta = sbg_utils.create_ffmeta(sbg, **event.get('ff_meta'))
key = event.get('ff_keys')
ff_keys = sbg_utils.get_access_keys() if not key else key
workflow_post_resp = ff_meta.post(key=ff_keys)
return {
"workflow": sbg.as_dict(),
"res": workflow_post_resp,
"ff_meta": ff_meta.as_dict()
}
def handler(event, context):
'''
this is to check if the task run is done:
http://docs.sevenbridges.com/reference#get-task-execution-details
'''
# get data
sbg = sbg_utils.create_sbg_workflow(**event.get('workflow'))
ff_meta = sbg_utils.create_ffmeta(sbg, **event.get('ff_meta'))
pf_meta = event.get('pf_meta')
for idx, export in enumerate(sbg.export_report):
filename = export['filename']
export_id = export['export_id']
export_res = sbg.check_export(export_id)
status = export_res.get('state')
sbg.export_report[idx]['status'] = status
if status == 'COMPLETED':
patch_meta = OUTFILE_UPDATERS[sbg.app_name]('uploaded', sbg,
ff_meta)
if pf_meta:
pf_meta = update_processed_file_metadata('uploaded', pf_meta)
elif status in ['PENDING', 'RUNNING']:
patch_meta = OUTFILE_UPDATERS[sbg.app_name]('uploading', sbg,
ff_meta)
raise Exception("Export of file %s is still running" % filename)
elif status in ['FAILED']:
patch_meta = OUTFILE_UPDATERS[sbg.app_name]('upload failed', sbg,
ff_meta)
raise Exception("Failed to export file %s \n sbg result: %s" %
(filename, export_res))
# if we got all the exports let's go ahead and update our ff_metadata object
ff_meta.run_status = "output_file_transfer_finished"
# allow for a simple way for updater to add appropriate meta_data
if patch_meta:
ff_meta.__dict__.update(patch_meta)
# make all the file export meta-data stuff here
# TODO: fix bugs with ff_meta mapping for output and input file
ff_meta.post(key=sbg_utils.get_access_keys())
return {
'workflow': sbg.as_dict(),
'ff_meta': ff_meta.as_dict(),
'pf_meta': pf_meta
}
def handler(event, context):
'''
this is to check if the task run is done:
http://docs.sevenbridges.com/reference#get-task-execution-details
'''
# get data
sbg = sbg_utils.create_sbg_workflow(**event.get('workflow'))
ff_meta = sbg_utils.create_ffmeta(sbg, **event.get('ff_meta'))
pf_meta = event.get('pf_meta')
for idx, export in enumerate(sbg.export_report):
filename = export['filename']
export_id = export['export_id']
export_res = sbg.check_export(export_id)
status = export_res.get('state')
sbg.export_report[idx]['status'] = status
if status == 'COMPLETED':
patch_meta = OUTFILE_UPDATERS[sbg.app_name]('uploaded', sbg, ff_meta)
if pf_meta:
pf_meta = update_processed_file_metadata('uploaded', pf_meta)
elif status in ['PENDING', 'RUNNING']:
patch_meta = OUTFILE_UPDATERS[sbg.app_name]('uploading', sbg, ff_meta)
raise Exception("Export of file %s is still running" % filename)
elif status in ['FAILED']:
patch_meta = OUTFILE_UPDATERS[sbg.app_name]('upload failed', sbg, ff_meta)
raise Exception("Failed to export file %s \n sbg result: %s" % (filename, export_res))
# if we got all the exports let's go ahead and update our ff_metadata object
ff_meta.run_status = "output_file_transfer_finished"
# allow for a simple way for updater to add appropriate meta_data
if patch_meta:
ff_meta.__dict__.update(patch_meta)
# make all the file export meta-data stuff here
# TODO: fix bugs with ff_meta mapping for output and input file
ff_meta.post(key=sbg_utils.get_access_keys())
return {'workflow': sbg.as_dict(),
'ff_meta': ff_meta.as_dict(),
'pf_meta': pf_meta
}
def handler(event, context):
# get data
input_file_args = event.get('input_file_args')
parameter_dict = event.get('parameter_dict')
sbg = sbg_utils.create_sbg_workflow(**event.get('workflow'))
ff_meta = sbg_utils.create_ffmeta(sbg, **event.get('ff_meta'))
import_ids = sbg.import_id_list
# create Task for workflow run later
task_input = sbg_utils.SBGTaskInput(sbg.app_name,
project=sbg.project_id,
inputs=parameter_dict)
for idx, import_id in enumerate(import_ids):
data = json.dumps({"import_id": import_id})
# TODO: Let this be a funciton of SBGWorkflowRun
# Soo: We already had this function in SBGWorkflowRun. What happened to it?
res = _api._get("/storage/imports/" + import_id, data).json()
if res.get('state') != 'COMPLETED':
raise Exception("file still uploading")
else:
# No idea why, but sometimes it comes back without
# results as a sub object
results = res.get('result', res)
sbg_file_name = results.get('name')
sbg_file_id = results.get('id')
arg_name = input_file_args[idx].get('workflow_argument_name')
# arg_uuid = input_file_args[idx].get('uuid')
task_input.add_inputfile(sbg_file_name, sbg_file_id, arg_name)
sbg.task_input = task_input
# ff_meta.input_files.append({'workflow_argument_name': arg_name, 'value': arg_uuid})
# Soo: This information was alreadyin ff_meta that was passed into this function.
# make all the file export meta-data stuff here
# TODO: fix ff_meta bugs with input / output files
ff_meta.post(key=sbg_utils.get_access_keys())
return {'workflow': sbg.as_dict(),
'ff_meta': ff_meta.as_dict()
}
def handler(event, context):
# get data
input_file_args = event.get('input_file_args')
parameter_dict = event.get('parameter_dict')
sbg = sbg_utils.create_sbg_workflow(**event.get('workflow'))
ff_meta = sbg_utils.create_ffmeta(sbg, **event.get('ff_meta'))
import_ids = sbg.import_id_list
# create Task for workflow run later
task_input = sbg_utils.SBGTaskInput(sbg.app_name,
project=sbg.project_id,
inputs=parameter_dict)
for idx, import_id in enumerate(import_ids):
data = json.dumps({"import_id": import_id})
# TODO: Let this be a funciton of SBGWorkflowRun
# Soo: We already had this function in SBGWorkflowRun. What happened to it?
res = _api._get("/storage/imports/" + import_id, data).json()
if res.get('state') != 'COMPLETED':
raise Exception("file still uploading")
else:
# No idea why, but sometimes it comes back without
# results as a sub object
results = res.get('result', res)
sbg_file_name = results.get('name')
sbg_file_id = results.get('id')
arg_name = input_file_args[idx].get('workflow_argument_name')
# arg_uuid = input_file_args[idx].get('uuid')
task_input.add_inputfile(sbg_file_name, sbg_file_id, arg_name)
sbg.task_input = task_input
# ff_meta.input_files.append({'workflow_argument_name': arg_name, 'value': arg_uuid})
# Soo: This information was alreadyin ff_meta that was passed into this function.
# make all the file export meta-data stuff here
# TODO: fix ff_meta bugs with input / output files
ff_meta.post(key=sbg_utils.get_access_keys())
return {'workflow': sbg.as_dict(), 'ff_meta': ff_meta.as_dict()}
def handler(event, context):
'''
this is to check if the task run is done:
http://docs.sevenbridges.com/reference#get-task-execution-details
consider this format:
{'data': event.get('workflow'), # actully in reverse
'data_name': 'workflow', # how to get the data
'conversion_routine': 'function to run, taking workflow as argument'
}
then put / patch data_name with data
#
'''
# get data
sbg = sbg_utils.create_sbg_workflow(**event.get('workflow'))
# run_response = event.get('run_response')
ff_meta = sbg_utils.create_ffmeta(sbg, **event.get('ff_meta'))
key = event.get('ff_keys')
ff_keys = sbg_utils.get_access_keys() if not key else key
workflow_post_resp = ff_meta.post(key=ff_keys)
return{"workflow": sbg.as_dict(),
"res": workflow_post_resp,
"ff_meta": ff_meta.as_dict()}
def handler(event, context):
'''
this is generic function to run sbg workflow
based on the data passed in
workflow_uuid : for now, pass this on. Later we can add a code to automatically retrieve this from app_name.
Note multiple workflow_uuids can be available for an app_name
(different versions of the same app could have a different uuid)
'''
print("hello?") # Soo
# get incomming data
input_file_list = event.get('input_files')
app_name = event.get('app_name').encode('utf8')
parameter_dict = event.get('parameters')
workflow_uuid = event.get('workflow_uuid').encode('utf8')
output_bucket = event.get('output_bucket')
tibanna = event.get('_tibanna', {})
# get necessary tokens
s3_keys = event.get('s3_keys')
if not s3_keys:
s3_keys = sbg_utils.get_s3_keys()
ff_keys = event.get('ff_keys')
if not ff_keys:
ff_keys = sbg_utils.get_access_keys()
# represents the SBG info we need
sbg = sbg_utils.create_sbg_workflow(app_name)
# represents the workflow metadata to be stored in fourfront
parameters, _ = sbg_utils.to_sbg_workflow_args(parameter_dict,
vals_as_string=True)
# get argument format & type info from workflow
workflow_info = sbg_utils.get_metadata(workflow_uuid, key=ff_keys)
arginfo = dict()
try:
for arg in workflow_info.get('arguments'):
if arg['argument_type'] in [
'Output processed file', 'Output report file',
'Output QC file'
]:
argname = arg['workflow_argument_name']
argformat = arg['argument_format'] if arg.has_key(
'argument_format') else ''
arginfo.update({
argname: {
'format': argformat,
'type': arg['argument_type']
}
})
except Exception as e:
print("Make sure your workflow metadata has 'argument' field. %s\n" %
e)
raise e
# create the ff_meta output info
input_files = [{
'workflow_argument_name': fil['workflow_argument_name'],
'value': fil['uuid']
} for fil in input_file_list]
# create empty output file info
output_files = [
{
'workflow_argument_name': argname,
'type': arginfo[argname]['type'],
'extension':
'', # processed file extention: later get it from file_processed schema
'format': arginfo[argname]['format']
} for argname in arginfo.keys()
]
ff_meta = sbg_utils.create_ffmeta(sbg,
workflow_uuid,
input_files,
parameters,
run_url=tibanna.get('url', ''),
output_files=output_files)
# store metadata so we know the run has started
ff_meta.post(key=ff_keys)
# mount all input files to sbg this will also update sbg to store the import_ids
_ = [mount_on_sbg(infile, s3_keys, sbg) for infile in input_file_list]
# create a link to the output directory as well
if output_bucket:
sbg_volume = sbg_utils.create_sbg_volume_details()
res = sbg.create_volumes(sbg_volume,
output_bucket,
public_key=s3_keys['key'],
secret_key=s3_keys['secret'])
vol_id = res.get('id')
if not vol_id:
# we got an error
raise Exception("Unable to mount output volume, error is %s " %
res)
sbg.output_volume_id = vol_id
# let's not pass keys in plain text parameters
return {
"input_file_args": input_file_list,
"workflow": sbg.as_dict(),
"ff_meta": ff_meta.as_dict(),
"parameter_dict": parameter_dict
}
'''
# -*- coding: utf-8 -*-
from core import utils, sbg_utils
import boto3
from collections import defaultdict
from core.fastqc_utils import parse_fastqc
s3 = boto3.resource('s3')
ff_key = sbg_utils.get_access_keys()
def donothing(status, sbg, ff_meta):
return None
def update_processed_file_metadata(status, pf_meta):
try:
for pf in pf_meta:
pf['status'] = status
except Exception as e:
raise Exception("Unable to update processed file metadata json : %s" %
e)
try:
for pf in pf_meta:
pfo = sbg_utils.ProcessedFileMetadata(**pf)
pfo.post(key=ff_key)
except Exception as e:
raise Exception("Unable to post processed file metadata : %s" % e)
return pf_meta
def fastqc_updater(status, sbg, ff_meta):
# -*- coding: utf-8 -*-
from core import utils, sbg_utils
import boto3
from collections import defaultdict
from core.fastqc_utils import parse_fastqc
s3 = boto3.resource('s3')
ff_key = sbg_utils.get_access_keys()
def donothing(status, sbg, ff_meta):
return None
def update_processed_file_metadata(status, pf_meta):
try:
for pf in pf_meta:
pf['status'] = status
except Exception as e:
raise Exception("Unable to update processed file metadata json : %s" % e)
try:
for pf in pf_meta:
pfo = sbg_utils.ProcessedFileMetadata(**pf)
pfo.post(key=ff_key)
except Exception as e:
raise Exception("Unable to post processed file metadata : %s" % e)
return pf_meta
def fastqc_updater(status, sbg, ff_meta):
def handler(event, context):
'''
this is generic function to run sbg workflow
based on the data passed in
workflow_uuid : for now, pass this on. Later we can add a code to automatically retrieve this from app_name.
Note multiple workflow_uuids can be available for an app_name
(different versions of the same app could have a different uuid)
'''
print("hello?") # Soo
# get incomming data
input_file_list = event.get('input_files')
app_name = event.get('app_name').encode('utf8')
parameter_dict = event.get('parameters')
workflow_uuid = event.get('workflow_uuid').encode('utf8')
output_bucket = event.get('output_bucket')
tibanna = event.get('_tibanna', {})
# get necessary tokens
s3_keys = event.get('s3_keys')
if not s3_keys:
s3_keys = sbg_utils.get_s3_keys()
ff_keys = event.get('ff_keys')
if not ff_keys:
ff_keys = sbg_utils.get_access_keys()
# represents the SBG info we need
sbg = sbg_utils.create_sbg_workflow(app_name)
# represents the workflow metadata to be stored in fourfront
parameters, _ = sbg_utils.to_sbg_workflow_args(parameter_dict, vals_as_string=True)
# get argument format & type info from workflow
workflow_info = sbg_utils.get_metadata(workflow_uuid, key=ff_keys)
arginfo = dict()
try:
for arg in workflow_info.get('arguments'):
if arg['argument_type'] in ['Output processed file','Output report file','Output QC file']:
argname = arg['workflow_argument_name']
argformat = arg['argument_format'] if arg.has_key('argument_format') else ''
arginfo.update({ argname: {'format': argformat,
'type': arg['argument_type']} })
except Exception as e:
print("Make sure your workflow metadata has 'argument' field. %s\n" % e)
raise e
# create the ff_meta output info
input_files = [{'workflow_argument_name': fil['workflow_argument_name'],
'value': fil['uuid']} for fil in input_file_list]
# create empty output file info
output_files = [{'workflow_argument_name': argname,
'type': arginfo[argname]['type'],
'extension': '', # processed file extention: later get it from file_processed schema
'format': arginfo[argname]['format']} for argname in arginfo.keys()]
ff_meta = sbg_utils.create_ffmeta(sbg, workflow_uuid, input_files, parameters,
run_url=tibanna.get('url', ''), output_files=output_files)
# store metadata so we know the run has started
ff_meta.post(key=ff_keys)
# mount all input files to sbg this will also update sbg to store the import_ids
_ = [mount_on_sbg(infile, s3_keys, sbg) for infile in input_file_list]
# create a link to the output directory as well
if output_bucket:
sbg_volume = sbg_utils.create_sbg_volume_details()
res = sbg.create_volumes(sbg_volume, output_bucket,
public_key=s3_keys['key'],
secret_key=s3_keys['secret'])
vol_id = res.get('id')
if not vol_id:
# we got an error
raise Exception("Unable to mount output volume, error is %s " % res)
sbg.output_volume_id = vol_id
# let's not pass keys in plain text parameters
return {"input_file_args": input_file_list,
"workflow": sbg.as_dict(),
"ff_meta": ff_meta.as_dict(),
"parameter_dict": parameter_dict}
'''
def ff_keys():
return sbg_utils.get_access_keys()
def ff_keys():
return sbg_utils.get_access_keys()