def handler(event, context): ''' this is to check if the task run is done: http://docs.sevenbridges.com/reference#get-task-execution-details consider this format: {'data': event.get('workflow'), # actully in reverse 'data_name': 'workflow', # how to get the data 'conversion_routine': 'function to run, taking workflow as argument' } then put / patch data_name with data # ''' # get data sbg = sbg_utils.create_sbg_workflow(**event.get('workflow')) # run_response = event.get('run_response') ff_meta = sbg_utils.create_ffmeta(sbg, **event.get('ff_meta')) key = event.get('ff_keys') ff_keys = sbg_utils.get_access_keys() if not key else key workflow_post_resp = ff_meta.post(key=ff_keys) return { "workflow": sbg.as_dict(), "res": workflow_post_resp, "ff_meta": ff_meta.as_dict() }
def handler(event, context): ''' this is to check if the task run is done: http://docs.sevenbridges.com/reference#get-task-execution-details ''' # get data sbg = sbg_utils.create_sbg_workflow(**event.get('workflow')) ff_meta = sbg_utils.create_ffmeta(sbg, **event.get('ff_meta')) pf_meta = event.get('pf_meta') for idx, export in enumerate(sbg.export_report): filename = export['filename'] export_id = export['export_id'] export_res = sbg.check_export(export_id) status = export_res.get('state') sbg.export_report[idx]['status'] = status if status == 'COMPLETED': patch_meta = OUTFILE_UPDATERS[sbg.app_name]('uploaded', sbg, ff_meta) if pf_meta: pf_meta = update_processed_file_metadata('uploaded', pf_meta) elif status in ['PENDING', 'RUNNING']: patch_meta = OUTFILE_UPDATERS[sbg.app_name]('uploading', sbg, ff_meta) raise Exception("Export of file %s is still running" % filename) elif status in ['FAILED']: patch_meta = OUTFILE_UPDATERS[sbg.app_name]('upload failed', sbg, ff_meta) raise Exception("Failed to export file %s \n sbg result: %s" % (filename, export_res)) # if we got all the exports let's go ahead and update our ff_metadata object ff_meta.run_status = "output_file_transfer_finished" # allow for a simple way for updater to add appropriate meta_data if patch_meta: ff_meta.__dict__.update(patch_meta) # make all the file export meta-data stuff here # TODO: fix bugs with ff_meta mapping for output and input file ff_meta.post(key=sbg_utils.get_access_keys()) return { 'workflow': sbg.as_dict(), 'ff_meta': ff_meta.as_dict(), 'pf_meta': pf_meta }
def handler(event, context): ''' this is to check if the task run is done: http://docs.sevenbridges.com/reference#get-task-execution-details ''' # get data sbg = sbg_utils.create_sbg_workflow(**event.get('workflow')) ff_meta = sbg_utils.create_ffmeta(sbg, **event.get('ff_meta')) pf_meta = event.get('pf_meta') for idx, export in enumerate(sbg.export_report): filename = export['filename'] export_id = export['export_id'] export_res = sbg.check_export(export_id) status = export_res.get('state') sbg.export_report[idx]['status'] = status if status == 'COMPLETED': patch_meta = OUTFILE_UPDATERS[sbg.app_name]('uploaded', sbg, ff_meta) if pf_meta: pf_meta = update_processed_file_metadata('uploaded', pf_meta) elif status in ['PENDING', 'RUNNING']: patch_meta = OUTFILE_UPDATERS[sbg.app_name]('uploading', sbg, ff_meta) raise Exception("Export of file %s is still running" % filename) elif status in ['FAILED']: patch_meta = OUTFILE_UPDATERS[sbg.app_name]('upload failed', sbg, ff_meta) raise Exception("Failed to export file %s \n sbg result: %s" % (filename, export_res)) # if we got all the exports let's go ahead and update our ff_metadata object ff_meta.run_status = "output_file_transfer_finished" # allow for a simple way for updater to add appropriate meta_data if patch_meta: ff_meta.__dict__.update(patch_meta) # make all the file export meta-data stuff here # TODO: fix bugs with ff_meta mapping for output and input file ff_meta.post(key=sbg_utils.get_access_keys()) return {'workflow': sbg.as_dict(), 'ff_meta': ff_meta.as_dict(), 'pf_meta': pf_meta }
def handler(event, context): # get data input_file_args = event.get('input_file_args') parameter_dict = event.get('parameter_dict') sbg = sbg_utils.create_sbg_workflow(**event.get('workflow')) ff_meta = sbg_utils.create_ffmeta(sbg, **event.get('ff_meta')) import_ids = sbg.import_id_list # create Task for workflow run later task_input = sbg_utils.SBGTaskInput(sbg.app_name, project=sbg.project_id, inputs=parameter_dict) for idx, import_id in enumerate(import_ids): data = json.dumps({"import_id": import_id}) # TODO: Let this be a funciton of SBGWorkflowRun # Soo: We already had this function in SBGWorkflowRun. What happened to it? res = _api._get("/storage/imports/" + import_id, data).json() if res.get('state') != 'COMPLETED': raise Exception("file still uploading") else: # No idea why, but sometimes it comes back without # results as a sub object results = res.get('result', res) sbg_file_name = results.get('name') sbg_file_id = results.get('id') arg_name = input_file_args[idx].get('workflow_argument_name') # arg_uuid = input_file_args[idx].get('uuid') task_input.add_inputfile(sbg_file_name, sbg_file_id, arg_name) sbg.task_input = task_input # ff_meta.input_files.append({'workflow_argument_name': arg_name, 'value': arg_uuid}) # Soo: This information was alreadyin ff_meta that was passed into this function. # make all the file export meta-data stuff here # TODO: fix ff_meta bugs with input / output files ff_meta.post(key=sbg_utils.get_access_keys()) return {'workflow': sbg.as_dict(), 'ff_meta': ff_meta.as_dict() }
def handler(event, context): # get data input_file_args = event.get('input_file_args') parameter_dict = event.get('parameter_dict') sbg = sbg_utils.create_sbg_workflow(**event.get('workflow')) ff_meta = sbg_utils.create_ffmeta(sbg, **event.get('ff_meta')) import_ids = sbg.import_id_list # create Task for workflow run later task_input = sbg_utils.SBGTaskInput(sbg.app_name, project=sbg.project_id, inputs=parameter_dict) for idx, import_id in enumerate(import_ids): data = json.dumps({"import_id": import_id}) # TODO: Let this be a funciton of SBGWorkflowRun # Soo: We already had this function in SBGWorkflowRun. What happened to it? res = _api._get("/storage/imports/" + import_id, data).json() if res.get('state') != 'COMPLETED': raise Exception("file still uploading") else: # No idea why, but sometimes it comes back without # results as a sub object results = res.get('result', res) sbg_file_name = results.get('name') sbg_file_id = results.get('id') arg_name = input_file_args[idx].get('workflow_argument_name') # arg_uuid = input_file_args[idx].get('uuid') task_input.add_inputfile(sbg_file_name, sbg_file_id, arg_name) sbg.task_input = task_input # ff_meta.input_files.append({'workflow_argument_name': arg_name, 'value': arg_uuid}) # Soo: This information was alreadyin ff_meta that was passed into this function. # make all the file export meta-data stuff here # TODO: fix ff_meta bugs with input / output files ff_meta.post(key=sbg_utils.get_access_keys()) return {'workflow': sbg.as_dict(), 'ff_meta': ff_meta.as_dict()}
def handler(event, context): ''' this is to check if the task run is done: http://docs.sevenbridges.com/reference#get-task-execution-details consider this format: {'data': event.get('workflow'), # actully in reverse 'data_name': 'workflow', # how to get the data 'conversion_routine': 'function to run, taking workflow as argument' } then put / patch data_name with data # ''' # get data sbg = sbg_utils.create_sbg_workflow(**event.get('workflow')) # run_response = event.get('run_response') ff_meta = sbg_utils.create_ffmeta(sbg, **event.get('ff_meta')) key = event.get('ff_keys') ff_keys = sbg_utils.get_access_keys() if not key else key workflow_post_resp = ff_meta.post(key=ff_keys) return{"workflow": sbg.as_dict(), "res": workflow_post_resp, "ff_meta": ff_meta.as_dict()}
def handler(event, context): ''' this is generic function to run sbg workflow based on the data passed in workflow_uuid : for now, pass this on. Later we can add a code to automatically retrieve this from app_name. Note multiple workflow_uuids can be available for an app_name (different versions of the same app could have a different uuid) ''' print("hello?") # Soo # get incomming data input_file_list = event.get('input_files') app_name = event.get('app_name').encode('utf8') parameter_dict = event.get('parameters') workflow_uuid = event.get('workflow_uuid').encode('utf8') output_bucket = event.get('output_bucket') tibanna = event.get('_tibanna', {}) # get necessary tokens s3_keys = event.get('s3_keys') if not s3_keys: s3_keys = sbg_utils.get_s3_keys() ff_keys = event.get('ff_keys') if not ff_keys: ff_keys = sbg_utils.get_access_keys() # represents the SBG info we need sbg = sbg_utils.create_sbg_workflow(app_name) # represents the workflow metadata to be stored in fourfront parameters, _ = sbg_utils.to_sbg_workflow_args(parameter_dict, vals_as_string=True) # get argument format & type info from workflow workflow_info = sbg_utils.get_metadata(workflow_uuid, key=ff_keys) arginfo = dict() try: for arg in workflow_info.get('arguments'): if arg['argument_type'] in [ 'Output processed file', 'Output report file', 'Output QC file' ]: argname = arg['workflow_argument_name'] argformat = arg['argument_format'] if arg.has_key( 'argument_format') else '' arginfo.update({ argname: { 'format': argformat, 'type': arg['argument_type'] } }) except Exception as e: print("Make sure your workflow metadata has 'argument' field. %s\n" % e) raise e # create the ff_meta output info input_files = [{ 'workflow_argument_name': fil['workflow_argument_name'], 'value': fil['uuid'] } for fil in input_file_list] # create empty output file info output_files = [ { 'workflow_argument_name': argname, 'type': arginfo[argname]['type'], 'extension': '', # processed file extention: later get it from file_processed schema 'format': arginfo[argname]['format'] } for argname in arginfo.keys() ] ff_meta = sbg_utils.create_ffmeta(sbg, workflow_uuid, input_files, parameters, run_url=tibanna.get('url', ''), output_files=output_files) # store metadata so we know the run has started ff_meta.post(key=ff_keys) # mount all input files to sbg this will also update sbg to store the import_ids _ = [mount_on_sbg(infile, s3_keys, sbg) for infile in input_file_list] # create a link to the output directory as well if output_bucket: sbg_volume = sbg_utils.create_sbg_volume_details() res = sbg.create_volumes(sbg_volume, output_bucket, public_key=s3_keys['key'], secret_key=s3_keys['secret']) vol_id = res.get('id') if not vol_id: # we got an error raise Exception("Unable to mount output volume, error is %s " % res) sbg.output_volume_id = vol_id # let's not pass keys in plain text parameters return { "input_file_args": input_file_list, "workflow": sbg.as_dict(), "ff_meta": ff_meta.as_dict(), "parameter_dict": parameter_dict } '''
# -*- coding: utf-8 -*- from core import utils, sbg_utils import boto3 from collections import defaultdict from core.fastqc_utils import parse_fastqc s3 = boto3.resource('s3') ff_key = sbg_utils.get_access_keys() def donothing(status, sbg, ff_meta): return None def update_processed_file_metadata(status, pf_meta): try: for pf in pf_meta: pf['status'] = status except Exception as e: raise Exception("Unable to update processed file metadata json : %s" % e) try: for pf in pf_meta: pfo = sbg_utils.ProcessedFileMetadata(**pf) pfo.post(key=ff_key) except Exception as e: raise Exception("Unable to post processed file metadata : %s" % e) return pf_meta def fastqc_updater(status, sbg, ff_meta):
def handler(event, context): ''' this is generic function to run sbg workflow based on the data passed in workflow_uuid : for now, pass this on. Later we can add a code to automatically retrieve this from app_name. Note multiple workflow_uuids can be available for an app_name (different versions of the same app could have a different uuid) ''' print("hello?") # Soo # get incomming data input_file_list = event.get('input_files') app_name = event.get('app_name').encode('utf8') parameter_dict = event.get('parameters') workflow_uuid = event.get('workflow_uuid').encode('utf8') output_bucket = event.get('output_bucket') tibanna = event.get('_tibanna', {}) # get necessary tokens s3_keys = event.get('s3_keys') if not s3_keys: s3_keys = sbg_utils.get_s3_keys() ff_keys = event.get('ff_keys') if not ff_keys: ff_keys = sbg_utils.get_access_keys() # represents the SBG info we need sbg = sbg_utils.create_sbg_workflow(app_name) # represents the workflow metadata to be stored in fourfront parameters, _ = sbg_utils.to_sbg_workflow_args(parameter_dict, vals_as_string=True) # get argument format & type info from workflow workflow_info = sbg_utils.get_metadata(workflow_uuid, key=ff_keys) arginfo = dict() try: for arg in workflow_info.get('arguments'): if arg['argument_type'] in ['Output processed file','Output report file','Output QC file']: argname = arg['workflow_argument_name'] argformat = arg['argument_format'] if arg.has_key('argument_format') else '' arginfo.update({ argname: {'format': argformat, 'type': arg['argument_type']} }) except Exception as e: print("Make sure your workflow metadata has 'argument' field. %s\n" % e) raise e # create the ff_meta output info input_files = [{'workflow_argument_name': fil['workflow_argument_name'], 'value': fil['uuid']} for fil in input_file_list] # create empty output file info output_files = [{'workflow_argument_name': argname, 'type': arginfo[argname]['type'], 'extension': '', # processed file extention: later get it from file_processed schema 'format': arginfo[argname]['format']} for argname in arginfo.keys()] ff_meta = sbg_utils.create_ffmeta(sbg, workflow_uuid, input_files, parameters, run_url=tibanna.get('url', ''), output_files=output_files) # store metadata so we know the run has started ff_meta.post(key=ff_keys) # mount all input files to sbg this will also update sbg to store the import_ids _ = [mount_on_sbg(infile, s3_keys, sbg) for infile in input_file_list] # create a link to the output directory as well if output_bucket: sbg_volume = sbg_utils.create_sbg_volume_details() res = sbg.create_volumes(sbg_volume, output_bucket, public_key=s3_keys['key'], secret_key=s3_keys['secret']) vol_id = res.get('id') if not vol_id: # we got an error raise Exception("Unable to mount output volume, error is %s " % res) sbg.output_volume_id = vol_id # let's not pass keys in plain text parameters return {"input_file_args": input_file_list, "workflow": sbg.as_dict(), "ff_meta": ff_meta.as_dict(), "parameter_dict": parameter_dict} '''
def ff_keys(): return sbg_utils.get_access_keys()