def run_workflow(ctx, input_json='', workflow=''): with open(input_json) as input_file: data = json.load(input_file) if workflow == '': resp = _run_workflow(data) else: resp = _run_workflow(data, workflow=workflow) run('open %s' % resp[_tibanna]['url'])
def rerun(exec_arn, workflow='tibanna_pony', override_config=None, app_name_filter=None): """rerun a specific job override_config : dictionary for overriding config (keys are the keys inside config) e.g. override_config = { 'instance_type': 't2.micro' } app_name_filter : app_name (e.g. hi-c-processing-pairs), if specified, then rerun only if it matches app_name """ client = boto3.client('stepfunctions') res = client.describe_execution(executionArn=exec_arn) awsem_template = json.loads(res['input']) # filter by app_name if app_name_filter: if 'app_name' not in awsem_template: return (None) if awsem_template['app_name'] != app_name_filter: return (None) clear_awsem_template(awsem_template) # override config if override_config: for k, v in override_config.iteritems(): awsem_template['config'][k] = v return (_run_workflow(awsem_template, workflow=workflow))
def run_fastqc(env, accession, uuid): if not accession.endswith(".fastq.gz"): accession += ".fastq.gz" input_json = make_input(env=env, workflow='fastqc-0-11-4-1', object_key=accession, uuid=uuid) return _run_workflow(input_json, accession)
def run_workflow(ctx, input_json='', sfn='', jobid=''): """run a workflow""" if not jobid: jobid = create_jobid() with open(input_json) as input_file: data = json.load(input_file) if sfn == '': resp = _run_workflow(data, sfn=TIBANNA_DEFAULT_STEP_FUNCTION_NAME, jobid=jobid) else: resp = _run_workflow(data, sfn=sfn, jobid=jobid) print("JOBID %s submitted" % resp['jobid']) print("EXECUTION ARN = %s" % resp[_tibanna]['exec_arn']) if 'cloudwatch_dashboard' in resp['config'] and resp['config']['cloudwatch_dashboard']: cw_db_url = 'https://console.aws.amazon.com/cloudwatch/' + \ 'home?region=%s#dashboards:name=awsem-%s' % (AWS_REGION, jobid) print("Cloudwatch Dashboard = %s" % cw_db_url) run('open %s' % resp[_tibanna]['url'])
def run_md5(ctx, env, accession, uuid): if not accession.endswith(".fastq.gz"): accession += ".fastq.gz" input_json = make_input(env=env, workflow='md5', accession=accession, uuid=uuid) return _run_workflow(input_json, accession)
def run_md5(env, accession, uuid): tibanna = Tibanna(env=env) meta_data = get_metadata(accession, key=tibanna.ff_keys) file_name = meta_data['upload_key'].split('/')[-1] input_json = make_input(env=env, workflow='md5', object_key=file_name, uuid=uuid) return _run_workflow(input_json, accession)
def collect_pairs_files_to_run_hi_c_processing_pairs( keypairs_file, webprod=True, wfuuid='c9e0e6f7-b0ed-4a42-9466-cadc2dd84df0', prev_workflow_uuid='023bfb3e-9a8b-42b9-a9d4-216079526f68', prev_output_argument_name='filtered_pairs', awsem_template_json='awsem_hicpairs_easy.json', input_argument_name='input_pairs', awsem_tag="0.2.5", parameters_to_override={'maxmem': '32g'}, parameters_to_delete=['custom_res', 'min_res'], datatype_filter=['in situ Hi-C', 'dilution Hi-C'], stepfunction_workflow='tibanna_pony'): """Very high-level function for collecting all legit pairs files and run hi-c-processing-pairs. It will become more generalized soon. """ re_restriction_file = {'MboI': 'files-reference/4DNFI823L812', 'HindIII': 'files-reference/4DNFI823MBKE', 'DpnII': 'files-reference/4DNFIBNAPW30'} connection = get_connection(keypairs_file) input_files_list = prep_input_file_entry_list_for_merging_expset(input_argument_name, prev_workflow_uuid, prev_output_argument_name, connection, addon='re', wfuuid=wfuuid, datatype_filter=datatype_filter) if input_files_list: for _, entry in input_files_list.iteritems(): print(entry) if entry['RE'] not in re_restriction_file: print('RE not found, skipping.. : ' + entry['RE']) continue re_entry = create_inputfile_entry(re_restriction_file[entry['RE']], 'restriction_file', connection) entry_list = [entry, re_entry] awsem_json = create_awsem_json_for_workflowrun(entry_list, awsem_template_json, awsem_tag=awsem_tag, parameters_to_override=parameters_to_override, parameters_to_delete=parameters_to_delete, webprod=webprod) resp = _run_workflow(awsem_json, workflow=stepfunction_workflow) print(resp) return({'input_files_list': input_files_list})
def collect_pairs_files_to_run_pairsqc( keypairs_file, webprod=True, wfuuid='ae3a87cb-3fa2-469e-97c7-540fc2d0a117', prev_workflow_uuid='023bfb3e-9a8b-42b9-a9d4-216079526f68', prev_output_argument_name='filtered_pairs', awsem_template_json='awsem_pairsqc.json', input_argument_name='input_pairs', awsem_tag="0.2.5", parameters_to_delete=None, datatype_filter=['in situ Hi-C', 'dilution Hi-C', 'capture Hi-C'], stepfunction_workflow='tibanna_pony'): """Very high-level function for collecting all legit pairs files and run hi-c-processing-pairs. It will become more generalized soon. """ re_cutter = {'HindIII': '6', 'DpnII': '4', 'MboI': '4', 'NcoI': '6'} connection = get_connection(keypairs_file) input_files_list = prep_input_file_entry_list_for_single_exp(input_argument_name, prev_workflow_uuid, prev_output_argument_name, connection, addon='re', wfuuid=wfuuid, datatype_filter=datatype_filter) if input_files_list: for _, entry in input_files_list.iteritems(): parameters_to_override = {'sample_name': entry['accession'], 'enzyme': re_cutter[entry['RE']]} awsem_json = create_awsem_json_for_workflowrun([entry], awsem_template_json, awsem_tag=awsem_tag, parameters_to_override=parameters_to_override, parameters_to_delete=parameters_to_delete, webprod=webprod) resp = _run_workflow(awsem_json, workflow=stepfunction_workflow) print(resp) return({'input_files_list': input_files_list})
def run_workflow(ctx, input_json=''): with open(input_json) as input_file: data = json.load(input_file) return _run_workflow(data)
def run_fastqc_workflow(ctx, bucket_name='elasticbeanstalk-encoded-4dn-files', accession='4DNFIW7Q5UDL.fastq.gz', uuid='02e3f7cf-6699-4281-96fa-528bf87b7741' ): input_json = make_input(bucket_name, accession, uuid) return _run_workflow(input_json, accession)
def run_workflow(ctx, input_json=''): with open(input_json) as input_file: data = json.load(input_file) return _run_workflow(data)
def run_fastqc_workflow(ctx, bucket_name='elasticbeanstalk-encoded-4dn-files', accession='4DNFIW7Q5UDL.fastq.gz', uuid='02e3f7cf-6699-4281-96fa-528bf87b7741'): input_json = make_input(bucket_name, accession, uuid) return _run_workflow(input_json, accession)