def run_workflow(ctx, input_json='', workflow=''):
with open(input_json) as input_file:
data = json.load(input_file)
if workflow == '':
resp = _run_workflow(data)
else:
resp = _run_workflow(data, workflow=workflow)
run('open %s' % resp[_tibanna]['url'])
def rerun(exec_arn,
workflow='tibanna_pony',
override_config=None,
app_name_filter=None):
"""rerun a specific job
override_config : dictionary for overriding config (keys are the keys inside config)
e.g. override_config = { 'instance_type': 't2.micro' }
app_name_filter : app_name (e.g. hi-c-processing-pairs), if specified,
then rerun only if it matches app_name
"""
client = boto3.client('stepfunctions')
res = client.describe_execution(executionArn=exec_arn)
awsem_template = json.loads(res['input'])
# filter by app_name
if app_name_filter:
if 'app_name' not in awsem_template:
return (None)
if awsem_template['app_name'] != app_name_filter:
return (None)
clear_awsem_template(awsem_template)
# override config
if override_config:
for k, v in override_config.iteritems():
awsem_template['config'][k] = v
return (_run_workflow(awsem_template, workflow=workflow))
def run_fastqc(env, accession, uuid):
if not accession.endswith(".fastq.gz"):
accession += ".fastq.gz"
input_json = make_input(env=env,
workflow='fastqc-0-11-4-1',
object_key=accession,
uuid=uuid)
return _run_workflow(input_json, accession)
def run_workflow(ctx, input_json='', sfn='', jobid=''):
"""run a workflow"""
if not jobid:
jobid = create_jobid()
with open(input_json) as input_file:
data = json.load(input_file)
if sfn == '':
resp = _run_workflow(data, sfn=TIBANNA_DEFAULT_STEP_FUNCTION_NAME, jobid=jobid)
else:
resp = _run_workflow(data, sfn=sfn, jobid=jobid)
print("JOBID %s submitted" % resp['jobid'])
print("EXECUTION ARN = %s" % resp[_tibanna]['exec_arn'])
if 'cloudwatch_dashboard' in resp['config'] and resp['config']['cloudwatch_dashboard']:
cw_db_url = 'https://console.aws.amazon.com/cloudwatch/' + \
'home?region=%s#dashboards:name=awsem-%s' % (AWS_REGION, jobid)
print("Cloudwatch Dashboard = %s" % cw_db_url)
run('open %s' % resp[_tibanna]['url'])
def run_md5(ctx, env, accession, uuid):
if not accession.endswith(".fastq.gz"):
accession += ".fastq.gz"
input_json = make_input(env=env,
workflow='md5',
accession=accession,
uuid=uuid)
return _run_workflow(input_json, accession)
def run_md5(env, accession, uuid):
tibanna = Tibanna(env=env)
meta_data = get_metadata(accession, key=tibanna.ff_keys)
file_name = meta_data['upload_key'].split('/')[-1]
input_json = make_input(env=env,
workflow='md5',
object_key=file_name,
uuid=uuid)
return _run_workflow(input_json, accession)
def collect_pairs_files_to_run_hi_c_processing_pairs(
keypairs_file,
webprod=True,
wfuuid='c9e0e6f7-b0ed-4a42-9466-cadc2dd84df0',
prev_workflow_uuid='023bfb3e-9a8b-42b9-a9d4-216079526f68',
prev_output_argument_name='filtered_pairs',
awsem_template_json='awsem_hicpairs_easy.json',
input_argument_name='input_pairs',
awsem_tag="0.2.5",
parameters_to_override={'maxmem': '32g'},
parameters_to_delete=['custom_res', 'min_res'],
datatype_filter=['in situ Hi-C', 'dilution Hi-C'],
stepfunction_workflow='tibanna_pony'):
"""Very high-level function for collecting all legit
pairs files and run hi-c-processing-pairs.
It will become more generalized soon.
"""
re_restriction_file = {'MboI': 'files-reference/4DNFI823L812',
'HindIII': 'files-reference/4DNFI823MBKE',
'DpnII': 'files-reference/4DNFIBNAPW30'}
connection = get_connection(keypairs_file)
input_files_list = prep_input_file_entry_list_for_merging_expset(input_argument_name,
prev_workflow_uuid,
prev_output_argument_name,
connection,
addon='re',
wfuuid=wfuuid,
datatype_filter=datatype_filter)
if input_files_list:
for _, entry in input_files_list.iteritems():
print(entry)
if entry['RE'] not in re_restriction_file:
print('RE not found, skipping.. : ' + entry['RE'])
continue
re_entry = create_inputfile_entry(re_restriction_file[entry['RE']], 'restriction_file', connection)
entry_list = [entry, re_entry]
awsem_json = create_awsem_json_for_workflowrun(entry_list, awsem_template_json,
awsem_tag=awsem_tag,
parameters_to_override=parameters_to_override,
parameters_to_delete=parameters_to_delete,
webprod=webprod)
resp = _run_workflow(awsem_json, workflow=stepfunction_workflow)
print(resp)
return({'input_files_list': input_files_list})
def collect_pairs_files_to_run_pairsqc(
keypairs_file,
webprod=True,
wfuuid='ae3a87cb-3fa2-469e-97c7-540fc2d0a117',
prev_workflow_uuid='023bfb3e-9a8b-42b9-a9d4-216079526f68',
prev_output_argument_name='filtered_pairs',
awsem_template_json='awsem_pairsqc.json',
input_argument_name='input_pairs',
awsem_tag="0.2.5",
parameters_to_delete=None,
datatype_filter=['in situ Hi-C', 'dilution Hi-C', 'capture Hi-C'],
stepfunction_workflow='tibanna_pony'):
"""Very high-level function for collecting all legit
pairs files and run hi-c-processing-pairs.
It will become more generalized soon.
"""
re_cutter = {'HindIII': '6', 'DpnII': '4', 'MboI': '4', 'NcoI': '6'}
connection = get_connection(keypairs_file)
input_files_list = prep_input_file_entry_list_for_single_exp(input_argument_name,
prev_workflow_uuid,
prev_output_argument_name,
connection,
addon='re',
wfuuid=wfuuid,
datatype_filter=datatype_filter)
if input_files_list:
for _, entry in input_files_list.iteritems():
parameters_to_override = {'sample_name': entry['accession'], 'enzyme': re_cutter[entry['RE']]}
awsem_json = create_awsem_json_for_workflowrun([entry], awsem_template_json,
awsem_tag=awsem_tag,
parameters_to_override=parameters_to_override,
parameters_to_delete=parameters_to_delete,
webprod=webprod)
resp = _run_workflow(awsem_json, workflow=stepfunction_workflow)
print(resp)
return({'input_files_list': input_files_list})
def run_workflow(ctx, input_json=''):
with open(input_json) as input_file:
data = json.load(input_file)
return _run_workflow(data)
def run_fastqc_workflow(ctx, bucket_name='elasticbeanstalk-encoded-4dn-files',
accession='4DNFIW7Q5UDL.fastq.gz',
uuid='02e3f7cf-6699-4281-96fa-528bf87b7741'
):
input_json = make_input(bucket_name, accession, uuid)
return _run_workflow(input_json, accession)
def run_workflow(ctx, input_json=''):
with open(input_json) as input_file:
data = json.load(input_file)
return _run_workflow(data)
def run_fastqc_workflow(ctx,
bucket_name='elasticbeanstalk-encoded-4dn-files',
accession='4DNFIW7Q5UDL.fastq.gz',
uuid='02e3f7cf-6699-4281-96fa-528bf87b7741'):
input_json = make_input(bucket_name, accession, uuid)
return _run_workflow(input_json, accession)