def test_md5_updater_newmd5(update_ffmeta_event_data_newmd5):
event = update_ffmeta_event_data_newmd5
tibanna_settings = event.get('_tibanna', {})
tibanna = pony_utils.Tibanna(**tibanna_settings)
awsem = Awsem(update_ffmeta_event_data_newmd5)
ouf = awsem.output_files()['report']
md5_updater('uploaded', ouf, None, tibanna)
def test_md5_updater_oldmd5(update_ffmeta_event_data):
event = update_ffmeta_event_data
tibanna_settings = event.get('_tibanna', {})
tibanna = Tibanna(**tibanna_settings)
awsem = Awsem(update_ffmeta_event_data)
ouf = awsem.output_files()[0]
md5_updater('uploaded', ouf, None, tibanna)
def test_get_output_files(update_ffmeta_event_data, tibanna_env):
update_ffmeta_event_data.update(tibanna_env)
awsem = Awsem(update_ffmeta_event_data)
of = awsem.output_files()
assert 1 == len(of)
assert of[0].runner == awsem
assert of[0].bucket == awsem.output_s3
assert of[0].key == 'lalala/md5_report'
assert of[0].argument_type == 'Output report file'
def test_get_input_files(update_ffmeta_event_data, tibanna_env):
update_ffmeta_event_data.update(tibanna_env)
awsem = Awsem(update_ffmeta_event_data)
infiles = awsem.input_files()
assert 1 == len(infiles)
assert infiles[0].runner == awsem
assert infiles[0].bucket == 'elasticbeanstalk-fourfront-webdev-files'
assert infiles[
0].key == 'f4864029-a8ad-4bb8-93e7-5108f462ccaa/4DNFIRSRJH45.fastq.gz'
assert infiles[0].accession == '4DNFIRSRJH45'
def test_get_inputfile_format_if_extra(update_ffmeta_event_data_extra_md5,
tibanna_env):
update_ffmeta_event_data_extra_md5.update(tibanna_env)
for _, wf_file in Awsem(
update_ffmeta_event_data_extra_md5).output_files().iteritems():
assert wf_file.runner.inputfile_format_if_extra[
'input_file'] == 'pairs_px2'
def test_create_awsem(update_ffmeta_event_data, tibanna_env):
update_ffmeta_event_data.update(tibanna_env)
awsem = Awsem(update_ffmeta_event_data)
assert awsem.args
assert awsem.config
assert awsem.app_name
assert awsem.output_s3
assert awsem.output_files_meta
def test_get_inputfile_accession(update_ffmeta_event_data, tibanna_env):
update_ffmeta_event_data.update(tibanna_env)
awsem = Awsem(update_ffmeta_event_data)
assert awsem.get_file_accessions('input_file')[0] == '4DNFIRSRJH45'
def real_handler(event, context):
# check the status and other details of import
'''
this is to check if the task run is done:
http://docs.sevenbridges.com/reference#get-task-execution-details
'''
# get data
# used to automatically determine the environment
tibanna_settings = event.get('_tibanna', {})
try:
tibanna = Tibanna(tibanna_settings['env'], settings=tibanna_settings)
except Exception as e:
raise TibannaStartException("%s" % e)
ff_meta = create_ffmeta_awsem(
app_name=event.get('ff_meta').get('awsem_app_name'),
**event.get('ff_meta'))
if event.get('error', False):
ff_meta.run_status = 'error'
ff_meta.description = event.get('error')
patch_res = ff_meta.patch(key=tibanna.ff_keys)
printlog("patch response: " + str(patch_res))
# sending a notification email before throwing error
if 'email' in event['config'] and event['config']['email']:
try:
send_notification_email(
event['_tibanna']['settings']['run_name'], event['jobid'],
ff_meta.run_status, event['_tibanna']['settings']['url'])
except Exception as e:
printlog("Cannot send email: %s" % e)
raise Exception(event.get('error'))
metadata_only = event.get('metadata_only', False)
pf_meta = [ProcessedFileMetadata(**pf) for pf in event.get('pf_meta')]
custom_qc_fields = event.get('custom_qc_fields', None)
# ensure this bad boy is always initialized
awsem = Awsem(event)
# go through this and replace awsemfile_report with awsf format
# actually interface should be look through ff_meta files and call
# give me the status of this thing from the runner, and runner.output_files.length
# so we just build a runner with interface to sbg and awsem
# runner.output_files.length()
# runner.output_files.file.status
# runner.output_files.file.loc
# runner.output_files.file.get
awsem_output = awsem.output_files()
awsem_output_extra = awsem.secondary_output_files()
ff_output = len(ff_meta.output_files)
if len(awsem_output) != ff_output:
ff_meta.run_status = 'error'
ff_meta.description = "%d files output expected %s" % (
ff_output, len(awsem_output))
ff_meta.patch(key=tibanna.ff_keys)
raise Exception(
"Failing the workflow because outputed files = %d and ffmeta = %d"
% (awsem_output, ff_output))
def update_metadata_from_awsemfile_list(awsemfile_list):
patch_meta = False
for awsemfile in awsemfile_list:
patch_meta = update_ffmeta_from_awsemfile(awsemfile, ff_meta,
tibanna,
custom_qc_fields)
if not metadata_only:
update_pfmeta_from_awsemfile(awsemfile, pf_meta, tibanna)
# allow for a simple way for updater to add appropriate meta_data
if patch_meta:
ff_meta.__dict__.update(patch_meta)
update_metadata_from_awsemfile_list(awsem_output)
update_metadata_from_awsemfile_list(awsem_output_extra)
# if we got all the awsemfiles let's go ahead and update our ff_metadata object
ff_meta.run_status = "complete"
# add postrunjson log file to ff_meta as a url
ff_meta.awsem_postrun_json = get_postrunjson_url(event)
# make all the file awsemfile meta-data stuff here
# TODO: fix bugs with ff_meta mapping for output and input file
try:
ff_meta.patch(key=tibanna.ff_keys)
except Exception as e:
raise Exception("Failed to update run_status %s" % str(e))
# patch processed files - update only status, extra_files, md5sum and file_size
if pf_meta:
patch_fields = [
'uuid', 'status', 'extra_files', 'md5sum', 'file_size',
'higlass_uid'
]
try:
for pf in pf_meta:
printlog(pf.as_dict())
pf.patch(key=tibanna.ff_keys, fields=patch_fields)
except Exception as e:
raise Exception("Failed to update processed metadata %s" % str(e))
event['ff_meta'] = ff_meta.as_dict()
event['pf_meta'] = [_.as_dict() for _ in pf_meta]
# sending a notification email after the job finishes
if 'email' in event['config'] and event['config']['email']:
try:
send_notification_email(event['_tibanna']['settings']['run_name'],
event['jobid'],
event['ff_meta']['run_status'],
event['_tibanna']['settings']['url'])
except Exception as e:
printlog("Cannot send email: %s" % e)
return event
