def test_get_extra_file_key(start_run_md5_data):
data = {'env': 'fourfront-webdev', 'settings': {'1': '1'}}
tibanna = TibannaSettings(**data)
fe_map = FormatExtensionMap(tibanna.ff_keys)
fe_map = FormatExtensionMap(tibanna.ff_keys)
infile_key = 'hahaha/lalala.bedGraph.gz'
infile_format = 'bg'
extra_file_format = 'bw'
extra_file_key = get_extra_file_key(infile_format, infile_key,
extra_file_format, fe_map)
assert extra_file_key == 'hahaha/lalala.bw'
def test_FormatExtensionMap_no_ffkey(realistic_format_search_result):
fe_map = FormatExtensionMap(ffe_all=realistic_format_search_result)
assert hasattr(fe_map, 'fe_dict')
assert fe_map.fe_dict == {'bam':{'standard_extension': 'bam',
'other_allowed_extensions': [],
'extrafile_formats': ['/file-formats/bai/']},
'bai':{'standard_extension': 'bam.bai',
'extrafile_formats': [],
'other_allowed_extensions': []}}
assert fe_map.get_extension('bam') == 'bam'
assert fe_map.get_other_extensions('bai') == []
assert fe_map.get_extension('fastq') is None
def test_FormatExtensionMap2(realistic_format_search_result):
with mock.patch("tibanna_ffcommon.file_format.search_metadata", return_value=realistic_format_search_result):
fe_map = FormatExtensionMap({'server': 'some_server'})
assert hasattr(fe_map, 'fe_dict')
assert fe_map.fe_dict == {'bam':{'standard_extension': 'bam',
'other_allowed_extensions': [],
'extrafile_formats': ['/file-formats/bai/']},
'bai':{'standard_extension': 'bam.bai',
'extrafile_formats': [],
'other_allowed_extensions': []}}
assert fe_map.get_extension('bam') == 'bam'
assert fe_map.get_other_extensions('bai') == []
assert fe_map.get_extension('fastq') is None
def test_FFInputFile_create_unicorn_arg_secondary_file_list(
minimal_file_metadata_w_extrafile,
minimal_file_metadata_w_extrafile_not_ready2,
fake_format_search_result):
ffinpf = FFInputFile(uuid=['someuuid', 'someuuid2'],
workflow_argument_name='somearg',
rename=['somename1.bwt', 'somename2.bwt'],
mount=True,
ff_env='data')
# cache a fake metadata for data without 4dn open data url
ffinpf._metadata['someuuid'] = minimal_file_metadata_w_extrafile
ffinpf._metadata[
'someuuid2'] = minimal_file_metadata_w_extrafile_not_ready2
ffinpf._fe_map = FormatExtensionMap(ffe_all=fake_format_search_result)
res = ffinpf.create_unicorn_arg_secondary_file()
assert res == {
'somearg': {
'bucket_name':
'elasticbeanstalk-fourfront-webprod-files',
'object_key': [['someuuid/someacc.sa', 'someuuid/someacc.ann'],
['someuuid2/someacc2.ann']],
'mount':
True,
'rename': [['somename1.sa', 'somename1.ann'], ['somename2.ann']]
}
}
def test_FFInputFile_list_4dn_opendata_w_extrafile(
minimal_file_metadata_4dn_opendata_w_extrafile,
minimal_file_metadata_4dn_opendata_w_extrafile2,
fake_format_search_result):
ffinpf = FFInputFile(uuid=['someuuid', 'someuuid2'],
workflow_argument_name='somearg',
rename=['somename.bwt', 'somename2.bwt'])
# cache a fake metadata for both one with 4dn open data url and one without
# This should be a problem since all files for a single argument should be
# from the same bucket.
ffinpf._metadata[
'someuuid'] = minimal_file_metadata_4dn_opendata_w_extrafile
ffinpf._metadata[
'someuuid2'] = minimal_file_metadata_4dn_opendata_w_extrafile2
ffinpf.ff_env = 'data'
ffinpf._fe_map = FormatExtensionMap(ffe_all=fake_format_search_result)
assert ffinpf.bucket_name == '4dn-open-data-public'
assert ffinpf.extra_file_s3_keys == [
[
'fourfront-webprod/files/someuuid/someacc.sa',
'fourfront-webprod/files/someuuid/someacc.ann'
],
[
'fourfront-webprod/files/someuuid2/someacc2.sa',
'fourfront-webprod/files/someuuid2/someacc2.ann'
]
]
assert ffinpf.extra_file_renames == [['somename.sa', 'somename.ann'],
['somename2.sa', 'somename2.ann']]
def test_get_extra_file_key(fake_format_search_result):
fe_map = FormatExtensionMap(ffe_all=fake_format_search_result)
infile_key = 'hahaha/lalala.bwt'
infile_format = 'bwt'
extra_file_format = 'sa'
extra_file_key = get_extra_file_key(infile_format, infile_key,
extra_file_format, fe_map)
assert extra_file_key == 'hahaha/lalala.sa'
def test_FFInputFile_create_unicorn_arg_secondary_file_format_if_extra(
minimal_file_metadata_w_extrafile, fake_format_search_result):
ffinpf = FFInputFile(uuid='someuuid',
workflow_argument_name='somearg',
ff_env='data',
format_if_extra='ann')
# cache a fake metadata for data without 4dn open data url
ffinpf._metadata['someuuid'] = minimal_file_metadata_w_extrafile
ffinpf._fe_map = FormatExtensionMap(ffe_all=fake_format_search_result)
assert not ffinpf.create_unicorn_arg_secondary_file()
def test_FFInputFile_w_extrafile_not_ready(
minimal_file_metadata_w_extrafile_not_ready,
fake_format_search_result):
ffinpf = FFInputFile(uuid='someuuid', workflow_argument_name='somearg')
# cache a fake metadata for data without 4dn open data url
ffinpf._metadata['someuuid'] = minimal_file_metadata_w_extrafile_not_ready
# bucket & object_key fill_in test for our 4dn bucket
ffinpf.ff_env = 'data'
ffinpf._fe_map = FormatExtensionMap(ffe_all=fake_format_search_result)
assert ffinpf.bucket_name == 'elasticbeanstalk-fourfront-webprod-files'
assert ffinpf.extra_file_s3_keys == 'someuuid/someacc.sa'
def test_FFInputFile_create_unicorn_arg_secondary_file(
minimal_file_metadata_w_extrafile, fake_format_search_result):
ffinpf = FFInputFile(uuid='someuuid',
workflow_argument_name='somearg',
ff_env='data')
# cache a fake metadata for data without 4dn open data url
ffinpf._metadata['someuuid'] = minimal_file_metadata_w_extrafile
ffinpf._fe_map = FormatExtensionMap(ffe_all=fake_format_search_result)
res = ffinpf.create_unicorn_arg_secondary_file()
assert res == {
'somearg': {
'bucket_name': 'elasticbeanstalk-fourfront-webprod-files',
'object_key': ['someuuid/someacc.sa',
'someuuid/someacc.ann'], # actual s3 key
'mount': False,
'rename': ''
}
}
def test_FFInputFile_4dn_opendata_w_extrafile(
minimal_file_metadata_4dn_opendata_w_extrafile,
fake_format_search_result):
ffinpf = FFInputFile(uuid='someuuid', workflow_argument_name='somearg')
# cache a fake metadata for data without 4dn open data url
ffinpf._metadata[
'someuuid'] = minimal_file_metadata_4dn_opendata_w_extrafile
# bucket & object_key fill_in test for our 4dn bucket
ffinpf.ff_env = 'data'
ffinpf._fe_map = FormatExtensionMap(ffe_all=fake_format_search_result)
assert ffinpf.get_extra_file_formats_from_uuid('someuuid') == ['sa', 'ann']
assert ffinpf.get_extra_file_s3_keys_from_uuid('someuuid') == [
'fourfront-webprod/files/someuuid/someacc.sa',
'fourfront-webprod/files/someuuid/someacc.ann'
]
assert ffinpf.bucket_name == '4dn-open-data-public'
assert ffinpf.extra_file_s3_keys == [
'fourfront-webprod/files/someuuid/someacc.sa',
'fourfront-webprod/files/someuuid/someacc.ann'
]
def test_FormatExtensionMap_neither_ffkey_ffe_all():
with pytest.raises(Exception) as ex_info:
fe_map = FormatExtensionMap()
assert "Either ff_keys or ffe_all must be specified" in str(ex_info.value)
def test_FormatExtensionMap_both_ffkey_ffe_all(realistic_format_search_result):
with pytest.raises(Exception) as ex_info:
fe_map = FormatExtensionMap({'server': 'some_server'}, ffe_all=realistic_format_search_result)
assert "Either ff_keys or ffe_all must be specified but not both" in str(ex_info.value)
def test_format_extension_map():
data = {'env': 'fourfront-webdev', 'settings': {'1': '1'}}
tibanna = TibannaSettings(**data)
fe_map = FormatExtensionMap(tibanna.ff_keys)
assert (fe_map)
assert 'bwt' in fe_map.fe_dict.keys()
