def test_get_extra_file_key(start_run_md5_data): data = {'env': 'fourfront-webdev', 'settings': {'1': '1'}} tibanna = TibannaSettings(**data) fe_map = FormatExtensionMap(tibanna.ff_keys) fe_map = FormatExtensionMap(tibanna.ff_keys) infile_key = 'hahaha/lalala.bedGraph.gz' infile_format = 'bg' extra_file_format = 'bw' extra_file_key = get_extra_file_key(infile_format, infile_key, extra_file_format, fe_map) assert extra_file_key == 'hahaha/lalala.bw'
def test_FormatExtensionMap_no_ffkey(realistic_format_search_result): fe_map = FormatExtensionMap(ffe_all=realistic_format_search_result) assert hasattr(fe_map, 'fe_dict') assert fe_map.fe_dict == {'bam':{'standard_extension': 'bam', 'other_allowed_extensions': [], 'extrafile_formats': ['/file-formats/bai/']}, 'bai':{'standard_extension': 'bam.bai', 'extrafile_formats': [], 'other_allowed_extensions': []}} assert fe_map.get_extension('bam') == 'bam' assert fe_map.get_other_extensions('bai') == [] assert fe_map.get_extension('fastq') is None
def test_FormatExtensionMap2(realistic_format_search_result): with mock.patch("tibanna_ffcommon.file_format.search_metadata", return_value=realistic_format_search_result): fe_map = FormatExtensionMap({'server': 'some_server'}) assert hasattr(fe_map, 'fe_dict') assert fe_map.fe_dict == {'bam':{'standard_extension': 'bam', 'other_allowed_extensions': [], 'extrafile_formats': ['/file-formats/bai/']}, 'bai':{'standard_extension': 'bam.bai', 'extrafile_formats': [], 'other_allowed_extensions': []}} assert fe_map.get_extension('bam') == 'bam' assert fe_map.get_other_extensions('bai') == [] assert fe_map.get_extension('fastq') is None
def test_FFInputFile_create_unicorn_arg_secondary_file_list( minimal_file_metadata_w_extrafile, minimal_file_metadata_w_extrafile_not_ready2, fake_format_search_result): ffinpf = FFInputFile(uuid=['someuuid', 'someuuid2'], workflow_argument_name='somearg', rename=['somename1.bwt', 'somename2.bwt'], mount=True, ff_env='data') # cache a fake metadata for data without 4dn open data url ffinpf._metadata['someuuid'] = minimal_file_metadata_w_extrafile ffinpf._metadata[ 'someuuid2'] = minimal_file_metadata_w_extrafile_not_ready2 ffinpf._fe_map = FormatExtensionMap(ffe_all=fake_format_search_result) res = ffinpf.create_unicorn_arg_secondary_file() assert res == { 'somearg': { 'bucket_name': 'elasticbeanstalk-fourfront-webprod-files', 'object_key': [['someuuid/someacc.sa', 'someuuid/someacc.ann'], ['someuuid2/someacc2.ann']], 'mount': True, 'rename': [['somename1.sa', 'somename1.ann'], ['somename2.ann']] } }
def test_FFInputFile_list_4dn_opendata_w_extrafile( minimal_file_metadata_4dn_opendata_w_extrafile, minimal_file_metadata_4dn_opendata_w_extrafile2, fake_format_search_result): ffinpf = FFInputFile(uuid=['someuuid', 'someuuid2'], workflow_argument_name='somearg', rename=['somename.bwt', 'somename2.bwt']) # cache a fake metadata for both one with 4dn open data url and one without # This should be a problem since all files for a single argument should be # from the same bucket. ffinpf._metadata[ 'someuuid'] = minimal_file_metadata_4dn_opendata_w_extrafile ffinpf._metadata[ 'someuuid2'] = minimal_file_metadata_4dn_opendata_w_extrafile2 ffinpf.ff_env = 'data' ffinpf._fe_map = FormatExtensionMap(ffe_all=fake_format_search_result) assert ffinpf.bucket_name == '4dn-open-data-public' assert ffinpf.extra_file_s3_keys == [ [ 'fourfront-webprod/files/someuuid/someacc.sa', 'fourfront-webprod/files/someuuid/someacc.ann' ], [ 'fourfront-webprod/files/someuuid2/someacc2.sa', 'fourfront-webprod/files/someuuid2/someacc2.ann' ] ] assert ffinpf.extra_file_renames == [['somename.sa', 'somename.ann'], ['somename2.sa', 'somename2.ann']]
def test_get_extra_file_key(fake_format_search_result): fe_map = FormatExtensionMap(ffe_all=fake_format_search_result) infile_key = 'hahaha/lalala.bwt' infile_format = 'bwt' extra_file_format = 'sa' extra_file_key = get_extra_file_key(infile_format, infile_key, extra_file_format, fe_map) assert extra_file_key == 'hahaha/lalala.sa'
def test_FFInputFile_create_unicorn_arg_secondary_file_format_if_extra( minimal_file_metadata_w_extrafile, fake_format_search_result): ffinpf = FFInputFile(uuid='someuuid', workflow_argument_name='somearg', ff_env='data', format_if_extra='ann') # cache a fake metadata for data without 4dn open data url ffinpf._metadata['someuuid'] = minimal_file_metadata_w_extrafile ffinpf._fe_map = FormatExtensionMap(ffe_all=fake_format_search_result) assert not ffinpf.create_unicorn_arg_secondary_file()
def test_FFInputFile_w_extrafile_not_ready( minimal_file_metadata_w_extrafile_not_ready, fake_format_search_result): ffinpf = FFInputFile(uuid='someuuid', workflow_argument_name='somearg') # cache a fake metadata for data without 4dn open data url ffinpf._metadata['someuuid'] = minimal_file_metadata_w_extrafile_not_ready # bucket & object_key fill_in test for our 4dn bucket ffinpf.ff_env = 'data' ffinpf._fe_map = FormatExtensionMap(ffe_all=fake_format_search_result) assert ffinpf.bucket_name == 'elasticbeanstalk-fourfront-webprod-files' assert ffinpf.extra_file_s3_keys == 'someuuid/someacc.sa'
def test_FFInputFile_create_unicorn_arg_secondary_file( minimal_file_metadata_w_extrafile, fake_format_search_result): ffinpf = FFInputFile(uuid='someuuid', workflow_argument_name='somearg', ff_env='data') # cache a fake metadata for data without 4dn open data url ffinpf._metadata['someuuid'] = minimal_file_metadata_w_extrafile ffinpf._fe_map = FormatExtensionMap(ffe_all=fake_format_search_result) res = ffinpf.create_unicorn_arg_secondary_file() assert res == { 'somearg': { 'bucket_name': 'elasticbeanstalk-fourfront-webprod-files', 'object_key': ['someuuid/someacc.sa', 'someuuid/someacc.ann'], # actual s3 key 'mount': False, 'rename': '' } }
def test_FFInputFile_4dn_opendata_w_extrafile( minimal_file_metadata_4dn_opendata_w_extrafile, fake_format_search_result): ffinpf = FFInputFile(uuid='someuuid', workflow_argument_name='somearg') # cache a fake metadata for data without 4dn open data url ffinpf._metadata[ 'someuuid'] = minimal_file_metadata_4dn_opendata_w_extrafile # bucket & object_key fill_in test for our 4dn bucket ffinpf.ff_env = 'data' ffinpf._fe_map = FormatExtensionMap(ffe_all=fake_format_search_result) assert ffinpf.get_extra_file_formats_from_uuid('someuuid') == ['sa', 'ann'] assert ffinpf.get_extra_file_s3_keys_from_uuid('someuuid') == [ 'fourfront-webprod/files/someuuid/someacc.sa', 'fourfront-webprod/files/someuuid/someacc.ann' ] assert ffinpf.bucket_name == '4dn-open-data-public' assert ffinpf.extra_file_s3_keys == [ 'fourfront-webprod/files/someuuid/someacc.sa', 'fourfront-webprod/files/someuuid/someacc.ann' ]
def test_FormatExtensionMap_neither_ffkey_ffe_all(): with pytest.raises(Exception) as ex_info: fe_map = FormatExtensionMap() assert "Either ff_keys or ffe_all must be specified" in str(ex_info.value)
def test_FormatExtensionMap_both_ffkey_ffe_all(realistic_format_search_result): with pytest.raises(Exception) as ex_info: fe_map = FormatExtensionMap({'server': 'some_server'}, ffe_all=realistic_format_search_result) assert "Either ff_keys or ffe_all must be specified but not both" in str(ex_info.value)
def test_format_extension_map(): data = {'env': 'fourfront-webdev', 'settings': {'1': '1'}} tibanna = TibannaSettings(**data) fe_map = FormatExtensionMap(tibanna.ff_keys) assert (fe_map) assert 'bwt' in fe_map.fe_dict.keys()