def test_generic_itemdata_too_few(tmp_path): tag = "foo" num_components = 1 map_options = htmap.MapOptions(stooge=["larry", "moe"], ) with pytest.raises(htmap.exceptions.MisalignedInputData) as exc_info: create_submit_object_and_itemdata( tag, tmp_path, num_components, map_options, ) assert "stooge" in exception_msg(exc_info)
def test_fewer_input_files_than_components(tmp_path): tag = "foo" num_components = 5 map_options = htmap.MapOptions(input_files=[["foo.txt"]], ) with pytest.raises(htmap.exceptions.MisalignedInputData) as exc_info: create_submit_object_and_itemdata( tag, tmp_path, num_components, map_options, ) assert "input_files" in exception_msg(exc_info)
def test_fewer_components_than_input_files(tmp_path): tag = 'foo' num_components = 1 map_options = htmap.MapOptions(input_files=[['foo.txt'], ['bar.txt'], ['buz.txt']], ) with pytest.raises(htmap.exceptions.MisalignedInputData) as exc_info: create_submit_object_and_itemdata( tag, tmp_path, num_components, map_options, ) assert 'input_files' in exception_msg(exc_info)
def test_list_of_transfer_path_input_files(tmp_path): tag = "foo" num_components = 3 map_options = htmap.MapOptions(input_files=[ htmap.TransferPath("foo.txt", protocol="file"), htmap.TransferPath("bar.txt", protocol="s3", location="s3.server.com"), htmap.TransferPath("buz.txt"), ], ) sub, itemdata = create_submit_object_and_itemdata( tag, tmp_path, num_components, map_options, ) expected = [ { "component": "0", "extra_input_files": "file:///foo.txt" }, { "component": "1", "extra_input_files": "s3://s3.server.com/bar.txt" }, { "component": "2", "extra_input_files": (Path.cwd() / "buz.txt").absolute().as_posix(), }, ] assert itemdata == expected
def test_list_of_list_of_path_input_files(): tag = 'foo' map_dir = Path().cwd() num_components = 3 map_options = htmap.MapOptions( input_files = [ [Path('foo.txt'), Path('foo2.txt')], [Path('bar.txt'), Path('bar2.txt')], [Path('buz.txt'), Path('buz2.txt')], ], ) sub, itemdata = create_submit_object_and_itemdata( tag, map_dir, num_components, map_options, ) expected = [ {'component': '0', 'extra_input_files': f"{Path('foo.txt').absolute().as_posix()}, {Path('foo2.txt').absolute().as_posix()}"}, {'component': '1', 'extra_input_files': f"{Path('bar.txt').absolute().as_posix()}, {Path('bar2.txt').absolute().as_posix()}"}, {'component': '2', 'extra_input_files': f"{Path('buz.txt').absolute().as_posix()}, {Path('buz2.txt').absolute().as_posix()}"}, ] assert itemdata == expected
def test_generic_itemdata(tmp_path): tag = "foo" num_components = 3 map_options = htmap.MapOptions(stooge=["larry", "moe", "curly"], ) sub, itemdata = create_submit_object_and_itemdata( tag, tmp_path, num_components, map_options, ) expected = [ { "component": "0", "itemdata_for_stooge": "larry" }, { "component": "1", "itemdata_for_stooge": "moe" }, { "component": "2", "itemdata_for_stooge": "curly" }, ] assert itemdata == expected
def test_generic_itemdata(tmp_path): tag = 'foo' num_components = 3 map_options = htmap.MapOptions(stooge=['larry', 'moe', 'curly'], ) sub, itemdata = create_submit_object_and_itemdata( tag, tmp_path, num_components, map_options, ) expected = [ { 'component': '0', 'itemdata_for_stooge': 'larry' }, { 'component': '1', 'itemdata_for_stooge': 'moe' }, { 'component': '2', 'itemdata_for_stooge': 'curly' }, ] assert itemdata == expected
def test_list_of_path_input_files(tmp_path): tag = "foo" num_components = 3 map_options = htmap.MapOptions( input_files=[Path("foo.txt"), Path("bar.txt"), Path("buz.txt")], ) sub, itemdata = create_submit_object_and_itemdata( tag, tmp_path, num_components, map_options, ) expected = [ { "component": "0", "extra_input_files": Path("foo.txt").absolute().as_posix() }, { "component": "1", "extra_input_files": Path("bar.txt").absolute().as_posix() }, { "component": "2", "extra_input_files": Path("buz.txt").absolute().as_posix() }, ] assert itemdata == expected
def test_list_of_path_input_files(tmp_path): tag = 'foo' num_components = 3 map_options = htmap.MapOptions( input_files=[Path('foo.txt'), Path('bar.txt'), Path('buz.txt')], ) sub, itemdata = create_submit_object_and_itemdata( tag, tmp_path, num_components, map_options, ) expected = [ { 'component': '0', 'extra_input_files': Path('foo.txt').absolute().as_posix() }, { 'component': '1', 'extra_input_files': Path('bar.txt').absolute().as_posix() }, { 'component': '2', 'extra_input_files': Path('buz.txt').absolute().as_posix() }, ] assert itemdata == expected
def test_generic_itemdata_too_few(): tag = 'foo' map_dir = Path().cwd() num_components = 1 map_options = htmap.MapOptions( stooge = ['larry', 'moe'], ) with pytest.raises(htmap.exceptions.MisalignedInputData) as exc_info: create_submit_object_and_itemdata( tag, map_dir, num_components, map_options, ) assert 'stooge' in exception_msg(exc_info)
def test_fewer_input_files_than_components(): tag = 'foo' map_dir = Path().cwd() num_components = 5 map_options = htmap.MapOptions( input_files = [['foo.txt']], ) with pytest.raises(htmap.exceptions.MisalignedInputData) as exc_info: create_submit_object_and_itemdata( tag, map_dir, num_components, map_options, ) assert 'input_files' in exception_msg(exc_info)
def test_single_shared_input_file(tmp_path): tag = 'foo' num_components = 1 map_options = htmap.MapOptions(fixed_input_files=['foo.txt'], ) sub, itemdata = create_submit_object_and_itemdata( tag, tmp_path, num_components, map_options, ) assert 'foo.txt' in sub['transfer_input_files']
def test_single_shared_input_file_can_be_single_str(tmp_path): tag = "foo" num_components = 1 map_options = htmap.MapOptions(fixed_input_files="foo.txt", ) sub, itemdata = create_submit_object_and_itemdata( tag, tmp_path, num_components, map_options, ) assert "foo.txt" in sub["transfer_input_files"]
def test_custom_options(key, tmp_path): tag = "test" num_components = 1 map_options = htmap.MapOptions(custom_options={key: "bar"}, ) sub, itemdata = create_submit_object_and_itemdata( tag, tmp_path, num_components, map_options, ) assert "MY.foo" in sub.keys()
def test_url_in_fixed_input_files(tmp_path): tag = 'foo' num_components = 1 url = 'http://www.baz.test' map_options = htmap.MapOptions(fixed_input_files=[url], ) sub, itemdata = create_submit_object_and_itemdata( tag, tmp_path, num_components, map_options, ) assert url in sub['transfer_input_files']
def test_url_in_input_files(tmp_path): tag = "foo" num_components = 1 url = "http://www.baz.test" map_options = htmap.MapOptions(input_files=[url], ) sub, itemdata = create_submit_object_and_itemdata( tag, tmp_path, num_components, map_options, ) assert url in itemdata[0]["extra_input_files"]
def test_custom_options(key): tag = 'test' map_dir = Path().cwd() num_components = 1 map_options = htmap.MapOptions( custom_options = {key: 'bar'}, ) sub, itemdata = create_submit_object_and_itemdata( tag, map_dir, num_components, map_options, ) assert '+foo' in sub.keys()
def test_two_urls_in_input_files(tmp_path): tag = "foo" num_components = 1 url_1 = "http://www.baz.test" url_2 = "http://www.bong.test" map_options = htmap.MapOptions(input_files=[(url_1, url_2)], ) sub, itemdata = create_submit_object_and_itemdata( tag, tmp_path, num_components, map_options, ) assert url_1 in itemdata[0]["extra_input_files"] assert url_2 in itemdata[0]["extra_input_files"]
def test_single_shared_input_file_can_be_single_str(): tag = 'foo' map_dir = Path().cwd() num_components = 1 map_options = htmap.MapOptions( fixed_input_files = 'foo.txt', ) sub, itemdata = create_submit_object_and_itemdata( tag, map_dir, num_components, map_options, ) assert 'foo.txt' in sub['transfer_input_files']
def test_url_in_input_files(): tag = 'foo' map_dir = Path().cwd() num_components = 1 url = 'http://www.baz.test' map_options = htmap.MapOptions( input_files = [url], ) sub, itemdata = create_submit_object_and_itemdata( tag, map_dir, num_components, map_options, ) assert url in itemdata[0]['extra_input_files']
def test_list_request_disk(rd): tag = 'foo' map_dir = Path().cwd() num_components = 1 map_options = htmap.MapOptions( request_disk = rd, ) sub, itemdata = create_submit_object_and_itemdata( tag, map_dir, num_components, map_options, ) expected = [ {'component': '0', 'itemdata_for_request_disk': '239GB'}, ] assert itemdata == expected
def test_list_request_disk(tmp_path): tag = 'foo' num_components = 2 map_options = htmap.MapOptions(request_disk=['239MB', '136MB'], ) sub, itemdata = create_submit_object_and_itemdata( tag, tmp_path, num_components, map_options, ) expected = [ { 'component': '0', 'itemdata_for_request_disk': '239MB' }, { 'component': '1', 'itemdata_for_request_disk': '136MB' }, ] assert itemdata == expected
def test_list_request_disk(tmp_path): tag = "foo" num_components = 2 map_options = htmap.MapOptions(request_disk=["239MB", "136MB"], ) sub, itemdata = create_submit_object_and_itemdata( tag, tmp_path, num_components, map_options, ) expected = [ { "component": "0", "itemdata_for_request_disk": "239MB" }, { "component": "1", "itemdata_for_request_disk": "136MB" }, ] assert itemdata == expected