def test_generic_itemdata_too_few(tmp_path):
tag = "foo"
num_components = 1
map_options = htmap.MapOptions(stooge=["larry", "moe"], )
with pytest.raises(htmap.exceptions.MisalignedInputData) as exc_info:
create_submit_object_and_itemdata(
tag,
tmp_path,
num_components,
map_options,
)
assert "stooge" in exception_msg(exc_info)
def test_fewer_input_files_than_components(tmp_path):
tag = "foo"
num_components = 5
map_options = htmap.MapOptions(input_files=[["foo.txt"]], )
with pytest.raises(htmap.exceptions.MisalignedInputData) as exc_info:
create_submit_object_and_itemdata(
tag,
tmp_path,
num_components,
map_options,
)
assert "input_files" in exception_msg(exc_info)
def test_fewer_components_than_input_files(tmp_path):
tag = 'foo'
num_components = 1
map_options = htmap.MapOptions(input_files=[['foo.txt'], ['bar.txt'],
['buz.txt']], )
with pytest.raises(htmap.exceptions.MisalignedInputData) as exc_info:
create_submit_object_and_itemdata(
tag,
tmp_path,
num_components,
map_options,
)
assert 'input_files' in exception_msg(exc_info)
def test_list_of_transfer_path_input_files(tmp_path):
tag = "foo"
num_components = 3
map_options = htmap.MapOptions(input_files=[
htmap.TransferPath("foo.txt", protocol="file"),
htmap.TransferPath("bar.txt", protocol="s3", location="s3.server.com"),
htmap.TransferPath("buz.txt"),
], )
sub, itemdata = create_submit_object_and_itemdata(
tag,
tmp_path,
num_components,
map_options,
)
expected = [
{
"component": "0",
"extra_input_files": "file:///foo.txt"
},
{
"component": "1",
"extra_input_files": "s3://s3.server.com/bar.txt"
},
{
"component": "2",
"extra_input_files":
(Path.cwd() / "buz.txt").absolute().as_posix(),
},
]
assert itemdata == expected
def test_list_of_list_of_path_input_files():
tag = 'foo'
map_dir = Path().cwd()
num_components = 3
map_options = htmap.MapOptions(
input_files = [
[Path('foo.txt'), Path('foo2.txt')],
[Path('bar.txt'), Path('bar2.txt')],
[Path('buz.txt'), Path('buz2.txt')],
],
)
sub, itemdata = create_submit_object_and_itemdata(
tag,
map_dir,
num_components,
map_options,
)
expected = [
{'component': '0', 'extra_input_files': f"{Path('foo.txt').absolute().as_posix()}, {Path('foo2.txt').absolute().as_posix()}"},
{'component': '1', 'extra_input_files': f"{Path('bar.txt').absolute().as_posix()}, {Path('bar2.txt').absolute().as_posix()}"},
{'component': '2', 'extra_input_files': f"{Path('buz.txt').absolute().as_posix()}, {Path('buz2.txt').absolute().as_posix()}"},
]
assert itemdata == expected
def test_generic_itemdata(tmp_path):
tag = "foo"
num_components = 3
map_options = htmap.MapOptions(stooge=["larry", "moe", "curly"], )
sub, itemdata = create_submit_object_and_itemdata(
tag,
tmp_path,
num_components,
map_options,
)
expected = [
{
"component": "0",
"itemdata_for_stooge": "larry"
},
{
"component": "1",
"itemdata_for_stooge": "moe"
},
{
"component": "2",
"itemdata_for_stooge": "curly"
},
]
assert itemdata == expected
def test_generic_itemdata(tmp_path):
tag = 'foo'
num_components = 3
map_options = htmap.MapOptions(stooge=['larry', 'moe', 'curly'], )
sub, itemdata = create_submit_object_and_itemdata(
tag,
tmp_path,
num_components,
map_options,
)
expected = [
{
'component': '0',
'itemdata_for_stooge': 'larry'
},
{
'component': '1',
'itemdata_for_stooge': 'moe'
},
{
'component': '2',
'itemdata_for_stooge': 'curly'
},
]
assert itemdata == expected
def test_list_of_path_input_files(tmp_path):
tag = "foo"
num_components = 3
map_options = htmap.MapOptions(
input_files=[Path("foo.txt"),
Path("bar.txt"),
Path("buz.txt")], )
sub, itemdata = create_submit_object_and_itemdata(
tag,
tmp_path,
num_components,
map_options,
)
expected = [
{
"component": "0",
"extra_input_files": Path("foo.txt").absolute().as_posix()
},
{
"component": "1",
"extra_input_files": Path("bar.txt").absolute().as_posix()
},
{
"component": "2",
"extra_input_files": Path("buz.txt").absolute().as_posix()
},
]
assert itemdata == expected
def test_list_of_path_input_files(tmp_path):
tag = 'foo'
num_components = 3
map_options = htmap.MapOptions(
input_files=[Path('foo.txt'),
Path('bar.txt'),
Path('buz.txt')], )
sub, itemdata = create_submit_object_and_itemdata(
tag,
tmp_path,
num_components,
map_options,
)
expected = [
{
'component': '0',
'extra_input_files': Path('foo.txt').absolute().as_posix()
},
{
'component': '1',
'extra_input_files': Path('bar.txt').absolute().as_posix()
},
{
'component': '2',
'extra_input_files': Path('buz.txt').absolute().as_posix()
},
]
assert itemdata == expected
def test_generic_itemdata_too_few():
tag = 'foo'
map_dir = Path().cwd()
num_components = 1
map_options = htmap.MapOptions(
stooge = ['larry', 'moe'],
)
with pytest.raises(htmap.exceptions.MisalignedInputData) as exc_info:
create_submit_object_and_itemdata(
tag,
map_dir,
num_components,
map_options,
)
assert 'stooge' in exception_msg(exc_info)
def test_fewer_input_files_than_components():
tag = 'foo'
map_dir = Path().cwd()
num_components = 5
map_options = htmap.MapOptions(
input_files = [['foo.txt']],
)
with pytest.raises(htmap.exceptions.MisalignedInputData) as exc_info:
create_submit_object_and_itemdata(
tag,
map_dir,
num_components,
map_options,
)
assert 'input_files' in exception_msg(exc_info)
def test_single_shared_input_file(tmp_path):
tag = 'foo'
num_components = 1
map_options = htmap.MapOptions(fixed_input_files=['foo.txt'], )
sub, itemdata = create_submit_object_and_itemdata(
tag,
tmp_path,
num_components,
map_options,
)
assert 'foo.txt' in sub['transfer_input_files']
def test_single_shared_input_file_can_be_single_str(tmp_path):
tag = "foo"
num_components = 1
map_options = htmap.MapOptions(fixed_input_files="foo.txt", )
sub, itemdata = create_submit_object_and_itemdata(
tag,
tmp_path,
num_components,
map_options,
)
assert "foo.txt" in sub["transfer_input_files"]
def test_custom_options(key, tmp_path):
tag = "test"
num_components = 1
map_options = htmap.MapOptions(custom_options={key: "bar"}, )
sub, itemdata = create_submit_object_and_itemdata(
tag,
tmp_path,
num_components,
map_options,
)
assert "MY.foo" in sub.keys()
def test_url_in_fixed_input_files(tmp_path):
tag = 'foo'
num_components = 1
url = 'http://www.baz.test'
map_options = htmap.MapOptions(fixed_input_files=[url], )
sub, itemdata = create_submit_object_and_itemdata(
tag,
tmp_path,
num_components,
map_options,
)
assert url in sub['transfer_input_files']
def test_url_in_input_files(tmp_path):
tag = "foo"
num_components = 1
url = "http://www.baz.test"
map_options = htmap.MapOptions(input_files=[url], )
sub, itemdata = create_submit_object_and_itemdata(
tag,
tmp_path,
num_components,
map_options,
)
assert url in itemdata[0]["extra_input_files"]
def test_custom_options(key):
tag = 'test'
map_dir = Path().cwd()
num_components = 1
map_options = htmap.MapOptions(
custom_options = {key: 'bar'},
)
sub, itemdata = create_submit_object_and_itemdata(
tag,
map_dir,
num_components,
map_options,
)
assert '+foo' in sub.keys()
def test_two_urls_in_input_files(tmp_path):
tag = "foo"
num_components = 1
url_1 = "http://www.baz.test"
url_2 = "http://www.bong.test"
map_options = htmap.MapOptions(input_files=[(url_1, url_2)], )
sub, itemdata = create_submit_object_and_itemdata(
tag,
tmp_path,
num_components,
map_options,
)
assert url_1 in itemdata[0]["extra_input_files"]
assert url_2 in itemdata[0]["extra_input_files"]
def test_single_shared_input_file_can_be_single_str():
tag = 'foo'
map_dir = Path().cwd()
num_components = 1
map_options = htmap.MapOptions(
fixed_input_files = 'foo.txt',
)
sub, itemdata = create_submit_object_and_itemdata(
tag,
map_dir,
num_components,
map_options,
)
assert 'foo.txt' in sub['transfer_input_files']
def test_url_in_input_files():
tag = 'foo'
map_dir = Path().cwd()
num_components = 1
url = 'http://www.baz.test'
map_options = htmap.MapOptions(
input_files = [url],
)
sub, itemdata = create_submit_object_and_itemdata(
tag,
map_dir,
num_components,
map_options,
)
assert url in itemdata[0]['extra_input_files']
def test_list_request_disk(rd):
tag = 'foo'
map_dir = Path().cwd()
num_components = 1
map_options = htmap.MapOptions(
request_disk = rd,
)
sub, itemdata = create_submit_object_and_itemdata(
tag,
map_dir,
num_components,
map_options,
)
expected = [
{'component': '0', 'itemdata_for_request_disk': '239GB'},
]
assert itemdata == expected
def test_list_request_disk(tmp_path):
tag = 'foo'
num_components = 2
map_options = htmap.MapOptions(request_disk=['239MB', '136MB'], )
sub, itemdata = create_submit_object_and_itemdata(
tag,
tmp_path,
num_components,
map_options,
)
expected = [
{
'component': '0',
'itemdata_for_request_disk': '239MB'
},
{
'component': '1',
'itemdata_for_request_disk': '136MB'
},
]
assert itemdata == expected
def test_list_request_disk(tmp_path):
tag = "foo"
num_components = 2
map_options = htmap.MapOptions(request_disk=["239MB", "136MB"], )
sub, itemdata = create_submit_object_and_itemdata(
tag,
tmp_path,
num_components,
map_options,
)
expected = [
{
"component": "0",
"itemdata_for_request_disk": "239MB"
},
{
"component": "1",
"itemdata_for_request_disk": "136MB"
},
]
assert itemdata == expected
