def test_merge_input_files_in_map_options():
a = htmap.MapOptions(input_files=["a", "b"])
b = htmap.MapOptions(input_files=["c", "d"])
merged = htmap.MapOptions.merge(a, b)
assert merged.input_files == ["a", "b"]
def test_merge_input_files_in_map_options():
a = htmap.MapOptions(input_files=['a', 'b'])
b = htmap.MapOptions(input_files=['c', 'd'])
merged = htmap.MapOptions.merge(a, b)
assert merged.input_files == ['a', 'b']
def test_merging_options_merges_fixed_input():
a = htmap.MapOptions(fixed_input_files="foo.txt", )
b = htmap.MapOptions(fixed_input_files="bar.txt", )
merged = htmap.MapOptions.merge(a, b)
assert set(merged.fixed_input_files) == {"foo.txt", "bar.txt"}
def test_merging_options_override():
a = htmap.MapOptions(foo='override', )
b = htmap.MapOptions(foo='hidden')
merged = htmap.MapOptions.merge(a, b)
assert merged['foo'] == 'override'
def test_merging_options_override():
a = htmap.MapOptions(foo="override", )
b = htmap.MapOptions(foo="hidden")
merged = htmap.MapOptions.merge(a, b)
assert merged["foo"] == "override"
def test_merging_options_side_by_side():
a = htmap.MapOptions(foo="me", )
b = htmap.MapOptions(bar="too")
merged = htmap.MapOptions.merge(a, b)
assert merged["foo"] == "me"
assert merged["bar"] == "too"
def test_merge_requirements_in_map_options():
a = htmap.MapOptions(requirements="foo")
b = htmap.MapOptions(requirements="bar")
merged = htmap.MapOptions.merge(a, b)
assert "foo" in merged["requirements"]
assert "bar" in merged["requirements"]
def test_merge_requirements_in_map_options():
a = htmap.MapOptions(requirements='foo')
b = htmap.MapOptions(requirements='bar')
merged = htmap.MapOptions.merge(a, b)
assert 'foo' in merged['requirements']
assert 'bar' in merged['requirements']
def test_merging_options_side_by_side():
a = htmap.MapOptions(foo='me', )
b = htmap.MapOptions(bar='too')
merged = htmap.MapOptions.merge(a, b)
assert merged['foo'] == 'me'
assert merged['bar'] == 'too'
def test_option_set_on_mapped_function_is_overridden():
@htmap.mapped(map_options=htmap.MapOptions(request_memory="123MB",))
def double(x):
return 2 * x
m = double.map(range(1), map_options=htmap.MapOptions(request_memory="456MB",))
sub = htio.load_submit(m._map_dir)
assert sub["request_memory"] == "456MB"
def test_list_of_path_input_files(tmp_path):
tag = "foo"
num_components = 3
map_options = htmap.MapOptions(
input_files=[Path("foo.txt"),
Path("bar.txt"),
Path("buz.txt")], )
sub, itemdata = create_submit_object_and_itemdata(
tag,
tmp_path,
num_components,
map_options,
)
expected = [
{
"component": "0",
"extra_input_files": Path("foo.txt").absolute().as_posix()
},
{
"component": "1",
"extra_input_files": Path("bar.txt").absolute().as_posix()
},
{
"component": "2",
"extra_input_files": Path("buz.txt").absolute().as_posix()
},
]
assert itemdata == expected
def test_list_of_list_of_path_input_files():
tag = 'foo'
map_dir = Path().cwd()
num_components = 3
map_options = htmap.MapOptions(
input_files = [
[Path('foo.txt'), Path('foo2.txt')],
[Path('bar.txt'), Path('bar2.txt')],
[Path('buz.txt'), Path('buz2.txt')],
],
)
sub, itemdata = create_submit_object_and_itemdata(
tag,
map_dir,
num_components,
map_options,
)
expected = [
{'component': '0', 'extra_input_files': f"{Path('foo.txt').absolute().as_posix()}, {Path('foo2.txt').absolute().as_posix()}"},
{'component': '1', 'extra_input_files': f"{Path('bar.txt').absolute().as_posix()}, {Path('bar2.txt').absolute().as_posix()}"},
{'component': '2', 'extra_input_files': f"{Path('buz.txt').absolute().as_posix()}, {Path('buz2.txt').absolute().as_posix()}"},
]
assert itemdata == expected
def test_output_remap_via_file_protocol(tmp_path):
target = tmp_path / "foo"
destination = htmap.TransferPath(target, protocol="file")
def func(_):
output = Path("remote-foo")
output.write_text("hi")
htmap.transfer_output_files(output)
return True
m = htmap.map(
func, [None], map_options=htmap.MapOptions(output_remaps={"remote-foo": destination}),
)
print(m.stdout.get(0))
print()
print(m.stderr.get(0))
print()
# Looking at the starter log may be helpful if the plugin is failing badly enough.
# You must set STARTER_LOG_NAME_APPEND=jobid in your HTCondor config for it to work
# starter_log = Path(htcondor.param['LOG']) / f"StarterLog.{m._cluster_ids[0]}.0"
# print(starter_log.read_text())
assert m.get(0)
assert target.read_text() == "hi"
# make sure we did NOT transfer it back as normal
assert "foo" not in (path.stem for path in m.output_files[0].iterdir())
def test_list_of_transfer_path_input_files(tmp_path):
tag = "foo"
num_components = 3
map_options = htmap.MapOptions(input_files=[
htmap.TransferPath("foo.txt", protocol="file"),
htmap.TransferPath("bar.txt", protocol="s3", location="s3.server.com"),
htmap.TransferPath("buz.txt"),
], )
sub, itemdata = create_submit_object_and_itemdata(
tag,
tmp_path,
num_components,
map_options,
)
expected = [
{
"component": "0",
"extra_input_files": "file:///foo.txt"
},
{
"component": "1",
"extra_input_files": "s3://s3.server.com/bar.txt"
},
{
"component": "2",
"extra_input_files":
(Path.cwd() / "buz.txt").absolute().as_posix(),
},
]
assert itemdata == expected
def test_generic_itemdata(tmp_path):
tag = 'foo'
num_components = 3
map_options = htmap.MapOptions(stooge=['larry', 'moe', 'curly'], )
sub, itemdata = create_submit_object_and_itemdata(
tag,
tmp_path,
num_components,
map_options,
)
expected = [
{
'component': '0',
'itemdata_for_stooge': 'larry'
},
{
'component': '1',
'itemdata_for_stooge': 'moe'
},
{
'component': '2',
'itemdata_for_stooge': 'curly'
},
]
assert itemdata == expected
def test_list_of_path_input_files(tmp_path):
tag = 'foo'
num_components = 3
map_options = htmap.MapOptions(
input_files=[Path('foo.txt'),
Path('bar.txt'),
Path('buz.txt')], )
sub, itemdata = create_submit_object_and_itemdata(
tag,
tmp_path,
num_components,
map_options,
)
expected = [
{
'component': '0',
'extra_input_files': Path('foo.txt').absolute().as_posix()
},
{
'component': '1',
'extra_input_files': Path('bar.txt').absolute().as_posix()
},
{
'component': '2',
'extra_input_files': Path('buz.txt').absolute().as_posix()
},
]
assert itemdata == expected
def late_noop():
@htmap.mapped(map_options=htmap.MapOptions(max_idle="1"))
def noop(_):
time.sleep(1)
return True
return noop
def test_generic_itemdata(tmp_path):
tag = "foo"
num_components = 3
map_options = htmap.MapOptions(stooge=["larry", "moe", "curly"], )
sub, itemdata = create_submit_object_and_itemdata(
tag,
tmp_path,
num_components,
map_options,
)
expected = [
{
"component": "0",
"itemdata_for_stooge": "larry"
},
{
"component": "1",
"itemdata_for_stooge": "moe"
},
{
"component": "2",
"itemdata_for_stooge": "curly"
},
]
assert itemdata == expected
def test_one_reserved_kwarg_raises():
with pytest.raises(htmap.exceptions.ReservedOptionKeyword) as exc_info:
htmap.MapOptions(
transfer_input_files = 'foobar',
)
assert 'keyword' in exception_msg(exc_info)
def test_two_reserved_kwarg_raises():
with pytest.raises(htmap.exceptions.ReservedOptionKeyword) as exc_info:
htmap.MapOptions(
transfer_input_files="foobar",
jobbatchname="pink",
)
assert "keywords" in exception_msg(exc_info)
def test_error_report_includes_input_files():
def dummy(x):
raise Exception
m = htmap.map(dummy, [0],
map_options=htmap.MapOptions(
fixed_input_files=Path(__file__), ))
err = m.get_err(0)
assert "test_error_handling.py" in err.report()
def test_option_set_on_mapped_function_is_inherited():
@htmap.mapped(map_options=htmap.MapOptions(request_memory='123MB', ))
def double(x):
return 2 * x
m = double.map(range(1))
sub = htio.load_submit(m._map_dir)
assert sub['request_memory'] == '123MB'
def test_single_shared_input_file_can_be_single_str(tmp_path):
tag = "foo"
num_components = 1
map_options = htmap.MapOptions(fixed_input_files="foo.txt", )
sub, itemdata = create_submit_object_and_itemdata(
tag,
tmp_path,
num_components,
map_options,
)
assert "foo.txt" in sub["transfer_input_files"]
def test_single_shared_input_file(tmp_path):
tag = 'foo'
num_components = 1
map_options = htmap.MapOptions(fixed_input_files=['foo.txt'], )
sub, itemdata = create_submit_object_and_itemdata(
tag,
tmp_path,
num_components,
map_options,
)
assert 'foo.txt' in sub['transfer_input_files']
def test_custom_options(key, tmp_path):
tag = "test"
num_components = 1
map_options = htmap.MapOptions(custom_options={key: "bar"}, )
sub, itemdata = create_submit_object_and_itemdata(
tag,
tmp_path,
num_components,
map_options,
)
assert "MY.foo" in sub.keys()
def test_url_in_fixed_input_files(tmp_path):
tag = 'foo'
num_components = 1
url = 'http://www.baz.test'
map_options = htmap.MapOptions(fixed_input_files=[url], )
sub, itemdata = create_submit_object_and_itemdata(
tag,
tmp_path,
num_components,
map_options,
)
assert url in sub['transfer_input_files']
def test_url_in_input_files(tmp_path):
tag = "foo"
num_components = 1
url = "http://www.baz.test"
map_options = htmap.MapOptions(input_files=[url], )
sub, itemdata = create_submit_object_and_itemdata(
tag,
tmp_path,
num_components,
map_options,
)
assert url in itemdata[0]["extra_input_files"]
def test_fewer_input_files_than_components(tmp_path):
tag = "foo"
num_components = 5
map_options = htmap.MapOptions(input_files=[["foo.txt"]], )
with pytest.raises(htmap.exceptions.MisalignedInputData) as exc_info:
create_submit_object_and_itemdata(
tag,
tmp_path,
num_components,
map_options,
)
assert "input_files" in exception_msg(exc_info)
def test_generic_itemdata_too_few(tmp_path):
tag = "foo"
num_components = 1
map_options = htmap.MapOptions(stooge=["larry", "moe"], )
with pytest.raises(htmap.exceptions.MisalignedInputData) as exc_info:
create_submit_object_and_itemdata(
tag,
tmp_path,
num_components,
map_options,
)
assert "stooge" in exception_msg(exc_info)
def test_fixed_input_files_are_transferred_as_list_of_paths(tmp_path):
f1 = tmp_path / "f1"
f2 = tmp_path / "f2"
f1.write_text("f1")
f2.write_text("f2")
def test(_):
return Path("f1").read_text() == "f1" and Path(
"f2").read_text() == "f2"
m = htmap.map(test, [None],
map_options=htmap.MapOptions(fixed_input_files=[f1, f2]))
assert m.get(0)