def test_list_of_transfer_path_input_files(tmp_path):
tag = "foo"
num_components = 3
map_options = htmap.MapOptions(input_files=[
htmap.TransferPath("foo.txt", protocol="file"),
htmap.TransferPath("bar.txt", protocol="s3", location="s3.server.com"),
htmap.TransferPath("buz.txt"),
], )
sub, itemdata = create_submit_object_and_itemdata(
tag,
tmp_path,
num_components,
map_options,
)
expected = [
{
"component": "0",
"extra_input_files": "file:///foo.txt"
},
{
"component": "1",
"extra_input_files": "s3://s3.server.com/bar.txt"
},
{
"component": "2",
"extra_input_files":
(Path.cwd() / "buz.txt").absolute().as_posix(),
},
]
assert itemdata == expected
def test_multiple_paths_in_args(tmp_path):
paths = [
htmap.TransferPath(tmp_path / "test-1.txt"),
htmap.TransferPath(tmp_path / "test-2.txt"),
htmap.TransferPath(tmp_path / "test-3.txt"),
]
for p in paths:
p.touch()
@htmap.mapped
def func(a, b, c):
return all(p.name in cwd_names() for p in (a, b, c))
m = func.starmap(args=[paths])
assert m.get(0)
def test_output_remap_via_file_protocol(tmp_path):
target = tmp_path / "foo"
destination = htmap.TransferPath(target, protocol="file")
def func(_):
output = Path("remote-foo")
output.write_text("hi")
htmap.transfer_output_files(output)
return True
m = htmap.map(
func, [None], map_options=htmap.MapOptions(output_remaps={"remote-foo": destination}),
)
print(m.stdout.get(0))
print()
print(m.stderr.get(0))
print()
# Looking at the starter log may be helpful if the plugin is failing badly enough.
# You must set STARTER_LOG_NAME_APPEND=jobid in your HTCondor config for it to work
# starter_log = Path(htcondor.param['LOG']) / f"StarterLog.{m._cluster_ids[0]}.0"
# print(starter_log.read_text())
assert m.get(0)
assert target.read_text() == "hi"
# make sure we did NOT transfer it back as normal
assert "foo" not in (path.stem for path in m.output_files[0].iterdir())
def func(_):
output = Path("remote-foo")
output.write_text("hi")
destination = htmap.TransferPath(target, protocol="file")
htmap.transfer_output_files((output, destination))
return True
def test_path_in_nested_dict(tmp_path):
@htmap.mapped
def func(dict_of_dict_of_paths):
return all(
all(p.name in cwd_names() for p in dict_of_paths.values())
for dict_of_paths in dict_of_dict_of_paths.values())
paths = [
htmap.TransferPath(tmp_path / "test-1.txt"),
htmap.TransferPath(tmp_path / "test-2.txt"),
htmap.TransferPath(tmp_path / "test-3.txt"),
]
for p in paths:
p.touch()
m = func.map(args=[{"paths": dict(zip(range(len(paths)), paths))}])
assert m.get(0)
def test_path_in_nested_list(tmp_path):
@htmap.mapped
def func(list_of_list_of_paths):
return all(
all(p.name in cwd_names() for p in list_of_paths)
for list_of_paths in list_of_list_of_paths)
paths = [
htmap.TransferPath(tmp_path / "test-1.txt"),
htmap.TransferPath(tmp_path / "test-2.txt"),
htmap.TransferPath(tmp_path / "test-3.txt"),
]
for p in paths:
p.touch()
m = func.map(args=[[paths]])
assert m.get(0)
def test_input_transfer_via_https_protocol(tmp_path):
f = htmap.TransferPath("status/200", protocol="https", location="httpbin.org")
def test(file):
return file.read_text() == ""
m = htmap.map(test, [f])
assert m.get(0)
def test_input_transfer_via_file_protocol(tmp_path):
f = htmap.TransferPath(__file__, protocol="file")
MARKER = 12345
def test(file):
return "MARKER = 12345" in file.read_text()
m = htmap.map(test, [f])
assert m.get(0)
def test_transfer_directory(tmp_path):
dir = htmap.TransferPath(tmp_path / "dir")
dir.mkdir()
file_in_dir = dir / "file"
file_in_dir.write_text("hi")
def test(dir):
return (dir / "file").read_text() == "hi"
m = htmap.map(test, [dir])
assert m.get(0)
def transfer_path(tmp_path):
p = htmap.TransferPath(tmp_path / TEST_FILE_NAME)
p.touch()
return p
def test_transfer_path_to_path():
assert isinstance(htmap.TransferPath(Path('foobar')), htmap.TransferPath)
