def test_list_of_transfer_path_input_files(tmp_path): tag = "foo" num_components = 3 map_options = htmap.MapOptions(input_files=[ htmap.TransferPath("foo.txt", protocol="file"), htmap.TransferPath("bar.txt", protocol="s3", location="s3.server.com"), htmap.TransferPath("buz.txt"), ], ) sub, itemdata = create_submit_object_and_itemdata( tag, tmp_path, num_components, map_options, ) expected = [ { "component": "0", "extra_input_files": "file:///foo.txt" }, { "component": "1", "extra_input_files": "s3://s3.server.com/bar.txt" }, { "component": "2", "extra_input_files": (Path.cwd() / "buz.txt").absolute().as_posix(), }, ] assert itemdata == expected
def test_multiple_paths_in_args(tmp_path): paths = [ htmap.TransferPath(tmp_path / "test-1.txt"), htmap.TransferPath(tmp_path / "test-2.txt"), htmap.TransferPath(tmp_path / "test-3.txt"), ] for p in paths: p.touch() @htmap.mapped def func(a, b, c): return all(p.name in cwd_names() for p in (a, b, c)) m = func.starmap(args=[paths]) assert m.get(0)
def test_output_remap_via_file_protocol(tmp_path): target = tmp_path / "foo" destination = htmap.TransferPath(target, protocol="file") def func(_): output = Path("remote-foo") output.write_text("hi") htmap.transfer_output_files(output) return True m = htmap.map( func, [None], map_options=htmap.MapOptions(output_remaps={"remote-foo": destination}), ) print(m.stdout.get(0)) print() print(m.stderr.get(0)) print() # Looking at the starter log may be helpful if the plugin is failing badly enough. # You must set STARTER_LOG_NAME_APPEND=jobid in your HTCondor config for it to work # starter_log = Path(htcondor.param['LOG']) / f"StarterLog.{m._cluster_ids[0]}.0" # print(starter_log.read_text()) assert m.get(0) assert target.read_text() == "hi" # make sure we did NOT transfer it back as normal assert "foo" not in (path.stem for path in m.output_files[0].iterdir())
def func(_): output = Path("remote-foo") output.write_text("hi") destination = htmap.TransferPath(target, protocol="file") htmap.transfer_output_files((output, destination)) return True
def test_path_in_nested_dict(tmp_path): @htmap.mapped def func(dict_of_dict_of_paths): return all( all(p.name in cwd_names() for p in dict_of_paths.values()) for dict_of_paths in dict_of_dict_of_paths.values()) paths = [ htmap.TransferPath(tmp_path / "test-1.txt"), htmap.TransferPath(tmp_path / "test-2.txt"), htmap.TransferPath(tmp_path / "test-3.txt"), ] for p in paths: p.touch() m = func.map(args=[{"paths": dict(zip(range(len(paths)), paths))}]) assert m.get(0)
def test_path_in_nested_list(tmp_path): @htmap.mapped def func(list_of_list_of_paths): return all( all(p.name in cwd_names() for p in list_of_paths) for list_of_paths in list_of_list_of_paths) paths = [ htmap.TransferPath(tmp_path / "test-1.txt"), htmap.TransferPath(tmp_path / "test-2.txt"), htmap.TransferPath(tmp_path / "test-3.txt"), ] for p in paths: p.touch() m = func.map(args=[[paths]]) assert m.get(0)
def test_input_transfer_via_https_protocol(tmp_path): f = htmap.TransferPath("status/200", protocol="https", location="httpbin.org") def test(file): return file.read_text() == "" m = htmap.map(test, [f]) assert m.get(0)
def test_input_transfer_via_file_protocol(tmp_path): f = htmap.TransferPath(__file__, protocol="file") MARKER = 12345 def test(file): return "MARKER = 12345" in file.read_text() m = htmap.map(test, [f]) assert m.get(0)
def test_transfer_directory(tmp_path): dir = htmap.TransferPath(tmp_path / "dir") dir.mkdir() file_in_dir = dir / "file" file_in_dir.write_text("hi") def test(dir): return (dir / "file").read_text() == "hi" m = htmap.map(test, [dir]) assert m.get(0)
def transfer_path(tmp_path): p = htmap.TransferPath(tmp_path / TEST_FILE_NAME) p.touch() return p
def test_transfer_path_to_path(): assert isinstance(htmap.TransferPath(Path('foobar')), htmap.TransferPath)