def test_local_worker():
with remote_runner.utility.ChangeToTemporaryDirectory():
tasks = [
MyLocalTask(wd=Path("t1"), message='1'),
MyLocalTask(wd=Path("t2"), message='2'),
MyLocalTask(wd=Path("t3"), message='3'),
]
worker = remote_runner.LocalWorker()
pool = remote_runner.Pool([worker])
pool.run(tasks)
assert Path("t1", "data").open().read() == "1"
assert Path("t2", "data").open().read() == "2"
assert Path("t3", "data").open().read() == "3"
def run(self):
time.sleep(1)
print(self.name)
wd = Path.cwd()
with remote_runner.utility.ChangeToTemporaryDirectory():
tasks = [
MyTask(name="1"),
MyTask(name="2")
]
with remote_runner.utility.ChangeDirectory(wd): # cd back to avoid .coverage.* files loss
remote_runner.Pool([
ssh_worker_factory(),
ssh_worker_factory()
]).run(tasks)
assert "1" in Path("1/stdout").open().read()
assert "2" in Path("2/stdout").open().read()
assert 0 == int(Path("1/exit_code").open().read().strip())
assert 0 == int(Path("2/exit_code").open().read().strip())
# remote root is cleaned
tmp_remote_dirs = list(
d for d in Path(ssh_worker_factory().remote_root.expanduser()).glob("*") if d.is_dir()
)
assert tmp_remote_dirs == []
with remote_runner.utility.ChangeToTemporaryDirectory():
N = 1000
random_data = [random.randint(0, 100) for i in range(N)]
task_name = "syncing_task"
reader = KeepReading(Path(task_name, "output").absolute(), period=0.4)
reader.start()
worker = sync_ssh_worker_factory()
tasks = [MyTask(name=task_name, data=random_data)]
with remote_runner.utility.ChangeDirectory(
wd): # cd back to avoid .coverage.* files loss
remote_runner.Pool([
worker,
]).run(tasks)
reader.stop_reading.set()
reader.join()
prev = reader.data[0]
n_diffs = 0
for curr in reader.data[1:]:
assert curr.startswith(prev), "File updates are not incrementing"
if curr != prev:
n_diffs += 1
prev = curr
assert n_diffs > 3, "No intermediate output copies"
with remote_runner.utility.ChangeDirectory(Path("TMP")):
tasks = [MyTask(name="one"), MyTask(name="two")]
with remote_runner.utility.ChangeDirectory(
wd): # cd back to avoid .coverage.* files loss
workers = [
remote_runner.LocalPbsWorker(resources="nodes=1:ppn=1",
remote_user_rc="""
unset PYTHONPATH
source ~/venv-3.8/bin/activate
python --version
pip list -v
""")
]
remote_runner.Pool(workers).run(tasks)
stdout_1 = Path("one/stdout").open().read()
stdout_2 = Path("two/stdout").open().read()
assert "one" in stdout_1
assert "two" in stdout_2
assert "bionmr-mom-00" in stdout_1
assert "bionmr-mom-00" in stdout_2
pbs_stdout_2 = Path("two/.pbs.stdout").open().read()
assert pbs_stdout_2.startswith("Python 3.")
assert "remote-runner" in pbs_stdout_2
assert "" == Path("one/stderr").open().read()