def test_jobs_deleted_local_directory(context):
path = context["directory"]
run = context["run_fn"]
jobs = context["jobs_fn"]
run(command=["touch", "ok"], outputs=["ok"], resref="myshell")
shutil.rmtree(path)
with swallow_logs(new_level=logging.ERROR) as log:
jobs(queries=[], status=True)
assert "no longer exists" in log.out
def test_jobs_show(context):
run = context["run_fn"]
jobs = context["jobs_fn"]
registry = context["registry"]
run(command=["touch", "ok"], outputs=["ok"], resref="myshell")
assert len(registry.find_job_files()) == 1
with swallow_outputs() as output:
jobs(queries=[], action="show", status=True)
# `show`, as opposed to `list`, is detailed, multiline display.
assert len(output.out.splitlines()) > 1
assert "myshell" in output.out
assert "status:" in output.out
def test_jobs_query_unknown(context):
run = context["run_fn"]
jobs = context["jobs_fn"]
registry = context["registry"]
run(command=["doesntmatter"], resref="myshell")
jobfiles = registry.find_job_files()
assert len(jobfiles) == 1
jobid = list(jobfiles.keys())[0]
with swallow_logs(new_level=logging.WARNING) as log:
jobs(queries=[jobid + "-trailing-garbage"])
assert "No jobs matched" in log.out
assert len(registry.find_job_files()) == 1
def test_run_and_follow(context):
path = context["directory"]
run = context["run_fn"]
jobs = context["jobs_fn"]
registry = context["registry"]
run(command=["touch", "ok"], outputs=["ok"], resref="myshell", follow=True)
with swallow_logs(new_level=logging.INFO) as log:
jobs(queries=[])
assert len(registry.find_job_files()) == 0
assert "No jobs" in log.out
assert op.exists(op.join(path, "ok"))
def test_jobs_auto_fetch_with_query(context):
path = context["directory"]
run = context["run_fn"]
jobs = context["jobs_fn"]
registry = context["registry"]
run(command=["touch", "ok"], outputs=["ok"], resref="myshell")
jobfiles = registry.find_job_files()
assert len(jobfiles) == 1
jobid = list(jobfiles.keys())[0]
with swallow_outputs():
jobs(queries=[jobid[3:]])
assert len(registry.find_job_files()) == 0
assert op.exists(op.join(path, "ok"))
def test_recursive_transfer(context):
run = context["run_fn"]
path = context["directory"]
jobs = context["jobs_fn"]
# Our script takes an inventory of the execute directory so we can be
# sure that all of the files have transferred. It then creates a tree
# that we verify is returned.
create_tree(
path, {
"script":
"find . > out_file ; mkdir out_dir ; touch out_dir/subfile",
"in_dir": {
"subfile": ""
}
})
with swallow_outputs() as output:
run(command=["sh", "-e", "script"],
inputs=["script", "in_dir"],
outputs=["out_file", "out_dir"],
resref="myshell")
try_fetch(lambda: jobs(queries=[], action="fetch", all_=True))
assert op.exists(op.join(path, "out_file"))
with open(op.join(path, "out_file")) as fo:
lines = [line.strip() for line in fo]
assert "./in_dir/subfile" in lines
assert op.exists(op.join(path, "out_dir", "subfile"))
def test_jobs_ambig_id_match(context):
run = context["run_fn"]
jobs = context["jobs_fn"]
registry = context["registry"]
run(command=["doesntmatter0"], resref="myshell")
run(command=["doesntmatter1"], resref="myshell")
jobfiles = registry.find_job_files()
assert len(jobfiles) == 2
jobid0, jobid1 = jobfiles.keys()
# Start of ID is year, so first characters should be the same.
assert jobid0[0] == jobid1[0], "test assumption is wrong"
with pytest.raises(ValueError) as exc:
jobs(queries=[jobid0[0], jobid1[0]])
assert "matches multiple jobs" in str(exc.value)
def test_jobs_delete(context):
path = context["directory"]
run = context["run_fn"]
jobs = context["jobs_fn"]
registry = context["registry"]
run(command=["touch", "ok"], outputs=["ok"], resref="myshell")
# Must explicit specify jobs to delete.
with pytest.raises(ValueError):
jobs(queries=[], action="delete")
jobfiles = registry.find_job_files()
assert len(jobfiles) == 1
jobid = list(jobfiles.keys())[0]
with swallow_outputs():
jobs(queries=[jobid[3:]], action="delete")
assert len(registry.find_job_files()) == 0
assert not op.exists(op.join(path, "ok"))
def test_jobs_orc_error(context):
run = context["run_fn"]
jobs = context["jobs_fn"]
registry = context["registry"]
run(command=["doesntmatter1"], resref="myshell")
jobfiles = registry.find_job_files()
assert len(jobfiles) == 1
with swallow_outputs() as output:
with swallow_logs(new_level=logging.ERROR) as log:
def die_orc(*args, **kwargs):
raise OrchestratorError("resurrection failed")
with patch("reproman.interface.jobs.show_oneline",
side_effect=die_orc):
jobs(queries=[], status=True)
assert "myshell" not in output.out
assert "resurrection failed" in log.out
def test_run_and_fetch(context):
path = context["directory"]
run = context["run_fn"]
jobs = context["jobs_fn"]
registry = context["registry"]
create_tree(path,
tree={
"js0.yaml": ("resource_name: myshell\n"
"command_str: 'touch ok'\n"
"outputs: ['ok']")
})
run(job_specs=["js0.yaml"])
with swallow_logs(new_level=logging.INFO) as log:
with swallow_outputs() as output:
jobs(queries=[], status=True)
assert "myshell" in output.out
assert len(registry.find_job_files()) == 1
jobs(queries=[], action="fetch", all_=True)
assert len(registry.find_job_files()) == 0
jobs(queries=[], status=True)
assert "No jobs" in log.out
assert op.exists(op.join(path, "ok"))
def test_jobs_deleted_resource(context):
run = context["run_fn"]
jobs = context["jobs_fn"]
registry = context["registry"]
resman = context["resource_manager"]
resman.create("todelete", resource_type="shell")
run(command=["doesntmatter0"], resref="todelete")
run(command=["doesntmatter1"], resref="myshell")
resman.delete(resman.get_resource("todelete"))
jobfiles = registry.find_job_files()
assert len(jobfiles) == 2
with swallow_outputs() as output:
with swallow_logs(new_level=logging.ERROR) as log:
jobs(queries=[], status=True)
assert "todelete" in log.out
# The deleted resource won't be there
assert "todelete" not in output.out
# ... but the alive one will.
assert "myshell" in output.out
def test_jobs_unknown_action(context):
run = context["run_fn"]
jobs = context["jobs_fn"]
run(command=["doesntmatter"], resref="myshell")
with pytest.raises(RuntimeError):
jobs(queries=[], action="you don't know me")
def jobs_fn(*args, **kwargs):
with patch("reproman.interface.jobs.get_manager",
return_value=resource_manager):
with patch("reproman.interface.jobs.LREG", registry):
return jobs(*args, **kwargs)
