def test_recursive_transfer(context):
run = context["run_fn"]
path = context["directory"]
jobs = context["jobs_fn"]
# Our script takes an inventory of the execute directory so we can be
# sure that all of the files have transferred. It then creates a tree
# that we verify is returned.
create_tree(
path, {
"script":
"find . > out_file ; mkdir out_dir ; touch out_dir/subfile",
"in_dir": {
"subfile": ""
}
})
with swallow_outputs() as output:
run(command=["sh", "-e", "script"],
inputs=["script", "in_dir"],
outputs=["out_file", "out_dir"],
resref="myshell")
try_fetch(lambda: jobs(queries=[], action="fetch", all_=True))
assert op.exists(op.join(path, "out_file"))
with open(op.join(path, "out_file")) as fo:
lines = [line.strip() for line in fo]
assert "./in_dir/subfile" in lines
assert op.exists(op.join(path, "out_dir", "subfile"))
def test_run_and_fetch(context):
path = context["directory"]
run = context["run_fn"]
jobs = context["jobs_fn"]
registry = context["registry"]
create_tree(path,
tree={
"js0.yaml": ("resource_name: myshell\n"
"command_str: 'touch ok'\n"
"outputs: ['ok']")
})
run(job_specs=["js0.yaml"])
with swallow_logs(new_level=logging.INFO) as log:
with swallow_outputs() as output:
jobs(queries=[], status=True)
assert "myshell" in output.out
assert len(registry.find_job_files()) == 1
jobs(queries=[], action="fetch", all_=True)
assert len(registry.find_job_files()) == 0
jobs(queries=[], status=True)
assert "No jobs" in log.out
assert op.exists(op.join(path, "ok"))
def test_jobs_deleted_local_directory(context):
path = context["directory"]
run = context["run_fn"]
jobs = context["jobs_fn"]
run(command=["touch", "ok"], outputs=["ok"], resref="myshell")
shutil.rmtree(path)
with swallow_logs(new_level=logging.ERROR) as log:
jobs(queries=[], status=True)
assert "no longer exists" in log.out
def test_run_and_follow(context):
path = context["directory"]
run = context["run_fn"]
jobs = context["jobs_fn"]
registry = context["registry"]
run(command=["touch", "ok"], outputs=["ok"], resref="myshell", follow=True)
with swallow_logs(new_level=logging.INFO) as log:
jobs(queries=[])
assert len(registry.find_job_files()) == 0
assert "No jobs" in log.out
assert op.exists(op.join(path, "ok"))
def test_jobs_show(context):
run = context["run_fn"]
jobs = context["jobs_fn"]
registry = context["registry"]
run(command=["touch", "ok"], outputs=["ok"], resref="myshell")
assert len(registry.find_job_files()) == 1
with swallow_outputs() as output:
jobs(queries=[], action="show", status=True)
# `show`, as opposed to `list`, is detailed, multiline display.
assert len(output.out.splitlines()) > 1
assert "myshell" in output.out
assert "status:" in output.out
def test_jobs_query_unknown(context):
run = context["run_fn"]
jobs = context["jobs_fn"]
registry = context["registry"]
run(command=["doesntmatter"], resref="myshell")
jobfiles = registry.find_job_files()
assert len(jobfiles) == 1
jobid = list(jobfiles.keys())[0]
with swallow_logs(new_level=logging.WARNING) as log:
jobs(queries=[jobid + "-trailing-garbage"])
assert "No jobs matched" in log.out
assert len(registry.find_job_files()) == 1
def test_jobs_auto_fetch_with_query(context):
path = context["directory"]
run = context["run_fn"]
jobs = context["jobs_fn"]
registry = context["registry"]
run(command=["touch", "ok"], outputs=["ok"], resref="myshell")
jobfiles = registry.find_job_files()
assert len(jobfiles) == 1
jobid = list(jobfiles.keys())[0]
with swallow_outputs():
try_fetch(lambda: jobs(queries=[jobid[3:]]))
assert len(registry.find_job_files()) == 0
assert op.exists(op.join(path, "ok"))
def test_jobs_ambig_id_match(context):
run = context["run_fn"]
jobs = context["jobs_fn"]
registry = context["registry"]
run(command=["doesntmatter0"], resref="myshell")
run(command=["doesntmatter1"], resref="myshell")
jobfiles = registry.find_job_files()
assert len(jobfiles) == 2
jobid0, jobid1 = jobfiles.keys()
# Start of ID is year, so first characters should be the same.
assert jobid0[0] == jobid1[0], "test assumption is wrong"
with pytest.raises(ValueError) as exc:
jobs(queries=[jobid0[0], jobid1[0]])
assert "matches multiple jobs" in str(exc.value)
def test_jobs_delete(context):
path = context["directory"]
run = context["run_fn"]
jobs = context["jobs_fn"]
registry = context["registry"]
run(command=["touch", "ok"], outputs=["ok"], resref="myshell")
# Must explicit specify jobs to delete.
with pytest.raises(ValueError):
jobs(queries=[], action="delete")
jobfiles = registry.find_job_files()
assert len(jobfiles) == 1
jobid = list(jobfiles.keys())[0]
with swallow_outputs():
jobs(queries=[jobid[3:]], action="delete")
assert len(registry.find_job_files()) == 0
assert not op.exists(op.join(path, "ok"))
def run_fn(*args, **kwargs):
with contextlib.ExitStack() as stack:
stack.enter_context(chpwd(path))
# Patch home to avoid populating testing machine with jobs when
# using local shell.
stack.enter_context(patch.dict(os.environ, {"HOME": home}))
stack.enter_context(patch("reproman.interface.run.get_manager",
return_value=resource_manager))
stack.enter_context(patch("reproman.interface.run.LocalRegistry",
job_registry))
return run(*args, **kwargs)
def test_jobs_orc_error(context):
run = context["run_fn"]
jobs = context["jobs_fn"]
registry = context["registry"]
run(command=["doesntmatter1"], resref="myshell")
jobfiles = registry.find_job_files()
assert len(jobfiles) == 1
with swallow_outputs() as output:
with swallow_logs(new_level=logging.ERROR) as log:
def die_orc(*args, **kwargs):
raise OrchestratorError("resurrection failed")
with patch("reproman.interface.jobs.show_oneline",
side_effect=die_orc):
jobs(queries=[], status=True)
assert "myshell" not in output.out
assert "resurrection failed" in log.out
def test_run_and_follow_action(context, action):
run = context["run_fn"]
expect = "does not support the 'stop' feature"
with swallow_logs(new_level=logging.INFO) as log:
run(command=["false"], resref="myshell",
follow=action)
if action.endswith("-if-success"):
assert expect not in log.out
else:
assert expect in log.out
if action != "delete":
with swallow_logs(new_level=logging.INFO) as log:
run(command=["true"], resref="myshell",
follow=action)
assert expect in log.out
if action.startswith("delete"):
resman = context["resource_manager"]
with pytest.raises(ResourceNotFoundError):
resman.get_resource("myshell", resref_type="name")
def test_run_resource_specification(context):
path = context["directory"]
run = context["run_fn"]
create_tree(
path,
tree={"js0.yaml": "resource_name: name-via-js",
"js1.yaml": ("resource_id: id-via-js\n"
"resource_name: name-via-js")})
# Can specify name via job spec.
with pytest.raises(ResourceNotFoundError) as exc:
run(command=["doesnt", "matter"],
job_specs=["js0.yaml"])
assert "name-via-js" in str(exc.value)
# If job spec as name and ID, ID takes precedence.
with pytest.raises(ResourceNotFoundError) as exc:
run(command=["doesnt", "matter"],
job_specs=["js1.yaml"])
assert "id-via-js" in str(exc.value)
# Command-line overrides job spec.
with pytest.raises(ResourceNotFoundError) as exc:
run(command=["doesnt", "matter"], resref="fromcli",
job_specs=["js1.yaml"])
assert "fromcli" in str(exc.value)
def test_jobs_deleted_resource(context):
run = context["run_fn"]
jobs = context["jobs_fn"]
registry = context["registry"]
resman = context["resource_manager"]
resman.create("todelete", resource_type="shell")
run(command=["doesntmatter0"], resref="todelete")
run(command=["doesntmatter1"], resref="myshell")
resman.delete(resman.get_resource("todelete"))
jobfiles = registry.find_job_files()
assert len(jobfiles) == 2
with swallow_outputs() as output:
with swallow_logs(new_level=logging.ERROR) as log:
jobs(queries=[], status=True)
assert "todelete" in log.out
# The deleted resource won't be there
assert "todelete" not in output.out
# ... but the alive one will.
assert "myshell" in output.out
def test_run_list(arg, expected):
with swallow_outputs() as output:
run(list_=arg)
for e in expected:
assert e in output.out
def test_run_no_resource():
with pytest.raises(ValueError) as exc:
run(command="blahbert")
assert "No resource" in str(exc.value)
def test_run_batch_spec_and_params():
with pytest.raises(ValueError):
run(command="blahbert",
batch_spec="anything", batch_parameters="anything")
def test_jobs_unknown_action(context):
run = context["run_fn"]
jobs = context["jobs_fn"]
run(command=["doesntmatter"], resref="myshell")
with pytest.raises(RuntimeError):
jobs(queries=[], action="you don't know me")
def test_run_no_command(command):
with pytest.raises(ValueError) as exc:
run(command=command)
assert "No command" in str(exc.value)
