def test_run_generic_commands(generic_command_fixture):
commands = [
"ls .",
"ls invalid-file-path",
]
with open(TEST_FILENAME, "w") as f_out:
for command in commands:
f_out.write(command + "\n")
inputs = GenericCommandInputs(TEST_FILENAME)
config = GenericCommandConfiguration(job_inputs=inputs)
for job_param in inputs.iter_jobs():
config.add_job(job_param)
assert config.get_num_jobs() == 2
config.dump(CONFIG_FILE)
cmds = (
f"{SUBMIT_JOBS} {CONFIG_FILE} --output={OUTPUT} -p 0.1",
# Test with higher queue depth. This exercises the code paths but
# doesn't actually verify the functionality.
# The infrastructure to do that is currently lacking. TODO
f"{SUBMIT_JOBS} {CONFIG_FILE} --output={OUTPUT} -p 0.1 -q 32",
)
for cmd in cmds:
ret = run_command(cmd)
assert ret == 0
def auto_config(cls,
inputs,
cancel_on_blocking_job_failure=False,
minutes_per_job=None,
**kwargs):
"""Create a configuration from all available inputs."""
if isinstance(inputs, str):
inputs = GenericCommandInputs(inputs)
config = GenericCommandConfiguration(**kwargs)
for job_param in inputs.iter_jobs():
job_param.cancel_on_blocking_job_failure = cancel_on_blocking_job_failure
job_param.estimated_run_minutes = minutes_per_job
config.add_job(job_param)
return config
def test_job_order(generic_command_fixture):
num_jobs = 50
commands = ["echo hello world"] * num_jobs
with open(TEST_FILENAME, "w") as f_out:
for command in commands:
f_out.write(command + "\n")
inputs = GenericCommandInputs(TEST_FILENAME)
config = GenericCommandConfiguration(job_inputs=inputs)
for job_param in inputs.iter_jobs():
config.add_job(job_param)
assert config.get_num_jobs() == num_jobs
job = config.get_job("1")
for i in range(10, 15):
job.blocked_by.add(i)
config.get_job("2").blocked_by.add("1")
config.get_job("21").blocked_by.add("30")
config.get_job("41").blocked_by.add("50")
config.dump(CONFIG_FILE)
os.environ["FAKE_HPC_CLUSTER"] = "True"
cmd = f"{SUBMIT_JOBS} {CONFIG_FILE} --output={OUTPUT} " \
"--per-node-batch-size=10 " \
"--max-nodes=4 " \
"--poll-interval=.1 " \
"--num-processes=10"
ret = run_command(cmd)
assert ret == 0
result_summary = ResultsSummary(OUTPUT)
results = result_summary.list_results()
assert len(results) == num_jobs
tracker = {}
for result in results:
tracker[result.name] = result
for i in range(10, 15):
assert tracker["1"].completion_time > tracker[str(i)].completion_time
assert tracker["2"].completion_time > tracker["1"].completion_time
assert tracker["21"].completion_time > tracker["30"].completion_time
assert tracker["41"].completion_time > tracker["50"].completion_time
def auto_config(cls, inputs, **kwargs):
"""Create a configuration from all available inputs."""
if isinstance(inputs, str):
job_inputs = GenericCommandInputs(inputs)
else:
job_inputs = inputs
config = GenericCommandConfiguration(job_inputs, **kwargs)
for job_param in config.inputs.iter_jobs():
config.add_job(job_param)
return config
def test_job_configuration__check_job_dependencies(job_fixture):
with open(TEST_FILENAME, "w") as f_out:
f_out.write("echo hello world\n")
inputs = GenericCommandInputs(TEST_FILENAME)
config = GenericCommandConfiguration(job_inputs=inputs)
for job_param in inputs.iter_jobs():
config.add_job(job_param)
assert config.get_num_jobs() == 1
job = config.get_job("1")
job.blocked_by.add("10")
with pytest.raises(InvalidConfiguration):
config.check_job_dependencies()
# While we have this setup, verify that submit-jobs calls this function.
config.dump(CONFIG_FILE)
cmd = f"{SUBMIT_JOBS} {CONFIG_FILE} --output={OUTPUT} " \
"--poll-interval=.1 "
ret = run_command(cmd)
assert ret != 0
def test_try_add_blocked_jobs(cleanup):
num_commands = 5
commands = ["ls ."] * num_commands
with open(TEST_FILENAME, "w") as f_out:
for command in commands:
f_out.write(command + "\n")
inputs = GenericCommandInputs(TEST_FILENAME)
config = GenericCommandConfiguration(job_inputs=inputs)
jobs = list(inputs.iter_jobs())
for i, job_param in enumerate(jobs):
if i == num_commands - 1:
job_param.blocked_by = set([1, 2, 3, 4])
config.add_job(job_param)
config.dump(CONFIG_FILE)
os.environ["FAKE_HPC_CLUSTER"] = "True"
for option in ("--try-add-blocked-jobs", "--no-try-add-blocked-jobs"):
cmd = f"{SUBMIT_JOBS} {CONFIG_FILE} --output={OUTPUT} -p 0.1 {option}"
ret = run_command(cmd)
assert ret == 0
events_file = os.path.join(OUTPUT, "submit_jobs_events.log")
events_summary = EventsSummary(OUTPUT, preload=True)
submit_events = events_summary.list_events(EVENT_NAME_HPC_SUBMIT)
if option == "--try-add-blocked-jobs":
assert len(submit_events) == 1
event = submit_events[0]
assert event.data["batch_size"] == num_commands
assert event.data["num_blocked"] == 0
shutil.rmtree(OUTPUT)
else:
assert len(submit_events) == 2
event1 = submit_events[0]
event2 = submit_events[1]
assert event1.data["batch_size"] == num_commands - 1
assert event2.data["batch_size"] == 1
assert event1.data["num_blocked"] == 1
assert event2.data["num_blocked"] == 0
def test_sorted_order(generic_command_fixture):
with open(TEST_FILENAME, "w") as f_out:
pass
inputs = GenericCommandInputs(TEST_FILENAME)
config = GenericCommandConfiguration(inputs)
num_jobs = 20
for i in range(num_jobs):
job = GenericCommandParameters("echo hello")
config.add_job(job)
assert config.get_num_jobs() == num_jobs
job_ids = [job.job_id for job in config.iter_jobs()]
assert job_ids == list(range(1, num_jobs + 1))