def run(self, names_of_jobs_to_run):
# if names_of_jobs_to_run, create job_queue which contains jobs that
# are to be run.
if not names_of_jobs_to_run:
job_queue = self.jobs
else:
job_queue = [
j for j in self.jobs if j.name() in names_of_jobs_to_run
]
init_message = Init(job_queue, self.simulation_id, self.ert_pid)
unused = set(names_of_jobs_to_run) - set([j.name() for j in job_queue])
if unused:
init_message.with_error(
"{} does not exist. Available jobs: {}".format(
unused, [j.name() for j in self.jobs]))
yield init_message
return
else:
yield init_message
for job in job_queue:
for status_update in job.run():
yield status_update
if not status_update.success():
yield Finish().with_error(
"Not all jobs completed successfully.")
return
yield Finish()
def test_report_startup_clearing_of_event_log_file(tmpdir):
reporter1 = Event(event_log=tmpdir / "event_log")
job1 = Job({"name": "job1", "stdout": "stdout", "stderr": "stderr"}, 0)
reporter1.report(Init([job1], 1, 19, ee_id="ee_id", real_id=0, stage_id=0))
reporter2 = Event(event_log=tmpdir / "event_log")
assert os.path.getsize(tmpdir / "event_log") == 0
def test_report_with_successful_exit_message_argument(tmpdir):
reporter = Event(event_log=tmpdir / "event_log")
job1 = Job({"name": "job1", "stdout": "stdout", "stderr": "stderr"}, 0)
reporter.report(Init([job1], 1, 19, ee_id="ee_id", real_id=0, stage_id=0))
reporter.report(Exited(job1, 0))
with open(reporter._event_log, "r") as f:
lines = f.readlines()
assert len(lines) == 2
event = json.loads(lines[1])
assert event["type"] == _FM_JOB_SUCCESS
def test_report_with_failed_exit_message_argument(tmpdir):
reporter = Event(event_log=tmpdir / "event_log")
job1 = Job({"name": "job1", "stdout": "stdout", "stderr": "stderr"}, 0)
reporter.report(Init([job1], 1, 19, ee_id="ee_id", real_id=0, stage_id=0))
reporter.report(Exited(job1, 1).with_error("massive_failure"))
with open(reporter._event_log, "r") as f:
lines = f.readlines()
assert len(lines) == 2
event = json.loads(lines[1])
assert event["type"] == _FM_JOB_FAILURE
assert event["data"]["error_msg"] == "massive_failure"
def test_report_only_job_running_for_successful_run(unused_tcp_port):
host = "localhost"
url = f"ws://{host}:{unused_tcp_port}"
reporter = Event(evaluator_url=url)
job1 = Job({"name": "job1", "stdout": "stdout", "stderr": "stderr"}, 0)
lines = []
with _mock_ws_thread(host, unused_tcp_port, lines):
reporter.report(Init([job1], 1, 19, ee_id="ee_id", real_id=0, step_id=0))
reporter.report(Running(job1, 100, 10))
reporter.report(Finish())
assert len(lines) == 1
def test_report_with_successful_finish_message_argument(tmpdir):
reporter = Event(event_log=tmpdir / "event_log")
job1 = Job({"name": "job1", "stdout": "stdout", "stderr": "stderr"}, 0)
reporter.report(Init([job1], 1, 19, ee_id="ee_id", real_id=0, stage_id=0))
reporter.report(Running(job1, 100, 10))
reporter.report(Finish())
with open(reporter._event_log, "r") as f:
lines = f.readlines()
assert len(lines) == 3
event = json.loads(lines[2])
assert event["type"] == _FM_STEP_SUCCESS
def test_report_with_failed_finish_message_argument(unused_tcp_port):
host = "localhost"
url = f"ws://{host}:{unused_tcp_port}"
reporter = Event(evaluator_url=url)
job1 = Job({"name": "job1", "stdout": "stdout", "stderr": "stderr"}, 0)
lines = []
with _mock_ws_thread(host, unused_tcp_port, lines):
reporter.report(Init([job1], 1, 19, ee_id="ee_id", real_id=0, step_id=0))
reporter.report(Running(job1, 100, 10))
reporter.report(Finish().with_error("massive_failure"))
assert len(lines) == 1
def test_report_with_running_message_argument(tmpdir):
reporter = Event(event_log=tmpdir / "event_log")
job1 = Job({"name": "job1", "stdout": "stdout", "stderr": "stderr"}, 0)
reporter.report(Init([job1], 1, 19, ee_id="ee_id", real_id=0, stage_id=0))
reporter.report(Running(job1, 100, 10))
with open(reporter._event_log, "r") as f:
lines = f.readlines()
assert len(lines) == 2
event = json.loads(lines[1])
assert event["type"] == _FM_JOB_RUNNING
assert event["data"]["max_memory_usage"] == 100
assert event["data"]["current_memory_usage"] == 10
def test_report_with_successful_start_message_argument(tmpdir):
reporter = Event(event_log=tmpdir / "event_log")
job1 = Job({"name": "job1", "stdout": "stdout", "stderr": "stderr"}, 0)
reporter.report(Init([job1], 1, 19, ee_id="ee_id", real_id=0, stage_id=0))
msg = Start(job1)
reporter.report(msg)
with open(reporter._event_log, "r") as f:
lines = f.readlines()
assert len(lines) == 2
event = json.loads(lines[1])
assert event["type"] == _FM_JOB_START
assert event["source"] == "/ert/ee/ee_id/real/0/stage/0/step/0/job/0"
def test_report_with_successful_exit_message_argument(unused_tcp_port):
host = "localhost"
url = f"ws://{host}:{unused_tcp_port}"
reporter = Event(evaluator_url=url)
job1 = Job({"name": "job1", "stdout": "stdout", "stderr": "stderr"}, 0)
lines = []
with _mock_ws_thread(host, unused_tcp_port, lines):
reporter.report(Init([job1], 1, 19, ee_id="ee_id", real_id=0, step_id=0))
reporter.report(Exited(job1, 0))
reporter.report(Finish().with_error("failed"))
assert len(lines) == 1
event = json.loads(lines[0])
assert event["type"] == _FM_JOB_SUCCESS
def test_job_dispatch_kills_itself_after_unsuccessful_job(self):
jobs_json = json.dumps({"ee_id": "_id_"})
with patch("job_runner.cli.os") as mock_os, patch(
"job_runner.cli.open",
new=mock_open(read_data=jobs_json)) as mock_file, patch(
"job_runner.cli.JobRunner") as mock_runner:
mock_runner.return_value.run.return_value = [
Init([], 0, 0),
Finish().with_error("overall bad run"),
]
mock_os.getpgid.return_value = 17
main(["script.py", "/foo/bar/baz"])
mock_os.killpg.assert_called_with(17, signal.SIGKILL)
def test_report_with_init_message_argument(self):
r = self.reporter
job1 = Job({"name": "job1", "stdout": "/stdout", "stderr": "/stderr"}, 0)
r.report(Init([job1], 1, 19))
with open(self.reporter.STATUS_file, "r") as f:
self.assertIn(
"Current host", f.readline(), "STATUS file missing expected value"
)
with open(self.reporter.STATUS_json, "r") as f:
contents = "".join(f.readlines())
self.assertIn('"name": "job1"', contents, "status.json missing job1")
self.assertIn(
'"status": "Waiting"', contents, "status.json missing Waiting status"
)
def test_report_with_running_message_argument(unused_tcp_port):
host = "localhost"
url = f"ws://{host}:{unused_tcp_port}"
reporter = Event(evaluator_url=url)
job1 = Job({"name": "job1", "stdout": "stdout", "stderr": "stderr"}, 0)
lines = []
with _mock_ws_thread(host, unused_tcp_port, lines):
reporter.report(Init([job1], 1, 19, ee_id="ee_id", real_id=0, step_id=0))
reporter.report(Running(job1, 100, 10))
reporter.report(Finish())
assert len(lines) == 1
event = json.loads(lines[0])
assert event["type"] == _FM_JOB_RUNNING
assert event["data"]["max_memory_usage"] == 100
assert event["data"]["current_memory_usage"] == 10
def test_report_with_successful_start_message_argument(unused_tcp_port):
host = "localhost"
url = f"ws://{host}:{unused_tcp_port}"
reporter = Event(evaluator_url=url)
job1 = Job({"name": "job1", "stdout": "stdout", "stderr": "stderr"}, 0)
lines = []
with _mock_ws_thread(host, unused_tcp_port, lines):
reporter.report(Init([job1], 1, 19, ee_id="ee_id", real_id=0, step_id=0))
reporter.report(Start(job1))
reporter.report(Finish())
assert len(lines) == 1
event = json.loads(lines[0])
assert event["type"] == _FM_JOB_START
assert event["source"] == "/ert/ee/ee_id/real/0/step/0/job/0"
assert os.path.basename(event["data"]["stdout"]) == "stdout"
assert os.path.basename(event["data"]["stderr"]) == "stderr"
def test_job_dispatch_kills_itself_after_unsuccessful_job(unused_tcp_port):
host = "localhost"
port = unused_tcp_port
jobs_json = json.dumps({"ee_id": "_id_", "dispatch_url": f"ws://localhost:{port}"})
with patch("job_runner.cli.os") as mock_os, patch(
"job_runner.cli.open", new=mock_open(read_data=jobs_json)
) as mock_file, patch("job_runner.cli.JobRunner") as mock_runner:
mock_runner.return_value.run.return_value = [
Init([], 0, 0),
Finish().with_error("overall bad run"),
]
mock_os.getpgid.return_value = 17
with _mock_ws_thread(host, port, []):
main(["script.py", "/foo/bar/baz"])
mock_os.killpg.assert_called_with(17, signal.SIGKILL)
def test_report_with_init_message_argument(tmpdir):
reporter = Event(event_log=tmpdir / "event_log")
job1 = Job({"name": "job1", "stdout": "stdout", "stderr": "stderr"}, 0)
reporter.report(Init([job1], 1, 19, ee_id="ee_id", real_id=0, stage_id=0))
with open(reporter._event_log, "r") as f:
lines = f.readlines()
assert len(lines) == 1
event = json.loads(lines[0])
job = event.get("data", {}).get("jobs", {}).get("0", {})
assert job
assert job["name"] == "job1"
assert job["stdout"].startswith("/") and job["stdout"].endswith(
"stdout")
assert job["stderr"].startswith("/") and job["stderr"].endswith(
"stderr")
assert event["type"] == _FM_STEP_START
assert event["source"] == "/ert/ee/ee_id/real/0/stage/0/step/0"
def test_report_with_failed_start_message_argument(unused_tcp_port):
host = "localhost"
url = f"ws://{host}:{unused_tcp_port}"
reporter = Event(evaluator_url=url)
job1 = Job({"name": "job1", "stdout": "stdout", "stderr": "stderr"}, 0)
lines = []
with _mock_ws_thread(host, unused_tcp_port, lines):
reporter.report(Init([job1], 1, 19, ee_id="ee_id", real_id=0, step_id=0))
msg = Start(job1).with_error("massive_failure")
reporter.report(msg)
reporter.report(Finish())
assert len(lines) == 2
event = json.loads(lines[1])
assert event["type"] == _FM_JOB_FAILURE
assert event["data"]["error_msg"] == "massive_failure"
def test_status_file_is_correct(self):
"""The STATUS file is a file to which we append data about jobs as they
are run. So this involves multiple reports, and should be tested as
such.
See https://github.com/equinor/libres/issues/764
"""
j_1 = Job({"name": "j_1", "executable": "", "argList": []}, 0)
j_2 = Job({"name": "j_2", "executable": "", "argList": []}, 0)
init = Init([j_1, j_2], 1, 1)
start_j_1 = Start(j_1)
exited_j_1 = Exited(j_1, 0)
start_j_2 = Start(j_2)
exited_j_2 = Exited(j_2, 9).with_error("failed horribly")
for msg in [init, start_j_1, exited_j_1, start_j_2, exited_j_2]:
self.reporter.report(msg)
expected_j1_line = (
"{:32}: {start_ts:%H:%M:%S} .... {end_ts:%H:%M:%S} \n".
format( # noqa
j_1.name(),
start_ts=start_j_1.timestamp,
end_ts=exited_j_1.timestamp))
expected_j2_line = "{:32}: {start_ts:%H:%M:%S} .... {end_ts:%H:%M:%S} EXIT: {code}/{msg}\n".format( # noqa
j_2.name(),
start_ts=start_j_2.timestamp,
end_ts=exited_j_2.timestamp,
code=exited_j_2.exit_code,
msg=exited_j_2.error_message,
)
with open(self.reporter.STATUS_file, "r") as f:
for expected in [
"Current host",
expected_j1_line,
expected_j2_line,
]: # noqa
self.assertIn(expected, f.readline())
# EOF
self.assertEqual("", f.readline())
def test_init_msg(self, post_mock):
self.reporter.report(Init([], 0, 1))
self.assertTrue(post_mock.called)