def run(self, names_of_jobs_to_run):
# if names_of_jobs_to_run, create job_queue which contains jobs that
# are to be run.
if not names_of_jobs_to_run:
job_queue = self.jobs
else:
job_queue = [
j for j in self.jobs if j.name() in names_of_jobs_to_run
]
init_message = Init(job_queue, self.simulation_id, self.ert_pid)
unused = set(names_of_jobs_to_run) - set([j.name() for j in job_queue])
if unused:
init_message.with_error(
"{} does not exist. Available jobs: {}".format(
unused, [j.name() for j in self.jobs]))
yield init_message
return
else:
yield init_message
for job in job_queue:
for status_update in job.run():
yield status_update
if not status_update.success():
yield Finish().with_error(
"Not all jobs completed successfully.")
return
yield Finish()
def test_successful_forward_model_reported(self, post_mock):
self.reporter.start_time = dt.now()
self.reporter.report(Finish())
_, data = post_mock.call_args
self.assertTrue(post_mock.called, "post not called on OK Finish")
self.assertIn('"status": "OK"', data["data"], "no OK in data")
def test_report_with_successful_finish_message_argument(self):
msg = Finish()
self.reporter.status_dict = self.reporter._init_job_status_dict(
msg.timestamp, 0, [])
self.reporter.report(msg, sync_disc_timeout=0)
with open(self.reporter.OK_file, "r") as f:
self.assertIn("All jobs complete", f.readline(),
"OK file missing expected value")
def test_job_dispatch_kills_itself_after_unsuccessful_job(self):
with patch("job_runner.cli.os") as mock_os, patch(
"job_runner.cli.JobRunner") as mock_runner:
mock_runner.return_value.run.return_value = [
Finish().with_error("overall bad run")
]
mock_os.getpgid.return_value = 17
main(["script.py", "/foo/bar/baz"])
mock_os.killpg.assert_called_with(17, signal.SIGKILL)
def test_report_with_failed_finish_message_argument(unused_tcp_port):
host = "localhost"
url = f"ws://{host}:{unused_tcp_port}"
reporter = Event(evaluator_url=url)
job1 = Job({"name": "job1", "stdout": "stdout", "stderr": "stderr"}, 0)
lines = []
with _mock_ws_thread(host, unused_tcp_port, lines):
reporter.report(Init([job1], 1, 19, ee_id="ee_id", real_id=0, step_id=0))
reporter.report(Running(job1, 100, 10))
reporter.report(Finish().with_error("massive_failure"))
assert len(lines) == 1
def test_report_only_job_running_for_successful_run(unused_tcp_port):
host = "localhost"
url = f"ws://{host}:{unused_tcp_port}"
reporter = Event(evaluator_url=url)
job1 = Job({"name": "job1", "stdout": "stdout", "stderr": "stderr"}, 0)
lines = []
with _mock_ws_thread(host, unused_tcp_port, lines):
reporter.report(Init([job1], 1, 19, ee_id="ee_id", real_id=0, step_id=0))
reporter.report(Running(job1, 100, 10))
reporter.report(Finish())
assert len(lines) == 1
def test_report_with_successful_finish_message_argument(tmpdir):
reporter = Event(event_log=tmpdir / "event_log")
job1 = Job({"name": "job1", "stdout": "stdout", "stderr": "stderr"}, 0)
reporter.report(Init([job1], 1, 19, ee_id="ee_id", real_id=0, stage_id=0))
reporter.report(Running(job1, 100, 10))
reporter.report(Finish())
with open(reporter._event_log, "r") as f:
lines = f.readlines()
assert len(lines) == 3
event = json.loads(lines[2])
assert event["type"] == _FM_STEP_SUCCESS
def test_report_with_failed_finish_message_argument(tmpdir):
reporter = Event(event_log=tmpdir / "event_log")
job1 = Job({"name": "job1", "stdout": "stdout", "stderr": "stderr"}, 0)
reporter.report(Init([job1], 1, 19, ee_id="ee_id", real_id=0, stage_id=0))
reporter.report(Running(job1, 100, 10))
reporter.report(Finish().with_error("massive_failure"))
with open(reporter._event_log, "r") as f:
lines = f.readlines()
assert len(lines) == 3
event = json.loads(lines[2])
assert event["type"] == _FM_STEP_FAILURE
assert event["data"]["error_msg"] == "massive_failure"
def test_report_with_successful_exit_message_argument(unused_tcp_port):
host = "localhost"
url = f"ws://{host}:{unused_tcp_port}"
reporter = Event(evaluator_url=url)
job1 = Job({"name": "job1", "stdout": "stdout", "stderr": "stderr"}, 0)
lines = []
with _mock_ws_thread(host, unused_tcp_port, lines):
reporter.report(Init([job1], 1, 19, ee_id="ee_id", real_id=0, step_id=0))
reporter.report(Exited(job1, 0))
reporter.report(Finish().with_error("failed"))
assert len(lines) == 1
event = json.loads(lines[0])
assert event["type"] == _FM_JOB_SUCCESS
def test_job_dispatch_kills_itself_after_unsuccessful_job(self):
jobs_json = json.dumps({"ee_id": "_id_"})
with patch("job_runner.cli.os") as mock_os, patch(
"job_runner.cli.open",
new=mock_open(read_data=jobs_json)) as mock_file, patch(
"job_runner.cli.JobRunner") as mock_runner:
mock_runner.return_value.run.return_value = [
Init([], 0, 0),
Finish().with_error("overall bad run"),
]
mock_os.getpgid.return_value = 17
main(["script.py", "/foo/bar/baz"])
mock_os.killpg.assert_called_with(17, signal.SIGKILL)
def test_report_with_running_message_argument(unused_tcp_port):
host = "localhost"
url = f"ws://{host}:{unused_tcp_port}"
reporter = Event(evaluator_url=url)
job1 = Job({"name": "job1", "stdout": "stdout", "stderr": "stderr"}, 0)
lines = []
with _mock_ws_thread(host, unused_tcp_port, lines):
reporter.report(Init([job1], 1, 19, ee_id="ee_id", real_id=0, step_id=0))
reporter.report(Running(job1, 100, 10))
reporter.report(Finish())
assert len(lines) == 1
event = json.loads(lines[0])
assert event["type"] == _FM_JOB_RUNNING
assert event["data"]["max_memory_usage"] == 100
assert event["data"]["current_memory_usage"] == 10
def test_report_with_successful_start_message_argument(unused_tcp_port):
host = "localhost"
url = f"ws://{host}:{unused_tcp_port}"
reporter = Event(evaluator_url=url)
job1 = Job({"name": "job1", "stdout": "stdout", "stderr": "stderr"}, 0)
lines = []
with _mock_ws_thread(host, unused_tcp_port, lines):
reporter.report(Init([job1], 1, 19, ee_id="ee_id", real_id=0, step_id=0))
reporter.report(Start(job1))
reporter.report(Finish())
assert len(lines) == 1
event = json.loads(lines[0])
assert event["type"] == _FM_JOB_START
assert event["source"] == "/ert/ee/ee_id/real/0/step/0/job/0"
assert os.path.basename(event["data"]["stdout"]) == "stdout"
assert os.path.basename(event["data"]["stderr"]) == "stderr"
def test_job_dispatch_kills_itself_after_unsuccessful_job(unused_tcp_port):
host = "localhost"
port = unused_tcp_port
jobs_json = json.dumps({"ee_id": "_id_", "dispatch_url": f"ws://localhost:{port}"})
with patch("job_runner.cli.os") as mock_os, patch(
"job_runner.cli.open", new=mock_open(read_data=jobs_json)
) as mock_file, patch("job_runner.cli.JobRunner") as mock_runner:
mock_runner.return_value.run.return_value = [
Init([], 0, 0),
Finish().with_error("overall bad run"),
]
mock_os.getpgid.return_value = 17
with _mock_ws_thread(host, port, []):
main(["script.py", "/foo/bar/baz"])
mock_os.killpg.assert_called_with(17, signal.SIGKILL)
def test_report_with_failed_start_message_argument(unused_tcp_port):
host = "localhost"
url = f"ws://{host}:{unused_tcp_port}"
reporter = Event(evaluator_url=url)
job1 = Job({"name": "job1", "stdout": "stdout", "stderr": "stderr"}, 0)
lines = []
with _mock_ws_thread(host, unused_tcp_port, lines):
reporter.report(Init([job1], 1, 19, ee_id="ee_id", real_id=0, step_id=0))
msg = Start(job1).with_error("massive_failure")
reporter.report(msg)
reporter.report(Finish())
assert len(lines) == 2
event = json.loads(lines[1])
assert event["type"] == _FM_JOB_FAILURE
assert event["data"]["error_msg"] == "massive_failure"
def test_failed_forward_model_not_reported(self, post_mock):
self.reporter.report(Finish().with_error("failed"))
self.assertFalse(post_mock.called, "post called on failed Finish")