def test_init_job_no_std():
job = Job(
{},
0,
)
assert job.std_err is None
assert job.std_out is None
def test_run_fails_using_exit_bash_builtin(self):
job = Job(
{
"name":
"exit 1",
"executable":
"/bin/bash",
"stdout":
"exit_out",
"stderr":
"exit_err",
"argList":
["-c", 'echo "failed with {}" 1>&2 ; exit {}'.format(1, 1)],
},
0,
)
statuses = list(job.run())
self.assertEqual(3, len(statuses), "Wrong statuses count")
self.assertEqual(1, statuses[2].exit_code,
"Exited status wrong exit_code")
self.assertEqual(
"Process exited with status code 1",
statuses[2].error_message,
"Exited status wrong error_message",
)
def __init__(self, jobs_file=JOBS_FILE):
try:
with open(jobs_file, "r") as json_file:
jobs_data = json.load(json_file)
except ValueError as e:
raise IOError("Job Runner failed to load JSON-file.{}".format(
str(e)))
os.umask(int(jobs_data["umask"], 8))
self._data_root = jobs_data.get("DATA_ROOT")
if self._data_root:
os.environ["DATA_ROOT"] = self._data_root
self.simulation_id = jobs_data.get("run_id")
self.ert_pid = jobs_data.get("ert_pid")
self.global_environment = jobs_data.get("global_environment")
self.global_update_path = jobs_data.get("global_update_path")
job_data_list = jobs_data["jobList"]
if self.simulation_id is not None:
os.environ["ERT_RUN_ID"] = self.simulation_id
self.jobs = []
for index, job_data in enumerate(job_data_list):
self.jobs.append(Job(job_data, index))
self._set_environment()
self._update_path()
def __init__(self, jobs_data):
os.umask(int(jobs_data["umask"], 8))
self._data_root = jobs_data.get("DATA_ROOT")
if self._data_root:
os.environ["DATA_ROOT"] = self._data_root
self.simulation_id = jobs_data.get("run_id")
self.ee_id = jobs_data.get("ee_id")
self.real_id = jobs_data.get("real_id")
self.step_id = jobs_data.get("step_id")
self.ert_pid = jobs_data.get("ert_pid")
self.global_environment = jobs_data.get("global_environment")
self.global_update_path = jobs_data.get("global_update_path")
job_data_list = jobs_data["jobList"]
if self.simulation_id is not None:
os.environ["ERT_RUN_ID"] = self.simulation_id
self.jobs = []
for index, job_data in enumerate(job_data_list):
self.jobs.append(Job(job_data, index))
self._set_environment()
self._update_path()
def test_report_with_successful_start_message_argument(self):
msg = Start(
Job(
{
"name": "job1",
"stdout": "/stdout.0",
"stderr": "/stderr.0",
"argList": ["--foo", "1", "--bar", "2"],
"executable": "/bin/bash",
},
0,
))
self.reporter.status_dict = self.reporter._init_job_status_dict(
msg.timestamp, 0, [msg.job])
self.reporter.report(msg)
with open(self.reporter.STATUS_file, "r") as f:
self.assertIn("job1", f.readline(), "STATUS file missing job1")
with open(self.reporter.LOG_file, "r") as f:
self.assertIn(
"Calling: /bin/bash --foo 1 --bar 2",
f.readline(),
"""JOB_LOG file missing executable and arguments""",
)
with open(self.reporter.STATUS_json, "r") as f:
contents = "".join(f.readlines())
self.assertIn('"status": "Running"', contents,
"status.json missing Running status")
self.assertNotIn('"start_time": null', contents,
"start_time not set")
def test_report_with_failed_exit_message_argument(self):
msg = Exited(Job({"name": "job1"}, 0), 1).with_error("massive_failure")
self.reporter.status_dict = self.reporter._init_job_status_dict(
msg.timestamp, 0, [msg.job])
self.reporter.report(msg)
with open(self.reporter.STATUS_file, "r") as f:
self.assertIn("EXIT: {}/{}".format(1, "massive_failure"),
f.readline())
with open(self.reporter.ERROR_file, "r") as f:
contents = "".join(f.readlines())
self.assertIn("<job>job1</job>", contents,
"ERROR file missing job")
self.assertIn(
"<reason>massive_failure</reason>",
contents,
"ERROR file missing reason",
)
self.assertIn(
"<stderr: Not redirected>",
contents,
"ERROR had invalid stderr information",
)
with open(self.reporter.STATUS_json, "r") as f:
contents = "".join(f.readlines())
self.assertIn('"status": "Failure"', contents,
"status.json missing Failure status")
self.assertIn(
'"error": "massive_failure"',
contents,
"status.json missing error message",
)
self.assertIsNotNone(self.reporter.status_dict["jobs"][0]["end_time"])
def test_report_with_failed_start_message_argument(self):
msg = Start(Job({"name": "job1"}, 0)).with_error("massive_failure")
self.reporter.status_dict = self.reporter._init_job_status_dict(
msg.timestamp, 0, [msg.job])
self.reporter.report(msg)
with open(self.reporter.STATUS_file, "r") as f:
self.assertIn(
"EXIT: {}/{}".format(-10, "massive_failure"),
f.readline(),
"STATUS file missing EXIT message",
)
with open(self.reporter.STATUS_json, "r") as f:
contents = "".join(f.readlines())
self.assertIn('"status": "Failure"', contents,
"status.json missing Failure status")
self.assertIn(
'"error": "massive_failure"',
contents,
"status.json missing error message",
)
self.assertIsNotNone(
self.reporter.status_dict["jobs"][0]["end_time"],
"end_time not set for job1",
)
def test_report_startup_clearing_of_event_log_file(tmpdir):
reporter1 = Event(event_log=tmpdir / "event_log")
job1 = Job({"name": "job1", "stdout": "stdout", "stderr": "stderr"}, 0)
reporter1.report(Init([job1], 1, 19, ee_id="ee_id", real_id=0, stage_id=0))
reporter2 = Event(event_log=tmpdir / "event_log")
assert os.path.getsize(tmpdir / "event_log") == 0
def test_init_job_with_std():
job = Job(
{
"stdout": "exit_out",
"stderr": "exit_err",
},
0,
)
assert job.std_err == "exit_err"
assert job.std_out == "exit_out"
def test_report_with_successful_exit_message_argument(tmpdir):
reporter = Event(event_log=tmpdir / "event_log")
job1 = Job({"name": "job1", "stdout": "stdout", "stderr": "stderr"}, 0)
reporter.report(Init([job1], 1, 19, ee_id="ee_id", real_id=0, stage_id=0))
reporter.report(Exited(job1, 0))
with open(reporter._event_log, "r") as f:
lines = f.readlines()
assert len(lines) == 2
event = json.loads(lines[1])
assert event["type"] == _FM_JOB_SUCCESS
def test_report_with_successful_exit_message_argument(self):
msg = Exited(Job({"name": "job1"}, 0), 0)
self.reporter.status_dict = self.reporter._init_job_status_dict(
msg.timestamp, 0, [msg.job])
self.reporter.report(msg)
with open(self.reporter.STATUS_json, "r") as f:
contents = "".join(f.readlines())
self.assertIn('"status": "Success"', contents,
"status.json missing Success status")
def test_status_file_is_correct(self):
"""The STATUS file is a file to which we append data about jobs as they
are run. So this involves multiple reports, and should be tested as
such.
See https://github.com/equinor/libres/issues/764
"""
j_1 = Job({"name": "j_1", "executable": "", "argList": []}, 0)
j_2 = Job({"name": "j_2", "executable": "", "argList": []}, 0)
init = Init([j_1, j_2], 1, 1)
start_j_1 = Start(j_1)
exited_j_1 = Exited(j_1, 0)
start_j_2 = Start(j_2)
exited_j_2 = Exited(j_2, 9).with_error("failed horribly")
for msg in [init, start_j_1, exited_j_1, start_j_2, exited_j_2]:
self.reporter.report(msg)
expected_j1_line = (
"{:32}: {start_ts:%H:%M:%S} .... {end_ts:%H:%M:%S} \n".
format( # noqa
j_1.name(),
start_ts=start_j_1.timestamp,
end_ts=exited_j_1.timestamp))
expected_j2_line = "{:32}: {start_ts:%H:%M:%S} .... {end_ts:%H:%M:%S} EXIT: {code}/{msg}\n".format( # noqa
j_2.name(),
start_ts=start_j_2.timestamp,
end_ts=exited_j_2.timestamp,
code=exited_j_2.exit_code,
msg=exited_j_2.error_message,
)
with open(self.reporter.STATUS_file, "r") as f:
for expected in [
"Current host",
expected_j1_line,
expected_j2_line,
]: # noqa
self.assertIn(expected, f.readline())
# EOF
self.assertEqual("", f.readline())
def test_report_with_failed_exit_message_argument(tmpdir):
reporter = Event(event_log=tmpdir / "event_log")
job1 = Job({"name": "job1", "stdout": "stdout", "stderr": "stderr"}, 0)
reporter.report(Init([job1], 1, 19, ee_id="ee_id", real_id=0, stage_id=0))
reporter.report(Exited(job1, 1).with_error("massive_failure"))
with open(reporter._event_log, "r") as f:
lines = f.readlines()
assert len(lines) == 2
event = json.loads(lines[1])
assert event["type"] == _FM_JOB_FAILURE
assert event["data"]["error_msg"] == "massive_failure"
def test_report_with_failed_finish_message_argument(unused_tcp_port):
host = "localhost"
url = f"ws://{host}:{unused_tcp_port}"
reporter = Event(evaluator_url=url)
job1 = Job({"name": "job1", "stdout": "stdout", "stderr": "stderr"}, 0)
lines = []
with _mock_ws_thread(host, unused_tcp_port, lines):
reporter.report(Init([job1], 1, 19, ee_id="ee_id", real_id=0, step_id=0))
reporter.report(Running(job1, 100, 10))
reporter.report(Finish().with_error("massive_failure"))
assert len(lines) == 1
def test_report_only_job_running_for_successful_run(unused_tcp_port):
host = "localhost"
url = f"ws://{host}:{unused_tcp_port}"
reporter = Event(evaluator_url=url)
job1 = Job({"name": "job1", "stdout": "stdout", "stderr": "stderr"}, 0)
lines = []
with _mock_ws_thread(host, unused_tcp_port, lines):
reporter.report(Init([job1], 1, 19, ee_id="ee_id", real_id=0, step_id=0))
reporter.report(Running(job1, 100, 10))
reporter.report(Finish())
assert len(lines) == 1
def test_report_with_successful_finish_message_argument(tmpdir):
reporter = Event(event_log=tmpdir / "event_log")
job1 = Job({"name": "job1", "stdout": "stdout", "stderr": "stderr"}, 0)
reporter.report(Init([job1], 1, 19, ee_id="ee_id", real_id=0, stage_id=0))
reporter.report(Running(job1, 100, 10))
reporter.report(Finish())
with open(reporter._event_log, "r") as f:
lines = f.readlines()
assert len(lines) == 3
event = json.loads(lines[2])
assert event["type"] == _FM_STEP_SUCCESS
def test_run_with_defined_executable_but_missing(self):
executable = os.path.join(os.getcwd(), "this/is/not/a/file")
job = Job(
{
"name": "TEST_EXECUTABLE_NOT_FOUND",
"executable": executable,
"stdout": "mkdir_out",
"stderr": "mkdir_err",
}, 0)
with self.assertRaises(IOError):
for _ in job.run():
pass
def test_report_with_running_message_argument(tmpdir):
reporter = Event(event_log=tmpdir / "event_log")
job1 = Job({"name": "job1", "stdout": "stdout", "stderr": "stderr"}, 0)
reporter.report(Init([job1], 1, 19, ee_id="ee_id", real_id=0, stage_id=0))
reporter.report(Running(job1, 100, 10))
with open(reporter._event_log, "r") as f:
lines = f.readlines()
assert len(lines) == 2
event = json.loads(lines[1])
assert event["type"] == _FM_JOB_RUNNING
assert event["data"]["max_memory_usage"] == 100
assert event["data"]["current_memory_usage"] == 10
def test_report_with_successful_start_message_argument(tmpdir):
reporter = Event(event_log=tmpdir / "event_log")
job1 = Job({"name": "job1", "stdout": "stdout", "stderr": "stderr"}, 0)
reporter.report(Init([job1], 1, 19, ee_id="ee_id", real_id=0, stage_id=0))
msg = Start(job1)
reporter.report(msg)
with open(reporter._event_log, "r") as f:
lines = f.readlines()
assert len(lines) == 2
event = json.loads(lines[1])
assert event["type"] == _FM_JOB_START
assert event["source"] == "/ert/ee/ee_id/real/0/stage/0/step/0/job/0"
def test_report_with_successful_exit_message_argument(unused_tcp_port):
host = "localhost"
url = f"ws://{host}:{unused_tcp_port}"
reporter = Event(evaluator_url=url)
job1 = Job({"name": "job1", "stdout": "stdout", "stderr": "stderr"}, 0)
lines = []
with _mock_ws_thread(host, unused_tcp_port, lines):
reporter.report(Init([job1], 1, 19, ee_id="ee_id", real_id=0, step_id=0))
reporter.report(Exited(job1, 0))
reporter.report(Finish().with_error("failed"))
assert len(lines) == 1
event = json.loads(lines[0])
assert event["type"] == _FM_JOB_SUCCESS
def test_failed_job_is_reported(self, post_mock):
self.reporter.start_time = dt.now()
job = Job(
{
"name": "failing job",
"executable": "/dev/null",
"argList": []
}, 0)
self.reporter.report(Exited(job, 9).with_error("failed"))
_, data = post_mock.call_args
self.assertTrue(post_mock.called, "post not called for failed Exit")
self.assertIn('"status": "exit"', data["data"], "no exit in data")
self.assertIn('"error": true', data["data"], "no set err flag in data")
def test_report_with_running_message_argument(self):
msg = Running(Job({"name": "job1"}, 0), 100, 10)
self.reporter.status_dict = self.reporter._init_job_status_dict(
msg.timestamp, 0, [msg.job])
self.reporter.report(msg)
with open(self.reporter.STATUS_json, "r") as f:
contents = "".join(f.readlines())
self.assertIn('"status": "Running"', contents,
"status.json missing status")
self.assertIn('"max_memory_usage": 100', contents,
"status.json missing max_memory_usage")
self.assertIn('"current_memory_usage": 10', contents,
"status.json missing current_memory_usage")
def test_report_with_init_message_argument(self):
r = self.reporter
job1 = Job({"name": "job1", "stdout": "/stdout", "stderr": "/stderr"}, 0)
r.report(Init([job1], 1, 19))
with open(self.reporter.STATUS_file, "r") as f:
self.assertIn(
"Current host", f.readline(), "STATUS file missing expected value"
)
with open(self.reporter.STATUS_json, "r") as f:
contents = "".join(f.readlines())
self.assertIn('"name": "job1"', contents, "status.json missing job1")
self.assertIn(
'"status": "Waiting"', contents, "status.json missing Waiting status"
)
def test_run_with_defined_executable_no_exec_bit(self):
non_executable = os.path.join(os.getcwd(), "foo")
with open(non_executable, "a"):
pass
job = Job(
{
"name": "TEST_EXECUTABLE_NOT_EXECUTABLE",
"executable": non_executable,
"stdout": "mkdir_out",
"stderr": "mkdir_err",
}, 0)
with self.assertRaises(IOError):
for _ in job.run():
pass
def test_report_with_successful_start_message_argument(unused_tcp_port):
host = "localhost"
url = f"ws://{host}:{unused_tcp_port}"
reporter = Event(evaluator_url=url)
job1 = Job({"name": "job1", "stdout": "stdout", "stderr": "stderr"}, 0)
lines = []
with _mock_ws_thread(host, unused_tcp_port, lines):
reporter.report(Init([job1], 1, 19, ee_id="ee_id", real_id=0, step_id=0))
reporter.report(Start(job1))
reporter.report(Finish())
assert len(lines) == 1
event = json.loads(lines[0])
assert event["type"] == _FM_JOB_START
assert event["source"] == "/ert/ee/ee_id/real/0/step/0/job/0"
assert os.path.basename(event["data"]["stdout"]) == "stdout"
assert os.path.basename(event["data"]["stderr"]) == "stderr"
def test_report_with_running_message_argument(unused_tcp_port):
host = "localhost"
url = f"ws://{host}:{unused_tcp_port}"
reporter = Event(evaluator_url=url)
job1 = Job({"name": "job1", "stdout": "stdout", "stderr": "stderr"}, 0)
lines = []
with _mock_ws_thread(host, unused_tcp_port, lines):
reporter.report(Init([job1], 1, 19, ee_id="ee_id", real_id=0, step_id=0))
reporter.report(Running(job1, 100, 10))
reporter.report(Finish())
assert len(lines) == 1
event = json.loads(lines[0])
assert event["type"] == _FM_JOB_RUNNING
assert event["data"]["max_memory_usage"] == 100
assert event["data"]["current_memory_usage"] == 10
def test_makedirs(monkeypatch, tmp_path):
"""
Test that the directories for the output process streams are created if
they don't exist
"""
monkeypatch.chdir(tmp_path)
job = Job(
{
"executable": "/usr/bin/true",
"stdout": "a/file",
"stderr": "b/c/file",
},
0,
)
for _ in job.run():
pass
assert (tmp_path / "a/file").is_file()
assert (tmp_path / "b/c/file").is_file()
def test_run_with_process_failing(
self, mock_process, mock_popen, mock_assert_file_executable
):
job = Job({}, 0)
type(mock_process.return_value.memory_info.return_value).rss = PropertyMock(
return_value=10
)
mock_process.return_value.wait.return_value = 9
run = job.run()
self.assertIsInstance(next(run), Start, "run did not yield Start message")
self.assertIsInstance(next(run), Running, "run did not yield Running message")
exited = next(run)
self.assertIsInstance(exited, Exited, "run did not yield Exited message")
self.assertEqual(9, exited.exit_code, "Exited message had unexpected exit code")
with self.assertRaises(StopIteration):
next(run)
def test_report_with_init_message_argument(tmpdir):
reporter = Event(event_log=tmpdir / "event_log")
job1 = Job({"name": "job1", "stdout": "stdout", "stderr": "stderr"}, 0)
reporter.report(Init([job1], 1, 19, ee_id="ee_id", real_id=0, stage_id=0))
with open(reporter._event_log, "r") as f:
lines = f.readlines()
assert len(lines) == 1
event = json.loads(lines[0])
job = event.get("data", {}).get("jobs", {}).get("0", {})
assert job
assert job["name"] == "job1"
assert job["stdout"].startswith("/") and job["stdout"].endswith(
"stdout")
assert job["stderr"].startswith("/") and job["stderr"].endswith(
"stderr")
assert event["type"] == _FM_STEP_START
assert event["source"] == "/ert/ee/ee_id/real/0/stage/0/step/0"
def test_report_with_failed_start_message_argument(unused_tcp_port):
host = "localhost"
url = f"ws://{host}:{unused_tcp_port}"
reporter = Event(evaluator_url=url)
job1 = Job({"name": "job1", "stdout": "stdout", "stderr": "stderr"}, 0)
lines = []
with _mock_ws_thread(host, unused_tcp_port, lines):
reporter.report(Init([job1], 1, 19, ee_id="ee_id", real_id=0, step_id=0))
msg = Start(job1).with_error("massive_failure")
reporter.report(msg)
reporter.report(Finish())
assert len(lines) == 2
event = json.loads(lines[1])
assert event["type"] == _FM_JOB_FAILURE
assert event["data"]["error_msg"] == "massive_failure"