def test_run_fails_using_exit_bash_builtin(self):
job = Job(
{
"name":
"exit 1",
"executable":
"/bin/bash",
"stdout":
"exit_out",
"stderr":
"exit_err",
"argList":
["-c", 'echo "failed with {}" 1>&2 ; exit {}'.format(1, 1)],
},
0,
)
statuses = list(job.run())
self.assertEqual(3, len(statuses), "Wrong statuses count")
self.assertEqual(1, statuses[2].exit_code,
"Exited status wrong exit_code")
self.assertEqual(
"Process exited with status code 1",
statuses[2].error_message,
"Exited status wrong error_message",
)
def test_run_with_defined_executable_but_missing(self):
executable = os.path.join(os.getcwd(), "this/is/not/a/file")
job = Job(
{
"name": "TEST_EXECUTABLE_NOT_FOUND",
"executable": executable,
"stdout": "mkdir_out",
"stderr": "mkdir_err",
}, 0)
with self.assertRaises(IOError):
for _ in job.run():
pass
def test_report_with_successful_start_message_argument(self):
msg = Start(
Job(
{
"name": "job1",
"stdout": "/stdout.0",
"stderr": "/stderr.0",
"argList": ["--foo", "1", "--bar", "2"],
"executable": "/bin/bash",
},
0,
))
self.reporter.status_dict = self.reporter._init_job_status_dict(
msg.timestamp, 0, [msg.job])
self.reporter.report(msg)
with open(self.reporter.STATUS_file, "r") as f:
self.assertIn("job1", f.readline(), "STATUS file missing job1")
with open(self.reporter.LOG_file, "r") as f:
self.assertIn(
"Calling: /bin/bash --foo 1 --bar 2",
f.readline(),
"""JOB_LOG file missing executable and arguments""",
)
with open(self.reporter.STATUS_json, "r") as f:
contents = "".join(f.readlines())
self.assertIn('"status": "Running"', contents,
"status.json missing Running status")
self.assertNotIn('"start_time": null', contents,
"start_time not set")
def __init__(self, jobs_data):
os.umask(int(jobs_data["umask"], 8))
self._data_root = jobs_data.get("DATA_ROOT")
if self._data_root:
os.environ["DATA_ROOT"] = self._data_root
self.simulation_id = jobs_data.get("run_id")
self.ee_id = jobs_data.get("ee_id")
self.real_id = jobs_data.get("real_id")
self.step_id = jobs_data.get("step_id")
self.ert_pid = jobs_data.get("ert_pid")
self.global_environment = jobs_data.get("global_environment")
self.global_update_path = jobs_data.get("global_update_path")
job_data_list = jobs_data["jobList"]
if self.simulation_id is not None:
os.environ["ERT_RUN_ID"] = self.simulation_id
self.jobs = []
for index, job_data in enumerate(job_data_list):
self.jobs.append(Job(job_data, index))
self._set_environment()
self._update_path()
def __init__(self, jobs_file=JOBS_FILE):
try:
with open(jobs_file, "r") as json_file:
jobs_data = json.load(json_file)
except ValueError as e:
raise IOError("Job Runner failed to load JSON-file.{}".format(
str(e)))
os.umask(int(jobs_data["umask"], 8))
self._data_root = jobs_data.get("DATA_ROOT")
if self._data_root:
os.environ["DATA_ROOT"] = self._data_root
self.simulation_id = jobs_data.get("run_id")
self.ert_pid = jobs_data.get("ert_pid")
self.global_environment = jobs_data.get("global_environment")
self.global_update_path = jobs_data.get("global_update_path")
job_data_list = jobs_data["jobList"]
if self.simulation_id is not None:
os.environ["ERT_RUN_ID"] = self.simulation_id
self.jobs = []
for index, job_data in enumerate(job_data_list):
self.jobs.append(Job(job_data, index))
self._set_environment()
self._update_path()
def test_init_job_no_std():
job = Job(
{},
0,
)
assert job.std_err is None
assert job.std_out is None
def test_report_with_failed_start_message_argument(self):
msg = Start(Job({"name": "job1"}, 0)).with_error("massive_failure")
self.reporter.status_dict = self.reporter._init_job_status_dict(
msg.timestamp, 0, [msg.job])
self.reporter.report(msg)
with open(self.reporter.STATUS_file, "r") as f:
self.assertIn(
"EXIT: {}/{}".format(-10, "massive_failure"),
f.readline(),
"STATUS file missing EXIT message",
)
with open(self.reporter.STATUS_json, "r") as f:
contents = "".join(f.readlines())
self.assertIn('"status": "Failure"', contents,
"status.json missing Failure status")
self.assertIn(
'"error": "massive_failure"',
contents,
"status.json missing error message",
)
self.assertIsNotNone(
self.reporter.status_dict["jobs"][0]["end_time"],
"end_time not set for job1",
)
def test_run_with_defined_executable_no_exec_bit(self):
non_executable = os.path.join(os.getcwd(), "foo")
with open(non_executable, "a"):
pass
job = Job(
{
"name": "TEST_EXECUTABLE_NOT_EXECUTABLE",
"executable": non_executable,
"stdout": "mkdir_out",
"stderr": "mkdir_err",
}, 0)
with self.assertRaises(IOError):
for _ in job.run():
pass
def test_report_with_failed_exit_message_argument(self):
msg = Exited(Job({"name": "job1"}, 0), 1).with_error("massive_failure")
self.reporter.status_dict = self.reporter._init_job_status_dict(
msg.timestamp, 0, [msg.job])
self.reporter.report(msg)
with open(self.reporter.STATUS_file, "r") as f:
self.assertIn("EXIT: {}/{}".format(1, "massive_failure"),
f.readline())
with open(self.reporter.ERROR_file, "r") as f:
contents = "".join(f.readlines())
self.assertIn("<job>job1</job>", contents,
"ERROR file missing job")
self.assertIn(
"<reason>massive_failure</reason>",
contents,
"ERROR file missing reason",
)
self.assertIn(
"<stderr: Not redirected>",
contents,
"ERROR had invalid stderr information",
)
with open(self.reporter.STATUS_json, "r") as f:
contents = "".join(f.readlines())
self.assertIn('"status": "Failure"', contents,
"status.json missing Failure status")
self.assertIn(
'"error": "massive_failure"',
contents,
"status.json missing error message",
)
self.assertIsNotNone(self.reporter.status_dict["jobs"][0]["end_time"])
def test_report_startup_clearing_of_event_log_file(tmpdir):
reporter1 = Event(event_log=tmpdir / "event_log")
job1 = Job({"name": "job1", "stdout": "stdout", "stderr": "stderr"}, 0)
reporter1.report(Init([job1], 1, 19, ee_id="ee_id", real_id=0, stage_id=0))
reporter2 = Event(event_log=tmpdir / "event_log")
assert os.path.getsize(tmpdir / "event_log") == 0
def test_makedirs(monkeypatch, tmp_path):
"""
Test that the directories for the output process streams are created if
they don't exist
"""
monkeypatch.chdir(tmp_path)
job = Job(
{
"executable": "/usr/bin/true",
"stdout": "a/file",
"stderr": "b/c/file",
},
0,
)
for _ in job.run():
pass
assert (tmp_path / "a/file").is_file()
assert (tmp_path / "b/c/file").is_file()
def test_init_job_with_std():
job = Job(
{
"stdout": "exit_out",
"stderr": "exit_err",
},
0,
)
assert job.std_err == "exit_err"
assert job.std_out == "exit_out"
def test_run_with_process_failing(
self, mock_process, mock_popen, mock_assert_file_executable
):
job = Job({}, 0)
type(mock_process.return_value.memory_info.return_value).rss = PropertyMock(
return_value=10
)
mock_process.return_value.wait.return_value = 9
run = job.run()
self.assertIsInstance(next(run), Start, "run did not yield Start message")
self.assertIsInstance(next(run), Running, "run did not yield Running message")
exited = next(run)
self.assertIsInstance(exited, Exited, "run did not yield Exited message")
self.assertEqual(9, exited.exit_code, "Exited message had unexpected exit code")
with self.assertRaises(StopIteration):
next(run)
def test_report_with_successful_exit_message_argument(tmpdir):
reporter = Event(event_log=tmpdir / "event_log")
job1 = Job({"name": "job1", "stdout": "stdout", "stderr": "stderr"}, 0)
reporter.report(Init([job1], 1, 19, ee_id="ee_id", real_id=0, stage_id=0))
reporter.report(Exited(job1, 0))
with open(reporter._event_log, "r") as f:
lines = f.readlines()
assert len(lines) == 2
event = json.loads(lines[1])
assert event["type"] == _FM_JOB_SUCCESS
def test_report_with_successful_exit_message_argument(self):
msg = Exited(Job({"name": "job1"}, 0), 0)
self.reporter.status_dict = self.reporter._init_job_status_dict(
msg.timestamp, 0, [msg.job])
self.reporter.report(msg)
with open(self.reporter.STATUS_json, "r") as f:
contents = "".join(f.readlines())
self.assertIn('"status": "Success"', contents,
"status.json missing Success status")
def test_report_with_failed_exit_message_argument(tmpdir):
reporter = Event(event_log=tmpdir / "event_log")
job1 = Job({"name": "job1", "stdout": "stdout", "stderr": "stderr"}, 0)
reporter.report(Init([job1], 1, 19, ee_id="ee_id", real_id=0, stage_id=0))
reporter.report(Exited(job1, 1).with_error("massive_failure"))
with open(reporter._event_log, "r") as f:
lines = f.readlines()
assert len(lines) == 2
event = json.loads(lines[1])
assert event["type"] == _FM_JOB_FAILURE
assert event["data"]["error_msg"] == "massive_failure"
def test_report_only_job_running_for_successful_run(unused_tcp_port):
host = "localhost"
url = f"ws://{host}:{unused_tcp_port}"
reporter = Event(evaluator_url=url)
job1 = Job({"name": "job1", "stdout": "stdout", "stderr": "stderr"}, 0)
lines = []
with _mock_ws_thread(host, unused_tcp_port, lines):
reporter.report(Init([job1], 1, 19, ee_id="ee_id", real_id=0, step_id=0))
reporter.report(Running(job1, 100, 10))
reporter.report(Finish())
assert len(lines) == 1
def test_report_with_failed_finish_message_argument(unused_tcp_port):
host = "localhost"
url = f"ws://{host}:{unused_tcp_port}"
reporter = Event(evaluator_url=url)
job1 = Job({"name": "job1", "stdout": "stdout", "stderr": "stderr"}, 0)
lines = []
with _mock_ws_thread(host, unused_tcp_port, lines):
reporter.report(Init([job1], 1, 19, ee_id="ee_id", real_id=0, step_id=0))
reporter.report(Running(job1, 100, 10))
reporter.report(Finish().with_error("massive_failure"))
assert len(lines) == 1
def test_report_with_successful_finish_message_argument(tmpdir):
reporter = Event(event_log=tmpdir / "event_log")
job1 = Job({"name": "job1", "stdout": "stdout", "stderr": "stderr"}, 0)
reporter.report(Init([job1], 1, 19, ee_id="ee_id", real_id=0, stage_id=0))
reporter.report(Running(job1, 100, 10))
reporter.report(Finish())
with open(reporter._event_log, "r") as f:
lines = f.readlines()
assert len(lines) == 3
event = json.loads(lines[2])
assert event["type"] == _FM_STEP_SUCCESS
def test_report_with_running_message_argument(tmpdir):
reporter = Event(event_log=tmpdir / "event_log")
job1 = Job({"name": "job1", "stdout": "stdout", "stderr": "stderr"}, 0)
reporter.report(Init([job1], 1, 19, ee_id="ee_id", real_id=0, stage_id=0))
reporter.report(Running(job1, 100, 10))
with open(reporter._event_log, "r") as f:
lines = f.readlines()
assert len(lines) == 2
event = json.loads(lines[1])
assert event["type"] == _FM_JOB_RUNNING
assert event["data"]["max_memory_usage"] == 100
assert event["data"]["current_memory_usage"] == 10
def test_report_with_successful_start_message_argument(tmpdir):
reporter = Event(event_log=tmpdir / "event_log")
job1 = Job({"name": "job1", "stdout": "stdout", "stderr": "stderr"}, 0)
reporter.report(Init([job1], 1, 19, ee_id="ee_id", real_id=0, stage_id=0))
msg = Start(job1)
reporter.report(msg)
with open(reporter._event_log, "r") as f:
lines = f.readlines()
assert len(lines) == 2
event = json.loads(lines[1])
assert event["type"] == _FM_JOB_START
assert event["source"] == "/ert/ee/ee_id/real/0/stage/0/step/0/job/0"
def test_report_with_successful_exit_message_argument(unused_tcp_port):
host = "localhost"
url = f"ws://{host}:{unused_tcp_port}"
reporter = Event(evaluator_url=url)
job1 = Job({"name": "job1", "stdout": "stdout", "stderr": "stderr"}, 0)
lines = []
with _mock_ws_thread(host, unused_tcp_port, lines):
reporter.report(Init([job1], 1, 19, ee_id="ee_id", real_id=0, step_id=0))
reporter.report(Exited(job1, 0))
reporter.report(Finish().with_error("failed"))
assert len(lines) == 1
event = json.loads(lines[0])
assert event["type"] == _FM_JOB_SUCCESS
def test_report_with_running_message_argument(self):
msg = Running(Job({"name": "job1"}, 0), 100, 10)
self.reporter.status_dict = self.reporter._init_job_status_dict(
msg.timestamp, 0, [msg.job])
self.reporter.report(msg)
with open(self.reporter.STATUS_json, "r") as f:
contents = "".join(f.readlines())
self.assertIn('"status": "Running"', contents,
"status.json missing status")
self.assertIn('"max_memory_usage": 100', contents,
"status.json missing max_memory_usage")
self.assertIn('"current_memory_usage": 10', contents,
"status.json missing current_memory_usage")
def test_failed_job_is_reported(self, post_mock):
self.reporter.start_time = dt.now()
job = Job(
{
"name": "failing job",
"executable": "/dev/null",
"argList": []
}, 0)
self.reporter.report(Exited(job, 9).with_error("failed"))
_, data = post_mock.call_args
self.assertTrue(post_mock.called, "post not called for failed Exit")
self.assertIn('"status": "exit"', data["data"], "no exit in data")
self.assertIn('"error": true', data["data"], "no set err flag in data")
def test_report_with_init_message_argument(self):
r = self.reporter
job1 = Job({"name": "job1", "stdout": "/stdout", "stderr": "/stderr"}, 0)
r.report(Init([job1], 1, 19))
with open(self.reporter.STATUS_file, "r") as f:
self.assertIn(
"Current host", f.readline(), "STATUS file missing expected value"
)
with open(self.reporter.STATUS_json, "r") as f:
contents = "".join(f.readlines())
self.assertIn('"name": "job1"', contents, "status.json missing job1")
self.assertIn(
'"status": "Waiting"', contents, "status.json missing Waiting status"
)
def test_report_with_running_message_argument(unused_tcp_port):
host = "localhost"
url = f"ws://{host}:{unused_tcp_port}"
reporter = Event(evaluator_url=url)
job1 = Job({"name": "job1", "stdout": "stdout", "stderr": "stderr"}, 0)
lines = []
with _mock_ws_thread(host, unused_tcp_port, lines):
reporter.report(Init([job1], 1, 19, ee_id="ee_id", real_id=0, step_id=0))
reporter.report(Running(job1, 100, 10))
reporter.report(Finish())
assert len(lines) == 1
event = json.loads(lines[0])
assert event["type"] == _FM_JOB_RUNNING
assert event["data"]["max_memory_usage"] == 100
assert event["data"]["current_memory_usage"] == 10
def test_report_with_successful_start_message_argument(unused_tcp_port):
host = "localhost"
url = f"ws://{host}:{unused_tcp_port}"
reporter = Event(evaluator_url=url)
job1 = Job({"name": "job1", "stdout": "stdout", "stderr": "stderr"}, 0)
lines = []
with _mock_ws_thread(host, unused_tcp_port, lines):
reporter.report(Init([job1], 1, 19, ee_id="ee_id", real_id=0, step_id=0))
reporter.report(Start(job1))
reporter.report(Finish())
assert len(lines) == 1
event = json.loads(lines[0])
assert event["type"] == _FM_JOB_START
assert event["source"] == "/ert/ee/ee_id/real/0/step/0/job/0"
assert os.path.basename(event["data"]["stdout"]) == "stdout"
assert os.path.basename(event["data"]["stderr"]) == "stderr"
def test_report_with_init_message_argument(tmpdir):
reporter = Event(event_log=tmpdir / "event_log")
job1 = Job({"name": "job1", "stdout": "stdout", "stderr": "stderr"}, 0)
reporter.report(Init([job1], 1, 19, ee_id="ee_id", real_id=0, stage_id=0))
with open(reporter._event_log, "r") as f:
lines = f.readlines()
assert len(lines) == 1
event = json.loads(lines[0])
job = event.get("data", {}).get("jobs", {}).get("0", {})
assert job
assert job["name"] == "job1"
assert job["stdout"].startswith("/") and job["stdout"].endswith(
"stdout")
assert job["stderr"].startswith("/") and job["stderr"].endswith(
"stderr")
assert event["type"] == _FM_STEP_START
assert event["source"] == "/ert/ee/ee_id/real/0/stage/0/step/0"
def test_report_with_failed_start_message_argument(unused_tcp_port):
host = "localhost"
url = f"ws://{host}:{unused_tcp_port}"
reporter = Event(evaluator_url=url)
job1 = Job({"name": "job1", "stdout": "stdout", "stderr": "stderr"}, 0)
lines = []
with _mock_ws_thread(host, unused_tcp_port, lines):
reporter.report(Init([job1], 1, 19, ee_id="ee_id", real_id=0, step_id=0))
msg = Start(job1).with_error("massive_failure")
reporter.report(msg)
reporter.report(Finish())
assert len(lines) == 2
event = json.loads(lines[1])
assert event["type"] == _FM_JOB_FAILURE
assert event["data"]["error_msg"] == "massive_failure"
def test_dump_error_file_with_stderr(self):
"""
Assert that, in the case of an stderr file, it is included in the XML
that constitutes the ERROR file.
The report system is left out, since this was tested in the fail case.
"""
with open("stderr.out.0", "a") as stderr:
stderr.write("Error:\n")
stderr.write("E_MASSIVE_FAILURE\n")
self.reporter._dump_error_file(
Job({"name": "job1", "stderr": "stderr.out.0"}, 0), "massive_failure"
)
with open(self.reporter.ERROR_file, "r") as f:
contents = "".join(f.readlines())
self.assertIn(
"E_MASSIVE_FAILURE", contents, "ERROR file missing stderr content"
)
self.assertIn("<stderr_file>", contents, "ERROR missing stderr_file part")
