def test_get_queue_info_no_run_id_ignore_completed(
self, m_ssh_exec_cmd, m_logger, config
):
job = watch_NEMO_hindcast._QstatHindcastJob(
Mock(name="ssh_client"),
Mock(name="sftp_client"),
"optimum",
config["run"]["hindcast hosts"]["optimum"]["users"],
Path(config["run"]["hindcast hosts"]["optimum"]["scratch dir"]),
)
m_ssh_exec_cmd.return_value = textwrap.dedent(
"""\
admin.default.domain:
Req'd Req'd Elap
Job ID Username Queue Jobname SessID NDS TSK Memory Time S Time
----------------------- ----------- -------- ---------------- ------ ----- ------ ------ --------- - ---------
62991.admin.default.do dlatorne mpi 01jan13hindcast 22390 14 280 -- 10:00:00 C --
62995.admin.default.do dlatorne mpi 11jan13hindcast 28434 14 280 -- 10:00:00 R 00:20:53
"""
)
queue_info = job._get_queue_info()
assert (
queue_info
== "62995.admin.default.do dlatorne mpi 11jan13hindcast 28434 14 280 -- 10:00:00 R 00:20:53"
)
def test_is_running_not_running(self, m_logger, config):
job = watch_NEMO_hindcast._QstatHindcastJob(
Mock(name="ssh_client"),
Mock(name="sftp_client"),
"optimum",
config["run"]["hindcast hosts"]["optimum"]["users"],
Path(config["run"]["hindcast hosts"]["optimum"]["scratch dir"]),
)
job._get_job_state = Mock(name="_get_job_state", return_value="UNKNOWN")
assert not job.is_running()
def test_get_queue_info_no_job_id_no_jobs(self, m_ssh_exec_cmd, m_logger, config):
job = watch_NEMO_hindcast._QstatHindcastJob(
Mock(name="ssh_client"),
Mock(name="sftp_client"),
"optimum",
config["run"]["hindcast hosts"]["optimum"]["users"],
Path(config["run"]["hindcast hosts"]["optimum"]["scratch dir"]),
)
m_ssh_exec_cmd.return_value = ""
with pytest.raises(nemo_nowcast.WorkerError):
job._get_queue_info()
def test_is_running(self, m_ssh_exec_cmd, m_logger, config):
job = watch_NEMO_hindcast._QstatHindcastJob(
Mock(name="ssh_client"),
Mock(name="sftp_client"),
"cedoptimumar",
config["run"]["hindcast hosts"]["graham"]["users"],
Path(config["run"]["hindcast hosts"]["graham"]["scratch dir"]),
)
job._get_job_state = Mock(name="_get_job_state", return_value="R")
job._report_progress = Mock(name="_report_progress")
assert job.is_running()
def test_is_running_no_time_step_file(self, m_ssh_exec_cmd, m_logger, config):
job = watch_NEMO_hindcast._QstatHindcastJob(
Mock(name="ssh_client"),
Mock(name="sftp_client"),
"optimum",
config["run"]["hindcast hosts"]["optimum"]["users"],
Path(config["run"]["hindcast hosts"]["optimum"]["scratch dir"]),
)
job._get_job_state = Mock(name="_get_job_state", return_value="R")
assert job.is_running()
assert m_logger.info.called
def test_get_queue_info_job_id_no_jobs(self, m_ssh_exec_cmd, m_logger, config):
job = watch_NEMO_hindcast._QstatHindcastJob(
Mock(name="ssh_client"),
Mock(name="sftp_client"),
"optimum",
config["run"]["hindcast hosts"]["optimum"]["users"],
Path(config["run"]["hindcast hosts"]["optimum"]["scratch dir"]),
)
job.job_id = "62991.admin"
m_ssh_exec_cmd.return_value = ""
queue_info = job._get_queue_info()
assert queue_info is None
def test_get_job_state_unknown(self, m_logger, exception, config):
job = watch_NEMO_hindcast._QstatHindcastJob(
Mock(name="ssh_client"),
Mock(name="sftp_client"),
"optimum",
config["run"]["hindcast hosts"]["optimum"]["users"],
Path(config["run"]["hindcast hosts"]["optimum"]["scratch dir"]),
)
job._get_queue_info = Mock(name="_get_queue_info", return_value=None)
state = job._get_job_state()
assert m_logger.info.called
assert state == "UNKNOWN"
def test_is_running_no_ocean_output(self, m_ssh_exec_cmd, m_logger, config):
job = watch_NEMO_hindcast._QstatHindcastJob(
Mock(name="ssh_client"),
Mock(name="sftp_client"),
"optimum",
config["run"]["hindcast hosts"]["optimum"]["users"],
Path(config["run"]["hindcast hosts"]["optimum"]["scratch dir"]),
)
job._get_job_state = Mock(name="_get_job_state", return_value="R")
job._report_progress = Mock(name="_report_progress")
assert not job.is_running()
assert m_logger.error.called
def test_is_queued_job_not_found(self, m_logger, config):
job = watch_NEMO_hindcast._QstatHindcastJob(
Mock(name="ssh_client"),
Mock(name="sftp_client"),
"optimum",
config["run"]["hindcast hosts"]["optimum"]["users"],
Path(config["run"]["hindcast hosts"]["optimum"]["scratch dir"]),
)
job._get_queue_info = Mock(name="_get_queue_info", return_value=None)
with pytest.raises(nemo_nowcast.WorkerError):
job.is_queued()
assert m_logger.error.called
def test_ssh_exec_command(self, m_ssh_exec_cmd, m_logger, config):
job = watch_NEMO_hindcast._QstatHindcastJob(
Mock(name="ssh_client"),
Mock(name="sftp_client"),
"optimum",
config["run"]["hindcast hosts"]["optimum"]["users"],
Path(config["run"]["hindcast hosts"]["optimum"]["scratch dir"]),
)
stdout = job._ssh_exec_command(f"ls -dtr {job.scratch_dir}/*hindcast*")
assert (
stdout == "/scratch/hindcast/01jan13hindcast_2019-07-06T122741.810587-0700"
)
assert not m_logger.info.called
def test_get_job_state_running(self, m_logger, config):
job = watch_NEMO_hindcast._QstatHindcastJob(
Mock(name="ssh_client"),
Mock(name="sftp_client"),
"optimum",
config["run"]["hindcast hosts"]["optimum"]["users"],
Path(config["run"]["hindcast hosts"]["optimum"]["scratch dir"]),
)
job._get_queue_info = Mock(
name="_get_queue_info",
return_value=(
"62991.admin.default.do dlatorne mpi 01jan13hindcast 22390 14 280 -- 06:00:00 R 00:20:53"
),
)
state = job._get_job_state()
assert state == "R"
def test_report_progress(self, m_logger, config):
job = watch_NEMO_hindcast._QstatHindcastJob(
Mock(name="ssh_client"),
Mock(name="sftp_client"),
"optimum",
config["run"]["hindcast hosts"]["optimum"]["users"],
Path(config["run"]["hindcast hosts"]["optimum"]["scratch dir"]),
)
job.run_id = "11jan13hindcast"
job.it000 = 21601
job.itend = 43200
job.date0 = arrow.get("2013-01-11")
job.rdt = 40.0
job._report_progress("29697\n")
m_logger.info.assert_called_once_with(
f"11jan13hindcast on {job.host_name}: timestep: "
f"29697 = 2013-01-14 17:57:20 UTC, 37.5% complete"
)
def test_get_tmp_run_dir(self, m_logger, config):
job = watch_NEMO_hindcast._QstatHindcastJob(
Mock(name="ssh_client"),
Mock(name="sftp_client"),
"optimum",
config["run"]["hindcast hosts"]["optimum"]["users"],
Path(config["run"]["hindcast hosts"]["optimum"]["scratch dir"]),
)
job.run_id = "21dec16hindcast"
job._ssh_exec_command = Mock(
name="_ssh_exec_command",
return_value="/scratch/hindcast/01jan13hindcast_2019-07-06T122741.810587-0700",
)
job.get_tmp_run_dir()
assert job.tmp_run_dir == Path(
"/scratch/hindcast/01jan13hindcast_2019-07-06T122741.810587-0700"
)
assert m_logger.debug.called
def test_is_running_ocean_output_errors_cancel_run(
self, m_ssh_exec_cmd, m_logger, config
):
job = watch_NEMO_hindcast._QstatHindcastJob(
Mock(name="ssh_client"),
Mock(name="sftp_client"),
"optimum",
config["run"]["hindcast hosts"]["optimum"]["users"],
Path(config["run"]["hindcast hosts"]["optimum"]["scratch dir"]),
)
job._get_job_state = Mock(name="_get_job_state", return_value="R")
job._report_progress = Mock(name="_report_progress")
job._ssh_exec_command = Mock(name="_ssh_exec_command")
assert not job.is_running()
assert m_logger.error.called
job._ssh_exec_command.assert_called_once_with(
f"/usr/bin/qdel {job.job_id}",
f"{job.run_id} on {job.host_name}: cancelled {job.job_id}",
)
def test_get_run_info(self, m_f90nml_read, m_logger, config):
job = watch_NEMO_hindcast._QstatHindcastJob(
Mock(name="ssh_client"),
Mock(name="sftp_client"),
"optimum",
config["run"]["hindcast hosts"]["optimum"]["users"],
Path(config["run"]["hindcast hosts"]["optimum"]["scratch dir"]),
)
job.tmp_run_dir = Path(
"/scratch/hindcast/01jan13hindcast_2019-07-06T122741.810587-0700"
)
m_f90nml_read.return_value = {
"namrun": {"nn_it000": 1, "nn_itend": 2160, "nn_date0": "20130101"},
"namdom": {"rn_rdt": 40.0},
}
job.get_run_info()
assert job.it000 == 1
assert job.itend == 2160
assert job.date0 == arrow.get("2013-01-01")
assert job.rdt == 40.0
assert m_logger.debug.call_count == 2