def test_is_running_not_running(self, m_logger, config):
job = watch_NEMO_hindcast._SqueueHindcastJob(
Mock(name="ssh_client"),
Mock(name="sftp_client"),
"graham",
config["run"]["hindcast hosts"]["graham"]["users"],
Path(config["run"]["hindcast hosts"]["graham"]["scratch dir"]),
)
job._get_job_state = Mock(name="_get_job_state", return_value="UNKNOWN")
assert not job.is_running()
def test_is_running(self, m_ssh_exec_cmd, m_logger, config):
job = watch_NEMO_hindcast._SqueueHindcastJob(
Mock(name="ssh_client"),
Mock(name="sftp_client"),
"graham",
config["run"]["hindcast hosts"]["graham"]["users"],
Path(config["run"]["hindcast hosts"]["graham"]["scratch dir"]),
)
job._get_job_state = Mock(name="_get_job_state", return_value="RUNNING")
job._report_progress = Mock(name="_report_progress")
assert job.is_running()
def test_get_job_state_unknown(self, m_logger, exception, config):
job = watch_NEMO_hindcast._SqueueHindcastJob(
Mock(name="ssh_client"),
Mock(name="sftp_client"),
"graham",
config["run"]["hindcast hosts"]["graham"]["users"],
Path(config["run"]["hindcast hosts"]["graham"]["scratch dir"]),
)
job._get_queue_info = Mock(name="_get_queue_info", return_value=None)
state = job._get_job_state()
assert m_logger.info.called
assert state == "UNKNOWN"
def test_is_queued_job_not_found(self, m_logger, config):
job = watch_NEMO_hindcast._SqueueHindcastJob(
Mock(name="ssh_client"),
Mock(name="sftp_client"),
"cedar",
config["run"]["hindcast hosts"]["graham"]["users"],
Path(config["run"]["hindcast hosts"]["graham"]["scratch dir"]),
)
job._get_queue_info = Mock(name="_get_queue_info", return_value=None)
with pytest.raises(nemo_nowcast.WorkerError):
job.is_queued()
assert m_logger.error.called
def test_ssh_exec_command(self, m_ssh_exec_cmd, m_logger, config):
job = watch_NEMO_hindcast._SqueueHindcastJob(
Mock(name="ssh_client"),
Mock(name="sftp_client"),
"graham",
config["run"]["hindcast hosts"]["graham"]["users"],
Path(config["run"]["hindcast hosts"]["graham"]["scratch dir"]),
)
stdout = job._ssh_exec_command(f"ls -dtr {job.scratch_dir}/*hindcast*")
assert (
stdout == "/scratch/hindcast/01jan17hindcast_2018-10-07T141411.374009-0700"
)
assert not m_logger.info.called
def test_is_queued_not_pending(self, m_logger, config):
job = watch_NEMO_hindcast._SqueueHindcastJob(
Mock(name="ssh_client"),
Mock(name="sftp_client"),
"cedar",
config["run"]["hindcast hosts"]["graham"]["users"],
Path(config["run"]["hindcast hosts"]["graham"]["scratch dir"]),
)
job._get_queue_info = Mock(
name="_get_queue_info",
return_value="12426878 21dec16hindcast RUNNING None 2018-10-07T14:13:59",
)
assert not job.is_queued()
def test_get_completion_state_unknown(self, m_logger, config):
job = watch_NEMO_hindcast._SqueueHindcastJob(
Mock(name="ssh_client"),
Mock(name="sftp_client"),
"graham",
config["run"]["hindcast hosts"]["graham"]["users"],
Path(config["run"]["hindcast hosts"]["graham"]["scratch dir"]),
)
job._ssh_exec_command = Mock(
name="_ssh_exec_command", return_value="State\n----------\n"
)
state = job.get_completion_state()
assert state == "unknown"
assert m_logger.debug.called
def test_get_job_state_running(self, m_logger, config):
job = watch_NEMO_hindcast._SqueueHindcastJob(
Mock(name="ssh_client"),
Mock(name="sftp_client"),
"graham",
config["run"]["hindcast hosts"]["graham"]["users"],
Path(config["run"]["hindcast hosts"]["graham"]["scratch dir"]),
)
job._get_queue_info = Mock(
name="_get_queue_info",
return_value="12426878 21dec16hindcast RUNNING N/A 2018-10-07T14:13:59",
)
state = job._get_job_state()
assert state == "RUNNING"
def test_is_queued_pending_due_to_resources(self, m_logger, config):
job = watch_NEMO_hindcast._SqueueHindcastJob(
Mock(name="ssh_client"),
Mock(name="sftp_client"),
"cedar",
config["run"]["hindcast hosts"]["graham"]["users"],
Path(config["run"]["hindcast hosts"]["graham"]["scratch dir"]),
)
job._get_queue_info = Mock(
name="_get_queue_info",
return_value="12426878 21dec16hindcast PENDING resources N/A",
)
assert job.is_queued()
assert m_logger.info.call_args[0][0].endswith("pending due to resources")
def test_get_run_id(self, m_logger, config):
job = watch_NEMO_hindcast._SqueueHindcastJob(
Mock(name="ssh_client"),
Mock(name="sftp_client"),
"cedar",
config["run"]["hindcast hosts"]["graham"]["users"],
Path(config["run"]["hindcast hosts"]["graham"]["scratch dir"]),
)
job._get_queue_info = Mock(
name="_get_queue_info",
return_value="12426878 21dec16hindcast RUNNING None 2018-10-07T14:13:59",
)
job.get_run_id()
assert job.job_id == "12426878"
assert job.run_id == "21dec16hindcast"
assert m_logger.info.called
def test_get_completion_state(self, m_logger, job_state, expected, config):
job = watch_NEMO_hindcast._SqueueHindcastJob(
Mock(name="ssh_client"),
Mock(name="sftp_client"),
"graham",
config["run"]["hindcast hosts"]["graham"]["users"],
Path(config["run"]["hindcast hosts"]["graham"]["scratch dir"]),
)
job.run_id = "01mar17hindcast"
job._ssh_exec_command = Mock(
name="_ssh_exec_command", return_value=f"State\n----------\n{job_state}\n"
)
state = job.get_completion_state()
assert state == expected
m_logger.info.assert_called_once_with(
f"{job.run_id} on {job.host_name}: {job_state}"
)
def test_report_progress(self, m_logger, config):
job = watch_NEMO_hindcast._SqueueHindcastJob(
Mock(name="ssh_client"),
Mock(name="sftp_client"),
"graham",
config["run"]["hindcast hosts"]["graham"]["users"],
Path(config["run"]["hindcast hosts"]["graham"]["scratch dir"]),
)
job.run_id = "01oct18hindcast"
job.it000 = 1_658_881
job.itend = 1_682_640
job.date0 = arrow.get("2016-12-21")
job.rdt = 40.0
job._report_progress("1658943\n")
m_logger.info.assert_called_once_with(
f"01oct18hindcast on {job.host_name}: timestep: "
f"1658943 = 2016-12-21 00:41:20 UTC, 0.3% complete"
)
def test_get_tmp_run_dir(self, m_logger, config):
job = watch_NEMO_hindcast._SqueueHindcastJob(
Mock(name="ssh_client"),
Mock(name="sftp_client"),
"graham",
config["run"]["hindcast hosts"]["graham"]["users"],
Path(config["run"]["hindcast hosts"]["graham"]["scratch dir"]),
)
job.run_id = "21dec16hindcast"
job._ssh_exec_command = Mock(
name="_ssh_exec_command",
return_value="/scratch/hindcast/21dec16hindcast_2018-10-06T195251.255493-0700",
)
job.get_tmp_run_dir()
assert job.tmp_run_dir == Path(
"/scratch/hindcast/21dec16hindcast_2018-10-06T195251.255493-0700"
)
assert m_logger.debug.called
def test_is_running_ocean_output_errors_cancel_run(
self, m_ssh_exec_cmd, m_logger, config
):
job = watch_NEMO_hindcast._SqueueHindcastJob(
Mock(name="ssh_client"),
Mock(name="sftp_client"),
"graham",
config["run"]["hindcast hosts"]["graham"]["users"],
Path(config["run"]["hindcast hosts"]["graham"]["scratch dir"]),
)
job._get_job_state = Mock(name="_get_job_state", return_value="RUNNING")
job._report_progress = Mock(name="_report_progress")
job._ssh_exec_command = Mock(name="_ssh_exec_command")
assert not job.is_running()
assert m_logger.error.called
job._ssh_exec_command.assert_called_once_with(
f"/opt/software/slurm/bin/scancel {job.job_id}",
f"{job.run_id} on {job.host_name}: cancelled {job.job_id}",
)
def test_get_run_info(self, m_f90nml_read, m_logger, config):
job = watch_NEMO_hindcast._SqueueHindcastJob(
Mock(name="ssh_client"),
Mock(name="sftp_client"),
"graham",
config["run"]["hindcast hosts"]["graham"]["users"],
Path(config["run"]["hindcast hosts"]["graham"]["scratch dir"]),
)
job.tmp_run_dir = Path(
"/scratch/hindcast/21dec16hindcast_2018-10-06T195251.255493-0700"
)
m_f90nml_read.return_value = {
"namrun": {
"nn_it000": 1_658_881,
"nn_itend": 1_682_640,
"nn_date0": "20161221",
},
"namdom": {"rn_rdt": 40.0},
}
def test_handle_stuck_job(self, m_logger, config):
job = watch_NEMO_hindcast._SqueueHindcastJob(
Mock(name="ssh_client"),
Mock(name="sftp_client"),
"graham",
config["run"]["hindcast hosts"]["graham"]["users"],
Path(config["run"]["hindcast hosts"]["graham"]["scratch dir"]),
)
job.run_id = "21dec16hindcast"
job.tmp_run_dir = (
"/scratch/hindcast/21dec16hindcast_2018-10-06T195251.255493-0700"
)
job._ssh_exec_command = Mock(
name="_ssh_exec_command",
side_effect=[
"",
"/scratch/hindcast/01jan17hindcast_2018-10-07T141411.374009-0700",
"",
],
)
job._get_queue_info = Mock(
name="_get_queue_info",
return_value="12426878 21dec16hindcast PENDING resources 2018-10-07T14:13:59",
)
job._handle_stuck_job()
assert job._ssh_exec_command.call_args_list == [
call(
"/opt/software/slurm/bin/sbatch "
"/scratch/hindcast/21dec16hindcast_2018-10-06T195251.255493-0700/SalishSeaNEMO.sh",
"21dec16hindcast on graham: re-queued",
),
call(f"ls -dtr {job.scratch_dir}/*hindcast*"),
call(
"/opt/software/slurm/bin/sbatch -d afterok:12426878 "
"/scratch/hindcast/01jan17hindcast_2018-10-07T141411.374009-0700/SalishSeaNEMO.sh",
"01jan17hindcast on graham: re-queued",
),
]
assert m_logger.debug.called