Пример #1
0
 def test_is_running_not_running(self, m_logger, config):
     job = watch_NEMO_hindcast._SqueueHindcastJob(
         Mock(name="ssh_client"),
         Mock(name="sftp_client"),
         "graham",
         config["run"]["hindcast hosts"]["graham"]["users"],
         Path(config["run"]["hindcast hosts"]["graham"]["scratch dir"]),
     )
     job._get_job_state = Mock(name="_get_job_state", return_value="UNKNOWN")
     assert not job.is_running()
Пример #2
0
 def test_is_running(self, m_ssh_exec_cmd, m_logger, config):
     job = watch_NEMO_hindcast._SqueueHindcastJob(
         Mock(name="ssh_client"),
         Mock(name="sftp_client"),
         "graham",
         config["run"]["hindcast hosts"]["graham"]["users"],
         Path(config["run"]["hindcast hosts"]["graham"]["scratch dir"]),
     )
     job._get_job_state = Mock(name="_get_job_state", return_value="RUNNING")
     job._report_progress = Mock(name="_report_progress")
     assert job.is_running()
Пример #3
0
 def test_get_job_state_unknown(self, m_logger, exception, config):
     job = watch_NEMO_hindcast._SqueueHindcastJob(
         Mock(name="ssh_client"),
         Mock(name="sftp_client"),
         "graham",
         config["run"]["hindcast hosts"]["graham"]["users"],
         Path(config["run"]["hindcast hosts"]["graham"]["scratch dir"]),
     )
     job._get_queue_info = Mock(name="_get_queue_info", return_value=None)
     state = job._get_job_state()
     assert m_logger.info.called
     assert state == "UNKNOWN"
Пример #4
0
 def test_is_queued_job_not_found(self, m_logger, config):
     job = watch_NEMO_hindcast._SqueueHindcastJob(
         Mock(name="ssh_client"),
         Mock(name="sftp_client"),
         "cedar",
         config["run"]["hindcast hosts"]["graham"]["users"],
         Path(config["run"]["hindcast hosts"]["graham"]["scratch dir"]),
     )
     job._get_queue_info = Mock(name="_get_queue_info", return_value=None)
     with pytest.raises(nemo_nowcast.WorkerError):
         job.is_queued()
     assert m_logger.error.called
Пример #5
0
 def test_ssh_exec_command(self, m_ssh_exec_cmd, m_logger, config):
     job = watch_NEMO_hindcast._SqueueHindcastJob(
         Mock(name="ssh_client"),
         Mock(name="sftp_client"),
         "graham",
         config["run"]["hindcast hosts"]["graham"]["users"],
         Path(config["run"]["hindcast hosts"]["graham"]["scratch dir"]),
     )
     stdout = job._ssh_exec_command(f"ls -dtr {job.scratch_dir}/*hindcast*")
     assert (
         stdout == "/scratch/hindcast/01jan17hindcast_2018-10-07T141411.374009-0700"
     )
     assert not m_logger.info.called
Пример #6
0
 def test_is_queued_not_pending(self, m_logger, config):
     job = watch_NEMO_hindcast._SqueueHindcastJob(
         Mock(name="ssh_client"),
         Mock(name="sftp_client"),
         "cedar",
         config["run"]["hindcast hosts"]["graham"]["users"],
         Path(config["run"]["hindcast hosts"]["graham"]["scratch dir"]),
     )
     job._get_queue_info = Mock(
         name="_get_queue_info",
         return_value="12426878 21dec16hindcast RUNNING None 2018-10-07T14:13:59",
     )
     assert not job.is_queued()
Пример #7
0
 def test_get_completion_state_unknown(self, m_logger, config):
     job = watch_NEMO_hindcast._SqueueHindcastJob(
         Mock(name="ssh_client"),
         Mock(name="sftp_client"),
         "graham",
         config["run"]["hindcast hosts"]["graham"]["users"],
         Path(config["run"]["hindcast hosts"]["graham"]["scratch dir"]),
     )
     job._ssh_exec_command = Mock(
         name="_ssh_exec_command", return_value="State\n----------\n"
     )
     state = job.get_completion_state()
     assert state == "unknown"
     assert m_logger.debug.called
Пример #8
0
 def test_get_job_state_running(self, m_logger, config):
     job = watch_NEMO_hindcast._SqueueHindcastJob(
         Mock(name="ssh_client"),
         Mock(name="sftp_client"),
         "graham",
         config["run"]["hindcast hosts"]["graham"]["users"],
         Path(config["run"]["hindcast hosts"]["graham"]["scratch dir"]),
     )
     job._get_queue_info = Mock(
         name="_get_queue_info",
         return_value="12426878 21dec16hindcast RUNNING N/A 2018-10-07T14:13:59",
     )
     state = job._get_job_state()
     assert state == "RUNNING"
Пример #9
0
 def test_is_queued_pending_due_to_resources(self, m_logger, config):
     job = watch_NEMO_hindcast._SqueueHindcastJob(
         Mock(name="ssh_client"),
         Mock(name="sftp_client"),
         "cedar",
         config["run"]["hindcast hosts"]["graham"]["users"],
         Path(config["run"]["hindcast hosts"]["graham"]["scratch dir"]),
     )
     job._get_queue_info = Mock(
         name="_get_queue_info",
         return_value="12426878 21dec16hindcast PENDING resources N/A",
     )
     assert job.is_queued()
     assert m_logger.info.call_args[0][0].endswith("pending due to resources")
Пример #10
0
 def test_get_run_id(self, m_logger, config):
     job = watch_NEMO_hindcast._SqueueHindcastJob(
         Mock(name="ssh_client"),
         Mock(name="sftp_client"),
         "cedar",
         config["run"]["hindcast hosts"]["graham"]["users"],
         Path(config["run"]["hindcast hosts"]["graham"]["scratch dir"]),
     )
     job._get_queue_info = Mock(
         name="_get_queue_info",
         return_value="12426878 21dec16hindcast RUNNING None 2018-10-07T14:13:59",
     )
     job.get_run_id()
     assert job.job_id == "12426878"
     assert job.run_id == "21dec16hindcast"
     assert m_logger.info.called
Пример #11
0
 def test_get_completion_state(self, m_logger, job_state, expected, config):
     job = watch_NEMO_hindcast._SqueueHindcastJob(
         Mock(name="ssh_client"),
         Mock(name="sftp_client"),
         "graham",
         config["run"]["hindcast hosts"]["graham"]["users"],
         Path(config["run"]["hindcast hosts"]["graham"]["scratch dir"]),
     )
     job.run_id = "01mar17hindcast"
     job._ssh_exec_command = Mock(
         name="_ssh_exec_command", return_value=f"State\n----------\n{job_state}\n"
     )
     state = job.get_completion_state()
     assert state == expected
     m_logger.info.assert_called_once_with(
         f"{job.run_id} on {job.host_name}: {job_state}"
     )
Пример #12
0
 def test_report_progress(self, m_logger, config):
     job = watch_NEMO_hindcast._SqueueHindcastJob(
         Mock(name="ssh_client"),
         Mock(name="sftp_client"),
         "graham",
         config["run"]["hindcast hosts"]["graham"]["users"],
         Path(config["run"]["hindcast hosts"]["graham"]["scratch dir"]),
     )
     job.run_id = "01oct18hindcast"
     job.it000 = 1_658_881
     job.itend = 1_682_640
     job.date0 = arrow.get("2016-12-21")
     job.rdt = 40.0
     job._report_progress("1658943\n")
     m_logger.info.assert_called_once_with(
         f"01oct18hindcast on {job.host_name}: timestep: "
         f"1658943 = 2016-12-21 00:41:20 UTC, 0.3% complete"
     )
Пример #13
0
 def test_get_tmp_run_dir(self, m_logger, config):
     job = watch_NEMO_hindcast._SqueueHindcastJob(
         Mock(name="ssh_client"),
         Mock(name="sftp_client"),
         "graham",
         config["run"]["hindcast hosts"]["graham"]["users"],
         Path(config["run"]["hindcast hosts"]["graham"]["scratch dir"]),
     )
     job.run_id = "21dec16hindcast"
     job._ssh_exec_command = Mock(
         name="_ssh_exec_command",
         return_value="/scratch/hindcast/21dec16hindcast_2018-10-06T195251.255493-0700",
     )
     job.get_tmp_run_dir()
     assert job.tmp_run_dir == Path(
         "/scratch/hindcast/21dec16hindcast_2018-10-06T195251.255493-0700"
     )
     assert m_logger.debug.called
Пример #14
0
 def test_is_running_ocean_output_errors_cancel_run(
     self, m_ssh_exec_cmd, m_logger, config
 ):
     job = watch_NEMO_hindcast._SqueueHindcastJob(
         Mock(name="ssh_client"),
         Mock(name="sftp_client"),
         "graham",
         config["run"]["hindcast hosts"]["graham"]["users"],
         Path(config["run"]["hindcast hosts"]["graham"]["scratch dir"]),
     )
     job._get_job_state = Mock(name="_get_job_state", return_value="RUNNING")
     job._report_progress = Mock(name="_report_progress")
     job._ssh_exec_command = Mock(name="_ssh_exec_command")
     assert not job.is_running()
     assert m_logger.error.called
     job._ssh_exec_command.assert_called_once_with(
         f"/opt/software/slurm/bin/scancel {job.job_id}",
         f"{job.run_id} on {job.host_name}: cancelled {job.job_id}",
     )
Пример #15
0
 def test_get_run_info(self, m_f90nml_read, m_logger, config):
     job = watch_NEMO_hindcast._SqueueHindcastJob(
         Mock(name="ssh_client"),
         Mock(name="sftp_client"),
         "graham",
         config["run"]["hindcast hosts"]["graham"]["users"],
         Path(config["run"]["hindcast hosts"]["graham"]["scratch dir"]),
     )
     job.tmp_run_dir = Path(
         "/scratch/hindcast/21dec16hindcast_2018-10-06T195251.255493-0700"
     )
     m_f90nml_read.return_value = {
         "namrun": {
             "nn_it000": 1_658_881,
             "nn_itend": 1_682_640,
             "nn_date0": "20161221",
         },
         "namdom": {"rn_rdt": 40.0},
     }
Пример #16
0
 def test_handle_stuck_job(self, m_logger, config):
     job = watch_NEMO_hindcast._SqueueHindcastJob(
         Mock(name="ssh_client"),
         Mock(name="sftp_client"),
         "graham",
         config["run"]["hindcast hosts"]["graham"]["users"],
         Path(config["run"]["hindcast hosts"]["graham"]["scratch dir"]),
     )
     job.run_id = "21dec16hindcast"
     job.tmp_run_dir = (
         "/scratch/hindcast/21dec16hindcast_2018-10-06T195251.255493-0700"
     )
     job._ssh_exec_command = Mock(
         name="_ssh_exec_command",
         side_effect=[
             "",
             "/scratch/hindcast/01jan17hindcast_2018-10-07T141411.374009-0700",
             "",
         ],
     )
     job._get_queue_info = Mock(
         name="_get_queue_info",
         return_value="12426878 21dec16hindcast PENDING resources 2018-10-07T14:13:59",
     )
     job._handle_stuck_job()
     assert job._ssh_exec_command.call_args_list == [
         call(
             "/opt/software/slurm/bin/sbatch "
             "/scratch/hindcast/21dec16hindcast_2018-10-06T195251.255493-0700/SalishSeaNEMO.sh",
             "21dec16hindcast on graham: re-queued",
         ),
         call(f"ls -dtr {job.scratch_dir}/*hindcast*"),
         call(
             "/opt/software/slurm/bin/sbatch -d afterok:12426878 "
             "/scratch/hindcast/01jan17hindcast_2018-10-07T141411.374009-0700/SalishSeaNEMO.sh",
             "01jan17hindcast on graham: re-queued",
         ),
     ]
     assert m_logger.debug.called