def test_debug_arg(self): worker = NowcastWorker("worker_name", "description") worker.init_cli() assert isinstance( worker.cli.parser._get_option_tuples("--debug")[0][0], argparse._StoreTrueAction, )
def test_msg_debug_mode(self, m_logging_config, config, worker_name, exp_msg): worker = NowcastWorker(worker_name, "description") worker.config._dict = config worker._parsed_args = SimpleNamespace(debug=True) msg = worker._configure_logging() assert msg == "**debug mode** writing log messages to console"
def test_success_tell_manager(self): worker = NowcastWorker("worker_name", "description") worker.init_cli() worker.logger = Mock(name="logger") worker.tell_manager = Mock(name="tell_manager") worker.worker_func = Mock(name="worker_func", return_value="checklist") worker.success = Mock(name="success_func", return_value="success") worker._do_work() worker.tell_manager.assert_called_once_with("success", "checklist")
def test_port_in_use(self, m_logging_config, config, worker_name, exception, exp_msg): worker = NowcastWorker(worker_name, "description") worker._socket = Mock(name="zmq_socket") worker.config._dict = config worker._parsed_args = SimpleNamespace(debug=False) m_logging_config.dictConfig.side_effect = (exception, None) msg = worker._configure_logging() assert msg == exp_msg
def test_success_func(self): worker = NowcastWorker("worker_name", "description") worker.init_cli() worker.logger = Mock(name="logger") worker.tell_manager = Mock(name="tell_manager") worker.worker_func = Mock(name="worker_func") worker.success = Mock(name="success_func") worker._do_work() worker.success.assert_called_once_with(worker._parsed_args)
def test_failure_tell_manager(self): worker = NowcastWorker("worker_name", "description") worker.init_cli() worker.logger = Mock(name="logger") worker.tell_manager = Mock(name="tell_manager") worker.worker_func = Mock(name="worker_func", side_effect=WorkerError) worker.failure = Mock(name="failure_func", return_value="failure") worker._do_work() worker.tell_manager.assert_called_once_with("failure")
def test_zmq_handler_root_topic(self, m_logging, m_logging_config): worker = NowcastWorker("test_worker", "description") worker.config._dict = self.zmq_logging_config_ports_list worker._parsed_args = SimpleNamespace(debug=False) m_handler = Mock(name="m_zmq_handler", spec=zmq.log.handlers.PUBHandler) m_logging.getLogger.return_value = Mock(root=Mock( handlers=[m_handler])) worker._configure_logging() assert m_handler.root_topic == "test_worker"
def test_debug_mode_no_console_handler(self, m_logging, m_logging_config, config, worker_name): worker = NowcastWorker(worker_name, "description") worker.config._dict = config worker._parsed_args = SimpleNamespace(debug=True) m_file_handler = Mock(name="m_file_handler") m_logging.getLogger.return_value = Mock(root=Mock( handlers=[m_file_handler])) worker._configure_logging() m_file_handler.setLevel.assert_called_once_with(100)
def test_different_hosts_different_ports(self, m_logging, m_logging_config): worker = NowcastWorker("remote_worker", "description") worker.config._dict = self.zmq_logging_config_different_hosts_different_ports worker._parsed_args = SimpleNamespace(debug=False) m_handler = Mock(name="m_zmq_handler", spec=zmq.log.handlers.PUBHandler) m_logging.getLogger.return_value = Mock(root=Mock( handlers=[m_handler])) with pytest.raises(WorkerError): worker._configure_logging()
def test_logging_dictConfig(self, m_logging_config, config, worker_name): worker = NowcastWorker(worker_name, "description") worker.config._dict = config worker._parsed_args = SimpleNamespace(debug=False) worker._configure_logging() if "publisher" in worker.config["logging"]: m_logging_config.dictConfig.assert_called_once_with( worker.config["logging"]["publisher"]) else: m_logging_config.dictConfig.assert_called_once_with( worker.config["logging"])
def test_system_exit_context_destroy(self): worker = NowcastWorker("worker_name", "description") worker.init_cli() worker.logger = Mock(name="logger") worker._context = Mock(name="context") worker.worker_func = Mock(name="worker_func", side_effect=SystemExit) worker._do_work() assert worker._context.destroy.call_count == 1
def test_logger_critical_unhandled_exception(self): worker = NowcastWorker("worker_name", "description") worker.logger = Mock(name="logger") worker.tell_manager = Mock(name="tell_manager") worker._context = Mock(name="context") worker.worker_func = Mock(name="worker_func", side_effect=Exception) worker._do_work() worker.logger.critical.assert_called_once_with("unhandled exception:", exc_info=True)
def test_zmq_handler_formatters(self, m_logging, m_logging_config): worker = NowcastWorker("test_worker", "description") worker.config._dict = self.zmq_logging_config_ports_list worker._parsed_args = SimpleNamespace(debug=False) m_handler = Mock(name="m_zmq_handler", spec=zmq.log.handlers.PUBHandler) m_logging.getLogger.return_value = Mock(root=Mock( handlers=[m_handler])) worker._configure_logging() expected = { m_logging.DEBUG: m_logging.Formatter("%(message)s\n"), m_logging.INFO: m_logging.Formatter("%(message)s\n"), m_logging.WARNING: m_logging.Formatter("%(message)s\n"), m_logging.ERROR: m_logging.Formatter("%(message)s\n"), m_logging.CRITICAL: m_logging.Formatter("%(message)s\n"), } assert m_handler.formatters == expected
def test_crash_tell_manager(self): worker = NowcastWorker("worker_name", "description") worker.logger = Mock(name="logger") worker.tell_manager = Mock(name="tell_manager") worker.worker_func = Mock(name="worker_func", side_effect=Exception) worker._do_work() worker.tell_manager.assert_called_once_with("crash")
def test_debug_mode(self): worker = NowcastWorker("test_worker", "description") worker._parsed_args = Mock(debug=True) worker.logger = Mock(name="logger") config = Config() config.file = "nowcast.yaml" worker.config._dict = { "message registry": { "workers": { "test_worker": { "success": "successful test" } } } } response_payload = worker.tell_manager("success", "payload") assert worker.logger.debug.call_count == 1 assert response_payload is None
def test_debug_mode(self): worker = NowcastWorker("worker_name", "description") worker._parsed_args = Mock(debug=True) worker.logger = Mock(name="logger") worker._context = Mock(name="context") worker._init_zmq_interface() assert worker.logger.debug.call_count == 1 assert not worker._context.socket.called
def test_logger_debug_task_completed(self): worker = NowcastWorker("worker_name", "description") worker.logger = Mock(name="logger") worker._context = Mock(name="context") worker.worker_func = Mock(name="worker_func", side_effect=SystemExit) worker._do_work() worker.logger.debug.assert_called_once_with( "shutting down", extra={"logger_name": "worker_name"})
def main(): """Set up and run the worker. For command-line usage see: :command:`python -m nowcast.workers.download_fvcom_results --help` """ worker = NowcastWorker(NAME, description=__doc__) worker.init_cli() worker.cli.add_argument( "host_name", help="Name of the host to download results files from") worker.cli.add_argument( "model_config", choices={"r12", "x2"}, help=""" Model configuration to download results files from: 'r12' means the r12 resolution 'x2' means the x2 resolution """, ) worker.cli.add_argument( "run_type", choices={"nowcast", "forecast"}, help="Type of run to download results files from.", ) worker.cli.add_date_option( "--run-date", default=arrow.now().floor("day"), help="Date of the run to download results files from.", ) worker.run(download_fvcom_results, success, failure)
def main(): """Set up and run the worker. For command-line usage see: :command:`python -m nowcast.workers.make_ww3_current_file --help` """ worker = NowcastWorker(NAME, description=__doc__) worker.init_cli() worker.cli.add_argument( 'host_name', help='Name of the host to create the currents file on') worker.cli.add_argument( 'run_type', choices={'forecast2', 'forecast'}, help=''' Type of run to create the currents file for: 'forecast2' means preliminary forecast run (after NEMO forecast2 run), 'forecast' means updated forecast run (after NEMO forecast run) ''', ) worker.cli.add_date_option( '--run-date', default=arrow.now().floor('day'), help='Start date of run to create the currents file for.') worker.run(make_ww3_current_file, success, failure)
def main(): """Set up and run the worker. For command-line usage see: :command:`python -m nowcast.workers.watch_NEMO --help` """ worker = NowcastWorker(NAME, description=__doc__) worker.init_cli() worker.cli.add_argument("host_name", help="Name of the host to monitor the run on") worker.cli.add_argument( "run_type", choices={ "nowcast", "nowcast-green", "nowcast-dev", "forecast", "forecast2" }, help=""" Type of run to monitor: 'nowcast' means nowcast physics run, 'nowcast-green' means nowcast green ocean run, 'forecast' means updated forecast run, 'forecast2' means preliminary forecast run, """, ) worker.run(watch_NEMO, success, failure)
def test_tell_manager(self): worker = NowcastWorker("test_worker", "description") worker._parsed_args = Mock(debug=False) worker._socket = Mock(name="_socket") worker.logger = Mock(name="logger") config = Config() config.file = "nowcast.yaml" worker.config._dict = { "message registry": { "manager": { "ack": "message acknowledged" }, "workers": { "test_worker": { "success": "successful test" } }, } } mgr_msg = Message(source="manager", type="ack") worker._socket.recv_string.return_value = mgr_msg.serialize() response = worker.tell_manager("success", "payload") worker._socket.send_string.assert_called_once_with( Message(source="test_worker", type="success", payload="payload").serialize()) worker._socket.recv_string.assert_called_once_with() assert worker.logger.debug.call_count == 2 assert response == mgr_msg
def main(): """Set up and run the worker. For command-line usage see: :command:`python -m nowcast.workers.run_NEMO_hindcast --help` """ worker = NowcastWorker(NAME, description=__doc__) worker.init_cli() worker.cli.add_argument("host_name", help="Name of the host to queue the run on") worker.cli.add_argument( "--full-month", action="store_true", help=""" Configure the hindcast run to be a calendar month in duration. The default run duration is 10 days, or the number of days remaining in the calendar month of the run. """, ) worker.cli.add_argument("--prev-run-date", default=None, help="Start date of the previous hindcast run.") worker.cli.add_argument( "--walltime", default=None, help=""" Walltime to request for the hindcast run. Defaults to 10 hours (10:00:00) for 10-ish day runs, and 30 hours (30:00:00) for --full-month runs. **IMPORTANT: Use seconds for walltime values greater than 23:59:59** """, ) worker.run(run_NEMO_hindcast, success, failure)
def main(): """Set up and run the worker. For command-line usage see: :command:`python -m nowcast.workers.run_ww3 --help` """ worker = NowcastWorker(NAME, description=__doc__) worker.init_cli() worker.cli.add_argument("host_name", help="Name of the host to execute the run on") worker.cli.add_argument( "run_type", choices={"forecast2", "nowcast", "forecast"}, help=""" Type of run to execute: 'forecast2' means preliminary forecast run (after NEMO forecast2 run), 'nowcast' means nowcast run (after NEMO forecast run) 'forecast' means updated forecast run (after WaveWatch3 forecast run) """, ) worker.cli.add_date_option( "--run-date", default=arrow.now().floor("day"), help="Date to execute the run for.", ) worker.run(run_ww3, success, failure) return worker
def main(): """Set up and run the worker. For command-line usage see: :command:`python -m nowcast.workers.watch_NEMO --help` """ worker = NowcastWorker(NAME, description=__doc__) worker.init_cli() worker.cli.add_argument( 'host_name', help='Name of the host to monitor the run on') worker.cli.add_argument( 'run_type', choices={ 'nowcast', 'nowcast-green', 'nowcast-dev', 'forecast', 'forecast2'}, help=''' Type of run to monitor: 'nowcast' means nowcast physics run, 'nowcast-green' means nowcast green ocean run, 'forecast' means updated forecast run, 'forecast2' means preliminary forecast run, ''', ) worker.cli.add_argument( 'pid', type=int, help='PID of the NEMO run bash script to monitor.' ) worker.run(watch_NEMO, success, failure)
def main(): """Set up and run the worker. For command-line usage see: :command:`python -m nowcast.workers.make_ww3_wind_file --help` """ worker = NowcastWorker(NAME, description=__doc__) worker.init_cli() worker.cli.add_argument( "host_name", help="Name of the host to create the wind file on" ) worker.cli.add_argument( "run_type", choices={"forecast2", "forecast", "nowcast"}, help=""" Type of run to create wind file for: 'forecast2' means preliminary forecast run (after NEMO forecast2 run), 'forecast' means updated forecast run (after NEMO forecast run), 'nowcast' means updated 1 day only (for hindcast runs) """, ) worker.cli.add_date_option( "--run-date", default=arrow.now().floor("day"), help="Start date of run to create the wind file for.", ) worker.run(make_ww3_wind_file, success, failure)
def main(): """Set up and run the worker. For command-line usage see: :command:`python -m nowcast.workers.upload_forcing --help` """ worker = NowcastWorker(NAME, description=__doc__) worker.init_cli() worker.cli.add_argument( "host_name", help="Name of the host to upload forcing files to" ) worker.cli.add_argument( "run_type", choices={"nowcast+", "forecast2", "ssh", "turbidity"}, help=""" Type of run to upload files for: 'nowcast+' means nowcast & 1st forecast runs, 'forecast2' means 2nd forecast run, 'ssh' means Neah Bay sea surface height files only (for forecast run), 'turbidity' means Fraser River turbidity file only (for nowcast-green run). """, ) worker.cli.add_date_option( "--run-date", default=arrow.now().floor("day"), help="Date of the run to upload files for.", ) worker.run(upload_forcing, success, failure) return worker
def main(): """Set up and run the worker. For command-line usage see: :command:`python -m nowcast.workers.collect_weather --help` """ worker = NowcastWorker(NAME, description=__doc__) worker.init_cli() worker.cli.add_argument( "forecast", choices={"00", "06", "12", "18"}, help="Name of forecast to collect files for.", ) worker.cli.add_argument( "resolution", choices={"1km", "2.5km"}, default="2.5km", help="Horizontal resolution of forecast to download files from.", ) worker.cli.add_argument( "--backfill", action="store_true", help="Download forecast files for previous day's date.", ) worker.cli.add_date_option( "--backfill-date", default=arrow.now().floor("day").shift(days=-1), help="Prior date to collect forecast files for.", ) worker.run(collect_weather, success, failure) return worker
def main(): """Set up and run the worker. For command-line usage see: :command:`python -m nowcast.workers.make_fvcom_atmos_forcing --help` """ worker = NowcastWorker(NAME, description=__doc__) worker.init_cli() worker.cli.add_argument( "model_config", choices={"r12", "x2"}, help=""" Model configuration to make atmospheric forcing file for: 'r12' means the r12 resolution 'x2' means the x2 resolution """, ) worker.cli.add_argument( "run_type", choices={"nowcast", "forecast"}, help=""" Type of run to make atmospheric forcing file for: 'nowcast' means run for present UTC day (after NEMO nowcast run), 'forecast' means updated forecast run (next 36h UTC, after NEMO forecast run) """, ) worker.cli.add_date_option( "--run-date", default=arrow.now().floor("day"), help="Date to make atmospheric forcing file for.", ) worker.run(make_fvcom_atmos_forcing, success, failure)
def main(): """Set up and run the worker. For command-line usage see: :command:`python -m nowcast.workers.run_NEMO --help` """ worker = NowcastWorker(NAME, description=__doc__) worker.init_cli() worker.cli.add_argument("host_name", help="Name of the host to execute the run on") worker.cli.add_argument( "run_type", choices={ "nowcast", "nowcast-green", "nowcast-dev", "forecast", "forecast2" }, help=""" Type of run to execute: 'nowcast' means nowcast physics run, 'nowcast-green' means nowcast green ocean run, 'nowcast-dev' means nowcast physics run with in-development features, 'forecast' means updated forecast run, 'forecast2' means preliminary forecast run, """, ) worker.cli.add_date_option( "--run-date", default=arrow.now().floor("day"), help="Date to execute the run for.", ) worker.run(run_NEMO, success, failure)
def main(): worker = NowcastWorker(NAME, description=__doc__) worker.init_cli() worker.cli.add_argument( "host_name", help="Name of the host to download results files from") worker.cli.add_argument( "run_type", choices={ "nowcast", "nowcast-green", "forecast", "forecast2", "hindcast", "nowcast-agrif", }, help="Type of run to download results files from.", ) worker.cli.add_argument( "--dest-host", default="localhost", help= "Name of the host to download results files to. Default is :kbd:`localhost`.", ) worker.cli.add_date_option( "--run-date", default=arrow.now().floor("day"), help="Date of the run to download results files from.", ) worker.run(download_results, success, failure) return worker