def test_debug_arg(self):
worker = NowcastWorker("worker_name", "description")
worker.init_cli()
assert isinstance(
worker.cli.parser._get_option_tuples("--debug")[0][0],
argparse._StoreTrueAction,
)
def test_msg_debug_mode(self, m_logging_config, config, worker_name,
exp_msg):
worker = NowcastWorker(worker_name, "description")
worker.config._dict = config
worker._parsed_args = SimpleNamespace(debug=True)
msg = worker._configure_logging()
assert msg == "**debug mode** writing log messages to console"
def test_success_tell_manager(self):
worker = NowcastWorker("worker_name", "description")
worker.init_cli()
worker.logger = Mock(name="logger")
worker.tell_manager = Mock(name="tell_manager")
worker.worker_func = Mock(name="worker_func", return_value="checklist")
worker.success = Mock(name="success_func", return_value="success")
worker._do_work()
worker.tell_manager.assert_called_once_with("success", "checklist")
def test_port_in_use(self, m_logging_config, config, worker_name,
exception, exp_msg):
worker = NowcastWorker(worker_name, "description")
worker._socket = Mock(name="zmq_socket")
worker.config._dict = config
worker._parsed_args = SimpleNamespace(debug=False)
m_logging_config.dictConfig.side_effect = (exception, None)
msg = worker._configure_logging()
assert msg == exp_msg
def test_success_func(self):
worker = NowcastWorker("worker_name", "description")
worker.init_cli()
worker.logger = Mock(name="logger")
worker.tell_manager = Mock(name="tell_manager")
worker.worker_func = Mock(name="worker_func")
worker.success = Mock(name="success_func")
worker._do_work()
worker.success.assert_called_once_with(worker._parsed_args)
def test_failure_tell_manager(self):
worker = NowcastWorker("worker_name", "description")
worker.init_cli()
worker.logger = Mock(name="logger")
worker.tell_manager = Mock(name="tell_manager")
worker.worker_func = Mock(name="worker_func", side_effect=WorkerError)
worker.failure = Mock(name="failure_func", return_value="failure")
worker._do_work()
worker.tell_manager.assert_called_once_with("failure")
def test_zmq_handler_root_topic(self, m_logging, m_logging_config):
worker = NowcastWorker("test_worker", "description")
worker.config._dict = self.zmq_logging_config_ports_list
worker._parsed_args = SimpleNamespace(debug=False)
m_handler = Mock(name="m_zmq_handler",
spec=zmq.log.handlers.PUBHandler)
m_logging.getLogger.return_value = Mock(root=Mock(
handlers=[m_handler]))
worker._configure_logging()
assert m_handler.root_topic == "test_worker"
def test_debug_mode_no_console_handler(self, m_logging, m_logging_config,
config, worker_name):
worker = NowcastWorker(worker_name, "description")
worker.config._dict = config
worker._parsed_args = SimpleNamespace(debug=True)
m_file_handler = Mock(name="m_file_handler")
m_logging.getLogger.return_value = Mock(root=Mock(
handlers=[m_file_handler]))
worker._configure_logging()
m_file_handler.setLevel.assert_called_once_with(100)
def test_different_hosts_different_ports(self, m_logging,
m_logging_config):
worker = NowcastWorker("remote_worker", "description")
worker.config._dict = self.zmq_logging_config_different_hosts_different_ports
worker._parsed_args = SimpleNamespace(debug=False)
m_handler = Mock(name="m_zmq_handler",
spec=zmq.log.handlers.PUBHandler)
m_logging.getLogger.return_value = Mock(root=Mock(
handlers=[m_handler]))
with pytest.raises(WorkerError):
worker._configure_logging()
def test_logging_dictConfig(self, m_logging_config, config, worker_name):
worker = NowcastWorker(worker_name, "description")
worker.config._dict = config
worker._parsed_args = SimpleNamespace(debug=False)
worker._configure_logging()
if "publisher" in worker.config["logging"]:
m_logging_config.dictConfig.assert_called_once_with(
worker.config["logging"]["publisher"])
else:
m_logging_config.dictConfig.assert_called_once_with(
worker.config["logging"])
def test_system_exit_context_destroy(self):
worker = NowcastWorker("worker_name", "description")
worker.init_cli()
worker.logger = Mock(name="logger")
worker._context = Mock(name="context")
worker.worker_func = Mock(name="worker_func", side_effect=SystemExit)
worker._do_work()
assert worker._context.destroy.call_count == 1
def test_logger_critical_unhandled_exception(self):
worker = NowcastWorker("worker_name", "description")
worker.logger = Mock(name="logger")
worker.tell_manager = Mock(name="tell_manager")
worker._context = Mock(name="context")
worker.worker_func = Mock(name="worker_func", side_effect=Exception)
worker._do_work()
worker.logger.critical.assert_called_once_with("unhandled exception:",
exc_info=True)
def test_zmq_handler_formatters(self, m_logging, m_logging_config):
worker = NowcastWorker("test_worker", "description")
worker.config._dict = self.zmq_logging_config_ports_list
worker._parsed_args = SimpleNamespace(debug=False)
m_handler = Mock(name="m_zmq_handler",
spec=zmq.log.handlers.PUBHandler)
m_logging.getLogger.return_value = Mock(root=Mock(
handlers=[m_handler]))
worker._configure_logging()
expected = {
m_logging.DEBUG: m_logging.Formatter("%(message)s\n"),
m_logging.INFO: m_logging.Formatter("%(message)s\n"),
m_logging.WARNING: m_logging.Formatter("%(message)s\n"),
m_logging.ERROR: m_logging.Formatter("%(message)s\n"),
m_logging.CRITICAL: m_logging.Formatter("%(message)s\n"),
}
assert m_handler.formatters == expected
def test_crash_tell_manager(self):
worker = NowcastWorker("worker_name", "description")
worker.logger = Mock(name="logger")
worker.tell_manager = Mock(name="tell_manager")
worker.worker_func = Mock(name="worker_func", side_effect=Exception)
worker._do_work()
worker.tell_manager.assert_called_once_with("crash")
def test_debug_mode(self):
worker = NowcastWorker("test_worker", "description")
worker._parsed_args = Mock(debug=True)
worker.logger = Mock(name="logger")
config = Config()
config.file = "nowcast.yaml"
worker.config._dict = {
"message registry": {
"workers": {
"test_worker": {
"success": "successful test"
}
}
}
}
response_payload = worker.tell_manager("success", "payload")
assert worker.logger.debug.call_count == 1
assert response_payload is None
def test_debug_mode(self):
worker = NowcastWorker("worker_name", "description")
worker._parsed_args = Mock(debug=True)
worker.logger = Mock(name="logger")
worker._context = Mock(name="context")
worker._init_zmq_interface()
assert worker.logger.debug.call_count == 1
assert not worker._context.socket.called
def test_logger_debug_task_completed(self):
worker = NowcastWorker("worker_name", "description")
worker.logger = Mock(name="logger")
worker._context = Mock(name="context")
worker.worker_func = Mock(name="worker_func", side_effect=SystemExit)
worker._do_work()
worker.logger.debug.assert_called_once_with(
"shutting down", extra={"logger_name": "worker_name"})
def main():
"""Set up and run the worker.
For command-line usage see:
:command:`python -m nowcast.workers.download_fvcom_results --help`
"""
worker = NowcastWorker(NAME, description=__doc__)
worker.init_cli()
worker.cli.add_argument(
"host_name", help="Name of the host to download results files from")
worker.cli.add_argument(
"model_config",
choices={"r12", "x2"},
help="""
Model configuration to download results files from:
'r12' means the r12 resolution
'x2' means the x2 resolution
""",
)
worker.cli.add_argument(
"run_type",
choices={"nowcast", "forecast"},
help="Type of run to download results files from.",
)
worker.cli.add_date_option(
"--run-date",
default=arrow.now().floor("day"),
help="Date of the run to download results files from.",
)
worker.run(download_fvcom_results, success, failure)
def main():
"""Set up and run the worker.
For command-line usage see:
:command:`python -m nowcast.workers.make_ww3_current_file --help`
"""
worker = NowcastWorker(NAME, description=__doc__)
worker.init_cli()
worker.cli.add_argument(
'host_name',
help='Name of the host to create the currents file on')
worker.cli.add_argument(
'run_type',
choices={'forecast2', 'forecast'},
help='''
Type of run to create the currents file for:
'forecast2' means preliminary forecast run (after NEMO forecast2 run),
'forecast' means updated forecast run (after NEMO forecast run)
''',
)
worker.cli.add_date_option(
'--run-date', default=arrow.now().floor('day'),
help='Start date of run to create the currents file for.')
worker.run(make_ww3_current_file, success, failure)
def main():
"""Set up and run the worker.
For command-line usage see:
:command:`python -m nowcast.workers.watch_NEMO --help`
"""
worker = NowcastWorker(NAME, description=__doc__)
worker.init_cli()
worker.cli.add_argument("host_name",
help="Name of the host to monitor the run on")
worker.cli.add_argument(
"run_type",
choices={
"nowcast", "nowcast-green", "nowcast-dev", "forecast", "forecast2"
},
help="""
Type of run to monitor:
'nowcast' means nowcast physics run,
'nowcast-green' means nowcast green ocean run,
'forecast' means updated forecast run,
'forecast2' means preliminary forecast run,
""",
)
worker.run(watch_NEMO, success, failure)
def test_tell_manager(self):
worker = NowcastWorker("test_worker", "description")
worker._parsed_args = Mock(debug=False)
worker._socket = Mock(name="_socket")
worker.logger = Mock(name="logger")
config = Config()
config.file = "nowcast.yaml"
worker.config._dict = {
"message registry": {
"manager": {
"ack": "message acknowledged"
},
"workers": {
"test_worker": {
"success": "successful test"
}
},
}
}
mgr_msg = Message(source="manager", type="ack")
worker._socket.recv_string.return_value = mgr_msg.serialize()
response = worker.tell_manager("success", "payload")
worker._socket.send_string.assert_called_once_with(
Message(source="test_worker", type="success",
payload="payload").serialize())
worker._socket.recv_string.assert_called_once_with()
assert worker.logger.debug.call_count == 2
assert response == mgr_msg
def main():
"""Set up and run the worker.
For command-line usage see:
:command:`python -m nowcast.workers.run_NEMO_hindcast --help`
"""
worker = NowcastWorker(NAME, description=__doc__)
worker.init_cli()
worker.cli.add_argument("host_name",
help="Name of the host to queue the run on")
worker.cli.add_argument(
"--full-month",
action="store_true",
help="""
Configure the hindcast run to be a calendar month in duration.
The default run duration is 10 days, or the number of days remaining
in the calendar month of the run.
""",
)
worker.cli.add_argument("--prev-run-date",
default=None,
help="Start date of the previous hindcast run.")
worker.cli.add_argument(
"--walltime",
default=None,
help="""
Walltime to request for the hindcast run.
Defaults to 10 hours (10:00:00) for 10-ish day runs,
and 30 hours (30:00:00) for --full-month runs.
**IMPORTANT: Use seconds for walltime values greater than 23:59:59**
""",
)
worker.run(run_NEMO_hindcast, success, failure)
def main():
"""Set up and run the worker.
For command-line usage see:
:command:`python -m nowcast.workers.run_ww3 --help`
"""
worker = NowcastWorker(NAME, description=__doc__)
worker.init_cli()
worker.cli.add_argument("host_name",
help="Name of the host to execute the run on")
worker.cli.add_argument(
"run_type",
choices={"forecast2", "nowcast", "forecast"},
help="""
Type of run to execute:
'forecast2' means preliminary forecast run (after NEMO forecast2 run),
'nowcast' means nowcast run (after NEMO forecast run)
'forecast' means updated forecast run (after WaveWatch3 forecast run)
""",
)
worker.cli.add_date_option(
"--run-date",
default=arrow.now().floor("day"),
help="Date to execute the run for.",
)
worker.run(run_ww3, success, failure)
return worker
def main():
"""Set up and run the worker.
For command-line usage see:
:command:`python -m nowcast.workers.watch_NEMO --help`
"""
worker = NowcastWorker(NAME, description=__doc__)
worker.init_cli()
worker.cli.add_argument(
'host_name',
help='Name of the host to monitor the run on')
worker.cli.add_argument(
'run_type',
choices={
'nowcast', 'nowcast-green', 'nowcast-dev', 'forecast', 'forecast2'},
help='''
Type of run to monitor:
'nowcast' means nowcast physics run,
'nowcast-green' means nowcast green ocean run,
'forecast' means updated forecast run,
'forecast2' means preliminary forecast run,
''',
)
worker.cli.add_argument(
'pid', type=int,
help='PID of the NEMO run bash script to monitor.'
)
worker.run(watch_NEMO, success, failure)
def main():
"""Set up and run the worker.
For command-line usage see:
:command:`python -m nowcast.workers.make_ww3_wind_file --help`
"""
worker = NowcastWorker(NAME, description=__doc__)
worker.init_cli()
worker.cli.add_argument(
"host_name", help="Name of the host to create the wind file on"
)
worker.cli.add_argument(
"run_type",
choices={"forecast2", "forecast", "nowcast"},
help="""
Type of run to create wind file for:
'forecast2' means preliminary forecast run (after NEMO forecast2 run),
'forecast' means updated forecast run (after NEMO forecast run),
'nowcast' means updated 1 day only (for hindcast runs)
""",
)
worker.cli.add_date_option(
"--run-date",
default=arrow.now().floor("day"),
help="Start date of run to create the wind file for.",
)
worker.run(make_ww3_wind_file, success, failure)
def main():
"""Set up and run the worker.
For command-line usage see:
:command:`python -m nowcast.workers.upload_forcing --help`
"""
worker = NowcastWorker(NAME, description=__doc__)
worker.init_cli()
worker.cli.add_argument(
"host_name", help="Name of the host to upload forcing files to"
)
worker.cli.add_argument(
"run_type",
choices={"nowcast+", "forecast2", "ssh", "turbidity"},
help="""
Type of run to upload files for:
'nowcast+' means nowcast & 1st forecast runs,
'forecast2' means 2nd forecast run,
'ssh' means Neah Bay sea surface height files only (for forecast run),
'turbidity' means Fraser River turbidity file only
(for nowcast-green run).
""",
)
worker.cli.add_date_option(
"--run-date",
default=arrow.now().floor("day"),
help="Date of the run to upload files for.",
)
worker.run(upload_forcing, success, failure)
return worker
def main():
"""Set up and run the worker.
For command-line usage see:
:command:`python -m nowcast.workers.collect_weather --help`
"""
worker = NowcastWorker(NAME, description=__doc__)
worker.init_cli()
worker.cli.add_argument(
"forecast",
choices={"00", "06", "12", "18"},
help="Name of forecast to collect files for.",
)
worker.cli.add_argument(
"resolution",
choices={"1km", "2.5km"},
default="2.5km",
help="Horizontal resolution of forecast to download files from.",
)
worker.cli.add_argument(
"--backfill",
action="store_true",
help="Download forecast files for previous day's date.",
)
worker.cli.add_date_option(
"--backfill-date",
default=arrow.now().floor("day").shift(days=-1),
help="Prior date to collect forecast files for.",
)
worker.run(collect_weather, success, failure)
return worker
def main():
"""Set up and run the worker.
For command-line usage see:
:command:`python -m nowcast.workers.make_fvcom_atmos_forcing --help`
"""
worker = NowcastWorker(NAME, description=__doc__)
worker.init_cli()
worker.cli.add_argument(
"model_config",
choices={"r12", "x2"},
help="""
Model configuration to make atmospheric forcing file for:
'r12' means the r12 resolution
'x2' means the x2 resolution
""",
)
worker.cli.add_argument(
"run_type",
choices={"nowcast", "forecast"},
help="""
Type of run to make atmospheric forcing file for:
'nowcast' means run for present UTC day (after NEMO nowcast run),
'forecast' means updated forecast run (next 36h UTC, after NEMO forecast run)
""",
)
worker.cli.add_date_option(
"--run-date",
default=arrow.now().floor("day"),
help="Date to make atmospheric forcing file for.",
)
worker.run(make_fvcom_atmos_forcing, success, failure)
def main():
"""Set up and run the worker.
For command-line usage see:
:command:`python -m nowcast.workers.run_NEMO --help`
"""
worker = NowcastWorker(NAME, description=__doc__)
worker.init_cli()
worker.cli.add_argument("host_name",
help="Name of the host to execute the run on")
worker.cli.add_argument(
"run_type",
choices={
"nowcast", "nowcast-green", "nowcast-dev", "forecast", "forecast2"
},
help="""
Type of run to execute:
'nowcast' means nowcast physics run,
'nowcast-green' means nowcast green ocean run,
'nowcast-dev' means nowcast physics run with in-development features,
'forecast' means updated forecast run,
'forecast2' means preliminary forecast run,
""",
)
worker.cli.add_date_option(
"--run-date",
default=arrow.now().floor("day"),
help="Date to execute the run for.",
)
worker.run(run_NEMO, success, failure)
def main():
worker = NowcastWorker(NAME, description=__doc__)
worker.init_cli()
worker.cli.add_argument(
"host_name", help="Name of the host to download results files from")
worker.cli.add_argument(
"run_type",
choices={
"nowcast",
"nowcast-green",
"forecast",
"forecast2",
"hindcast",
"nowcast-agrif",
},
help="Type of run to download results files from.",
)
worker.cli.add_argument(
"--dest-host",
default="localhost",
help=
"Name of the host to download results files to. Default is :kbd:`localhost`.",
)
worker.cli.add_date_option(
"--run-date",
default=arrow.now().floor("day"),
help="Date of the run to download results files from.",
)
worker.run(download_results, success, failure)
return worker
