def test_debug_arg(self):
worker = NowcastWorker("worker_name", "description")
worker.init_cli()
assert isinstance(
worker.cli.parser._get_option_tuples("--debug")[0][0],
argparse._StoreTrueAction,
)
def test_msg_debug_mode(self, m_logging_config, config, worker_name,
exp_msg):
worker = NowcastWorker(worker_name, "description")
worker.config._dict = config
worker._parsed_args = SimpleNamespace(debug=True)
msg = worker._configure_logging()
assert msg == "**debug mode** writing log messages to console"
def main():
"""Set up and run the worker.
For command-line usage see:
:command:`python -m nowcast.workers.make_ww3_wind_file --help`
"""
worker = NowcastWorker(NAME, description=__doc__)
worker.init_cli()
worker.cli.add_argument(
"host_name", help="Name of the host to create the wind file on"
)
worker.cli.add_argument(
"run_type",
choices={"forecast2", "forecast", "nowcast"},
help="""
Type of run to create wind file for:
'forecast2' means preliminary forecast run (after NEMO forecast2 run),
'forecast' means updated forecast run (after NEMO forecast run),
'nowcast' means updated 1 day only (for hindcast runs)
""",
)
worker.cli.add_date_option(
"--run-date",
default=arrow.now().floor("day"),
help="Start date of run to create the wind file for.",
)
worker.run(make_ww3_wind_file, success, failure)
def main():
"""Set up and run the worker.
For command-line usage see:
:command:`python -m nowcast.workers.upload_forcing --help`
"""
worker = NowcastWorker(NAME, description=__doc__)
worker.init_cli()
worker.cli.add_argument(
"host_name", help="Name of the host to upload forcing files to"
)
worker.cli.add_argument(
"run_type",
choices={"nowcast+", "forecast2", "ssh", "turbidity"},
help="""
Type of run to upload files for:
'nowcast+' means nowcast & 1st forecast runs,
'forecast2' means 2nd forecast run,
'ssh' means Neah Bay sea surface height files only (for forecast run),
'turbidity' means Fraser River turbidity file only
(for nowcast-green run).
""",
)
worker.cli.add_date_option(
"--run-date",
default=arrow.now().floor("day"),
help="Date of the run to upload files for.",
)
worker.run(upload_forcing, success, failure)
return worker
def main():
"""Set up and run the worker.
For command-line usage see:
:command:`python -m nowcast.workers.run_ww3 --help`
"""
worker = NowcastWorker(NAME, description=__doc__)
worker.init_cli()
worker.cli.add_argument("host_name",
help="Name of the host to execute the run on")
worker.cli.add_argument(
"run_type",
choices={"forecast2", "nowcast", "forecast"},
help="""
Type of run to execute:
'forecast2' means preliminary forecast run (after NEMO forecast2 run),
'nowcast' means nowcast run (after NEMO forecast run)
'forecast' means updated forecast run (after WaveWatch3 forecast run)
""",
)
worker.cli.add_date_option(
"--run-date",
default=arrow.now().floor("day"),
help="Date to execute the run for.",
)
worker.run(run_ww3, success, failure)
return worker
def main():
"""Set up and run the worker.
For command-line usage see:
:command:`python -m nowcast.workers.watch_NEMO --help`
"""
worker = NowcastWorker(NAME, description=__doc__)
worker.init_cli()
worker.cli.add_argument(
'host_name',
help='Name of the host to monitor the run on')
worker.cli.add_argument(
'run_type',
choices={
'nowcast', 'nowcast-green', 'nowcast-dev', 'forecast', 'forecast2'},
help='''
Type of run to monitor:
'nowcast' means nowcast physics run,
'nowcast-green' means nowcast green ocean run,
'forecast' means updated forecast run,
'forecast2' means preliminary forecast run,
''',
)
worker.cli.add_argument(
'pid', type=int,
help='PID of the NEMO run bash script to monitor.'
)
worker.run(watch_NEMO, success, failure)
def main():
"""Set up and run the worker.
For command-line usage see:
:command:`python -m nowcast.workers.make_fvcom_atmos_forcing --help`
"""
worker = NowcastWorker(NAME, description=__doc__)
worker.init_cli()
worker.cli.add_argument(
"model_config",
choices={"r12", "x2"},
help="""
Model configuration to make atmospheric forcing file for:
'r12' means the r12 resolution
'x2' means the x2 resolution
""",
)
worker.cli.add_argument(
"run_type",
choices={"nowcast", "forecast"},
help="""
Type of run to make atmospheric forcing file for:
'nowcast' means run for present UTC day (after NEMO nowcast run),
'forecast' means updated forecast run (next 36h UTC, after NEMO forecast run)
""",
)
worker.cli.add_date_option(
"--run-date",
default=arrow.now().floor("day"),
help="Date to make atmospheric forcing file for.",
)
worker.run(make_fvcom_atmos_forcing, success, failure)
def main():
"""Set up and run the worker.
For command-line usage see:
:command:`python -m nowcast.workers.run_NEMO_hindcast --help`
"""
worker = NowcastWorker(NAME, description=__doc__)
worker.init_cli()
worker.cli.add_argument("host_name",
help="Name of the host to queue the run on")
worker.cli.add_argument(
"--full-month",
action="store_true",
help="""
Configure the hindcast run to be a calendar month in duration.
The default run duration is 10 days, or the number of days remaining
in the calendar month of the run.
""",
)
worker.cli.add_argument("--prev-run-date",
default=None,
help="Start date of the previous hindcast run.")
worker.cli.add_argument(
"--walltime",
default=None,
help="""
Walltime to request for the hindcast run.
Defaults to 10 hours (10:00:00) for 10-ish day runs,
and 30 hours (30:00:00) for --full-month runs.
**IMPORTANT: Use seconds for walltime values greater than 23:59:59**
""",
)
worker.run(run_NEMO_hindcast, success, failure)
def main():
"""Set up and run the worker.
For command-line usage see:
:command:`python -m nowcast.workers.make_CHS_currents_file --help`
"""
worker = NowcastWorker(NAME, description=__doc__)
worker.init_cli()
worker.cli.add_argument(
"run_type",
choices={"nowcast", "forecast", "forecast2"},
help="""
Type of run the velocities come from:
'nowcast' means velocities stored in nowcast-blue directory
'forecast' means velocities stored in forecast directory
'forecast2' means velocities stored in forecast2 directory
""",
)
worker.cli.add_date_option(
"--run-date",
default=arrow.now().floor("day"),
help="Date to process the velocities for.",
)
worker.run(make_CHS_currents_file, success, failure)
def main():
worker = NowcastWorker(NAME, description=__doc__)
worker.init_cli()
worker.cli.add_argument(
"host_name", help="Name of the host to download results files from")
worker.cli.add_argument(
"run_type",
choices={
"nowcast",
"nowcast-green",
"forecast",
"forecast2",
"hindcast",
"nowcast-agrif",
},
help="Type of run to download results files from.",
)
worker.cli.add_argument(
"--dest-host",
default="localhost",
help=
"Name of the host to download results files to. Default is :kbd:`localhost`.",
)
worker.cli.add_date_option(
"--run-date",
default=arrow.now().floor("day"),
help="Date of the run to download results files from.",
)
worker.run(download_results, success, failure)
return worker
def main():
"""Set up and run the worker.
For command-line usage see:
:command:`python -m nowcast.workers.run_NEMO_agrif --help`
"""
worker = NowcastWorker(NAME, description=__doc__)
worker.init_cli()
worker.cli.add_argument("host_name",
help="Name of the host to queue the run on")
worker.cli.add_argument(
"run_type",
choices={"nowcast-agrif"},
help="""
Type of run to execute:
'nowcast-agrif' means nowcast green ocean run with AGRIF sub-grids
""",
)
worker.cli.add_date_option(
"--run-date",
default=arrow.now().floor("day"),
help="Date to execute the run for.",
)
worker.run(run_NEMO_agrif, success, failure)
def main():
"""Set up and run the worker.
For command-line usage see:
:command:`python -m nowcast.workers.make_forcing_links --help`
"""
worker = NowcastWorker(NAME, description=__doc__)
worker.init_cli()
worker.cli.add_argument(
'host_name', help='Name of the host to symlink forcing files on')
worker.cli.add_argument(
'run_type',
choices={'nowcast+', 'forecast2', 'ssh', 'nowcast-green'},
help='''
Type of run to symlink files for:
'nowcast+' means nowcast & 1st forecast runs,
'forecast2' means 2nd forecast run,
'ssh' means Neah Bay sea surface height files only (for forecast run).
'nowcast-green' means nowcast green ocean run,
''')
worker.cli.add_argument('--shared-storage',
action='store_true',
help='''
If running on a machine (Salish) that directly accesses
the repo datafiles, copy the forcing files instead of symlinking them
so that they do not get changed as a result of preparations for faster
runs on remote hosts
''')
worker.cli.add_date_option('--run-date',
default=(arrow.now().floor('day')),
help='Date of the run to symlink files for.')
worker.run(make_forcing_links, success, failure)
def main():
"""Set up and run the worker.
For command-line usage see:
:command:`python -m nowcast.workers.watch_fvcom --help`
"""
worker = NowcastWorker(NAME, description=__doc__)
worker.init_cli()
worker.cli.add_argument("host_name", help="Name of the host to monitor the run on")
worker.cli.add_argument(
"model_config",
choices={"r12", "x2"},
help="""
Model configuration to monitor run for for:
'r12' means the r12 resolution
'x2' means the x2 resolution
""",
)
worker.cli.add_argument(
"run_type",
choices={"nowcast", "forecast"},
help="""
Type of run to monitor:
'nowcast' means nowcast run (after NEMO nowcast run)
'forecast' means updated forecast run (next 36h UTC, after NEMO forecast run)
""",
)
worker.run(watch_fvcom, success, failure)
def main():
"""Set up and run the worker.
For command-line usage see:
:command:`python -m nowcast.workers.update_forecast_datasets -h`
"""
worker = NowcastWorker(NAME, description=__doc__)
worker.init_cli()
worker.cli.add_argument(
"model",
choices={"fvcom", "nemo", "wwatch3"},
help="""
Model to update the rolling forecast datasets for:
'fvcom' means the Vancouver Harbour Fraser River (VHFR) FVCOM model,
'nemo' means the Salish Sea NEMO model,
'wwatch3' means the Strait of Georgia WaveWatch3(TM) model.
""",
)
worker.cli.add_argument(
"run_type",
choices={"forecast", "forecast2"},
help="""
Type of run to update rolling forecast datasets for:
'forecast' means afternoon updated forecast runs,
'forecast2' means early morning preliminary forecast runs,
""",
)
worker.cli.add_date_option(
"--run-date",
default=(arrow.now().floor("day")),
help="Date of the run to update rolling forecast datasets for.",
)
worker.run(update_forecast_datasets, success, failure)
def main():
"""Set up and run the worker.
For command-line usage see:
:command:`python -m nowcast.workers.run_NEMO --help`
"""
worker = NowcastWorker(NAME, description=__doc__)
worker.init_cli()
worker.cli.add_argument("host_name",
help="Name of the host to execute the run on")
worker.cli.add_argument(
"run_type",
choices={
"nowcast", "nowcast-green", "nowcast-dev", "forecast", "forecast2"
},
help="""
Type of run to execute:
'nowcast' means nowcast physics run,
'nowcast-green' means nowcast green ocean run,
'nowcast-dev' means nowcast physics run with in-development features,
'forecast' means updated forecast run,
'forecast2' means preliminary forecast run,
""",
)
worker.cli.add_date_option(
"--run-date",
default=arrow.now().floor("day"),
help="Date to execute the run for.",
)
worker.run(run_NEMO, success, failure)
def main():
"""Set up and run the worker.
For command-line usage see:
:command:`python -m nowcast.workers.run_NEMO --help`
"""
worker = NowcastWorker(NAME, description=__doc__)
worker.init_cli()
worker.cli.add_argument('host_name',
help='Name of the host to execute the run on')
worker.cli.add_argument(
'run_type',
choices={
'nowcast', 'nowcast-green', 'nowcast-dev', 'forecast', 'forecast2'
},
help='''
Type of run to execute:
'nowcast' means nowcast physics run,
'nowcast-green' means nowcast green ocean run,
'nowcast-dev' means nowcast physics run with in-development features,
'forecast' means updated forecast run,
'forecast2' means preliminary forecast run,
''',
)
worker.cli.add_date_option('--run-date',
default=arrow.now().floor('day'),
help='Date to execute the run for.')
worker.run(run_NEMO, success, failure)
def main():
"""Set up and run the worker.
For command-line usage see:
:command:`python -m nowcast.workers.download_fvcom_results --help`
"""
worker = NowcastWorker(NAME, description=__doc__)
worker.init_cli()
worker.cli.add_argument(
"host_name", help="Name of the host to download results files from")
worker.cli.add_argument(
"model_config",
choices={"r12", "x2"},
help="""
Model configuration to download results files from:
'r12' means the r12 resolution
'x2' means the x2 resolution
""",
)
worker.cli.add_argument(
"run_type",
choices={"nowcast", "forecast"},
help="Type of run to download results files from.",
)
worker.cli.add_date_option(
"--run-date",
default=arrow.now().floor("day"),
help="Date of the run to download results files from.",
)
worker.run(download_fvcom_results, success, failure)
def main():
"""Set up and run the worker.
For command-line usage see:
:command:`python -m nowcast.workers.collect_weather --help`
"""
worker = NowcastWorker(NAME, description=__doc__)
worker.init_cli()
worker.cli.add_argument(
"forecast",
choices={"00", "06", "12", "18"},
help="Name of forecast to collect files for.",
)
worker.cli.add_argument(
"resolution",
choices={"1km", "2.5km"},
default="2.5km",
help="Horizontal resolution of forecast to download files from.",
)
worker.cli.add_argument(
"--backfill",
action="store_true",
help="Download forecast files for previous day's date.",
)
worker.cli.add_date_option(
"--backfill-date",
default=arrow.now().floor("day").shift(days=-1),
help="Prior date to collect forecast files for.",
)
worker.run(collect_weather, success, failure)
return worker
def main():
"""Set up and run the worker.
For command-line usage see:
:command:`python -m nowcast.workers.watch_NEMO --help`
"""
worker = NowcastWorker(NAME, description=__doc__)
worker.init_cli()
worker.cli.add_argument("host_name",
help="Name of the host to monitor the run on")
worker.cli.add_argument(
"run_type",
choices={
"nowcast", "nowcast-green", "nowcast-dev", "forecast", "forecast2"
},
help="""
Type of run to monitor:
'nowcast' means nowcast physics run,
'nowcast-green' means nowcast green ocean run,
'forecast' means updated forecast run,
'forecast2' means preliminary forecast run,
""",
)
worker.run(watch_NEMO, success, failure)
def main():
"""Set up and run the worker.
For command-line usage see:
:command:`python -m nowcast.workers.make_ww3_current_file --help`
"""
worker = NowcastWorker(NAME, description=__doc__)
worker.init_cli()
worker.cli.add_argument(
'host_name',
help='Name of the host to create the currents file on')
worker.cli.add_argument(
'run_type',
choices={'forecast2', 'forecast'},
help='''
Type of run to create the currents file for:
'forecast2' means preliminary forecast run (after NEMO forecast2 run),
'forecast' means updated forecast run (after NEMO forecast run)
''',
)
worker.cli.add_date_option(
'--run-date', default=arrow.now().floor('day'),
help='Start date of run to create the currents file for.')
worker.run(make_ww3_current_file, success, failure)
def test_crash_tell_manager(self):
worker = NowcastWorker("worker_name", "description")
worker.logger = Mock(name="logger")
worker.tell_manager = Mock(name="tell_manager")
worker.worker_func = Mock(name="worker_func", side_effect=Exception)
worker._do_work()
worker.tell_manager.assert_called_once_with("crash")
def test_tell_manager(self):
worker = NowcastWorker("test_worker", "description")
worker._parsed_args = Mock(debug=False)
worker._socket = Mock(name="_socket")
worker.logger = Mock(name="logger")
config = Config()
config.file = "nowcast.yaml"
worker.config._dict = {
"message registry": {
"manager": {
"ack": "message acknowledged"
},
"workers": {
"test_worker": {
"success": "successful test"
}
},
}
}
mgr_msg = Message(source="manager", type="ack")
worker._socket.recv_string.return_value = mgr_msg.serialize()
response = worker.tell_manager("success", "payload")
worker._socket.send_string.assert_called_once_with(
Message(source="test_worker", type="success",
payload="payload").serialize())
worker._socket.recv_string.assert_called_once_with()
assert worker.logger.debug.call_count == 2
assert response == mgr_msg
def test_debug_mode(self):
worker = NowcastWorker("worker_name", "description")
worker._parsed_args = Mock(debug=True)
worker.logger = Mock(name="logger")
worker._context = Mock(name="context")
worker._init_zmq_interface()
assert worker.logger.debug.call_count == 1
assert not worker._context.socket.called
def test_logger_debug_task_completed(self):
worker = NowcastWorker("worker_name", "description")
worker.logger = Mock(name="logger")
worker._context = Mock(name="context")
worker.worker_func = Mock(name="worker_func", side_effect=SystemExit)
worker._do_work()
worker.logger.debug.assert_called_once_with(
"shutting down", extra={"logger_name": "worker_name"})
def test_system_exit_context_destroy(self):
worker = NowcastWorker("worker_name", "description")
worker.init_cli()
worker.logger = Mock(name="logger")
worker._context = Mock(name="context")
worker.worker_func = Mock(name="worker_func", side_effect=SystemExit)
worker._do_work()
assert worker._context.destroy.call_count == 1
def test_unregistered_worker(self):
worker = NowcastWorker("test_worker", "description")
worker._parsed_args = Mock(debug=True)
worker.logger = Mock(name="logger")
config = Config()
config.file = "nowcast.yaml"
worker.config._dict = {"message registry": {"workers": {}}}
with pytest.raises(WorkerError):
worker.tell_manager("success", "payload")
def test_logger_critical_unhandled_exception(self):
worker = NowcastWorker("worker_name", "description")
worker.logger = Mock(name="logger")
worker.tell_manager = Mock(name="tell_manager")
worker._context = Mock(name="context")
worker.worker_func = Mock(name="worker_func", side_effect=Exception)
worker._do_work()
worker.logger.critical.assert_called_once_with("unhandled exception:",
exc_info=True)
def test_failure_tell_manager(self):
worker = NowcastWorker("worker_name", "description")
worker.init_cli()
worker.logger = Mock(name="logger")
worker.tell_manager = Mock(name="tell_manager")
worker.worker_func = Mock(name="worker_func", side_effect=WorkerError)
worker.failure = Mock(name="failure_func", return_value="failure")
worker._do_work()
worker.tell_manager.assert_called_once_with("failure")
def test_success_tell_manager(self):
worker = NowcastWorker("worker_name", "description")
worker.init_cli()
worker.logger = Mock(name="logger")
worker.tell_manager = Mock(name="tell_manager")
worker.worker_func = Mock(name="worker_func", return_value="checklist")
worker.success = Mock(name="success_func", return_value="success")
worker._do_work()
worker.tell_manager.assert_called_once_with("success", "checklist")
def test_success_func(self):
worker = NowcastWorker("worker_name", "description")
worker.init_cli()
worker.logger = Mock(name="logger")
worker.tell_manager = Mock(name="tell_manager")
worker.worker_func = Mock(name="worker_func")
worker.success = Mock(name="success_func")
worker._do_work()
worker.success.assert_called_once_with(worker._parsed_args)
