def test_activate_race_condition_mgmt(self, m_importlib):
mgr = manager.NowcastManager()
mgr.logger = Mock(name="logger")
mgr._update_checklist = Mock(name="_update_checklist")
mgr._next_workers_module = Mock(
name="nowcast.next_workers",
after_test_worker=Mock(
name="after_download_weather",
return_value=(
[
NextWorker("get_NeahBay_ssh"),
NextWorker("grib_to_netcdf"),
NextWorker("download_live_ocean"),
],
{"grib_to_netcdf", "make_live_ocean_files"},
),
),
)
msg = Message(source="test_worker", type="success")
_, next_workers = mgr._handle_continue_msg(msg)
assert mgr._race_condition_mgmt == {
"must finish": {"grib_to_netcdf", "make_live_ocean_files"},
"then launch": [],
}
assert next_workers == [
NextWorker("get_NeahBay_ssh"),
NextWorker("grib_to_netcdf"),
NextWorker("download_live_ocean"),
]
def test_send_reply(self):
mgr = manager.NowcastManager()
mgr._socket = Mock(name="_socket")
mgr._message_handler = Mock(name="_message_handler",
return_value=("reply", []))
mgr._try_messages()
mgr._socket.send_string.assert_called_once_with("reply")
def test_load_checklist(self):
mgr = manager.NowcastManager()
p_open = patch("nemo_nowcast.manager.open", mock_open(), create=True)
mgr.config = {"checklist file": "nowcast_checklist.yaml"}
mgr.logger = Mock(name="logger")
with p_open as m_open:
mgr._load_checklist()
m_open.assert_called_once_with("nowcast_checklist.yaml", "rt")
def test_change_rotating_logger_handler_to_watched(self, m_logging_config):
mgr = manager.NowcastManager()
mgr.config._dict = self.filesystem_logging_config
mgr._configure_logging()
handler = self.filesystem_logging_config["logging"]["handlers"][
"info_text"]
assert handler["class"] == "logging.handlers.WatchedFileHandler"
assert "backupCount" not in handler
def test_handle_unregistered_msg_type(self):
mgr = manager.NowcastManager()
mgr.logger = Mock(name="logger")
msg = Message(source="worker", type="foo")
reply = mgr._handle_unregistered_msg_type(msg)
assert mgr.logger.error.call_count == 1
assert Message.deserialize(reply) == Message(
source="manager", type="unregistered message type")
def test_worker_checklist_keyerror(self):
mgr = manager.NowcastManager()
mgr._msg_registry = {"workers": {"test_worker": {}}}
msg = Message(source="test_worker",
type="success",
payload={"foo": "bar"})
with pytest.raises(KeyError):
mgr._update_checklist(msg)
def test_worker_not_in_notifications_list(self, m_post):
mgr = manager.NowcastManager()
mgr.config = {"slack notifications": {"SLACK_URL": []}}
msg = Message(source="test_worker", type="success")
with patch.dict(os.environ,
{"SLACK_URL": "https://hooks.slack.com/services/..."}):
mgr._slack_notification(msg)
assert not m_post.called
def test_checklist_cleared_msg_type(self):
mgr = manager.NowcastManager()
mgr.checklist = {"foo": "bar"}
mgr.logger = Mock(name="logger")
mgr._write_checklist_to_disk = Mock(name="_write_checklist_to_disk")
reply = mgr._clear_checklist()
assert Message.deserialize(reply) == Message(source="manager",
type="checklist cleared")
def test_without_checklist_logging(self):
mgr = manager.NowcastManager()
mgr.checklist = {"foo": "bar"}
mgr.logger = Mock(name="logger")
mgr._write_checklist_to_disk = Mock(name="_write_checklist_to_disk")
mgr._clear_checklist()
assert mgr.checklist == {}
assert mgr.logger.info.call_count == 1
def test_zmq_handler_root_topic(self, m_logging, m_logging_config):
mgr = manager.NowcastManager()
mgr.config._dict = self.zmq_logging_config
mgr._configure_logging()
m_handler = Mock(name="m_zmq_handler",
spec=zmq.log.handlers.PUBHandler)
mgr.logger.root = Mock(handlers=[m_handler])
mgr._configure_logging()
assert m_handler.root_topic == "manager"
def test_launch_next_workers(self):
mgr = manager.NowcastManager()
mgr._socket = Mock(name="_socket")
next_worker = NextWorker("nowcast.workers.next_worker")
next_worker.launch = Mock(name="launch")
mgr._message_handler = Mock(name="_message_handler",
return_value=("reply", [next_worker]))
mgr._try_messages()
next_worker.launch.assert_called_once_with(mgr.config, mgr.name)
def test_notification_posted(self, m_post):
mgr = manager.NowcastManager()
mgr.config = {"slack notifications": {"SLACK_URL": ["test_worker"]}}
msg = Message(source="test_worker", type="success")
slack_url = "https://hooks.slack.com/services/..."
with patch.dict(os.environ, {"SLACK_URL": slack_url}):
mgr._slack_notification(msg)
m_post.assert_called_once_with(slack_url,
json={"text": "test_worker: success"})
def test_no_slack_url_envvar(self, m_post):
mgr = manager.NowcastManager()
mgr.logger = Mock(name="logger")
mgr.config = {"slack notifications": {"SLACK_URL": []}}
msg = Message(source="test_worker", type="success")
mgr._slack_notification(msg)
mgr.logger.debug.assert_called_once_with(
"slack notification environment variable not found: SLACK_URL")
assert not m_post.called
def test_missing_after_worker_function(self, m_importlib):
mgr = manager.NowcastManager()
mgr.logger = Mock(name="logger")
mgr._msg_registry = {"next workers module": "nowcast.next_workers"}
mgr._update_checklist = Mock(name="_update_checklist")
msg = Message(source="test_worker", type="success")
reply, next_workers = mgr._handle_continue_msg(msg)
assert Message.deserialize(reply) == Message(
source="manager", type="no after_worker function")
assert mgr.logger.critical.call_count == 1
def test_reload_next_workers_module(self, m_importlib):
mgr = manager.NowcastManager()
mgr._update_checklist = Mock(name="_update_checklist")
mgr._next_workers_module = Mock(
name="nowcast.next_workers",
after_test_worker=Mock(name="after_test_worker", return_value=[]),
)
msg = Message(source="test_worker", type="success")
mgr._handle_continue_msg(msg)
m_importlib.reload.assert_called_once_with(mgr._next_workers_module)
def test_slack_notification(self, m_importlib):
mgr = manager.NowcastManager()
mgr._slack_notification = Mock(name="_slack_notification")
mgr._next_workers_module = Mock(
name="nowcast.next_workers",
after_test_worker=Mock(name="after_test_worker", return_value=[]),
)
msg = Message(source="test_worker", type="success")
mgr._handle_continue_msg(msg)
assert mgr._slack_notification.called
def test_no_checklist_update_when_no_payload(self, m_importlib):
mgr = manager.NowcastManager()
mgr._update_checklist = Mock(name="_update_checklist")
mgr._next_workers_module = Mock(
name="nowcast.next_workers",
after_test_worker=Mock(name="after_test_worker", return_value=[]),
)
msg = Message(source="test_worker", type="success")
mgr._handle_continue_msg(msg)
assert not mgr._update_checklist.called
def test_logging_dictConfig(self, m_logging_config, config):
mgr = manager.NowcastManager()
mgr.config._dict = config
mgr._configure_logging()
if "publisher" in config["logging"]:
m_logging_config.dictConfig.assert_called_once_with(
config["logging"]["publisher"])
else:
m_logging_config.dictConfig.assert_called_once_with(
config["logging"])
class TestHandleNeedMsg:
"""Unit test for NowcastManager._handle_need_msg method.
"""
mgr = manager.NowcastManager()
mgr.checklist = {"info": "requested info"}
msg = Message(source="test_worker", type="need", payload="info")
reply = mgr._handle_need_msg(msg)
expected = Message("manager", "ack", payload="requested info").serialize()
assert reply == expected
def test_reply(self, m_importlib):
mgr = manager.NowcastManager()
mgr._update_checklist = Mock(name="_update_checklist")
mgr._next_workers_module = Mock(
name="nowcast.next_workers",
after_test_worker=Mock(name="after_test_worker", return_value=[]),
)
msg = Message(source="test_worker", type="success")
reply, next_workers = mgr._handle_continue_msg(msg)
assert Message.deserialize(reply) == Message(source="manager",
type="ack")
def test_unregistered_msg_type(self):
mgr = manager.NowcastManager()
mgr._msg_registry = {"workers": {"test_worker": {}}}
mgr._handle_unregistered_msg_type = Mock(
name="_handle_unregistered_msg_type")
mgr._log_received_msg = Mock(name="_log_received_msg")
msg = Message(source="test_worker", type="foo", payload=None)
reply, next_workers = mgr._message_handler(msg.serialize())
mgr._handle_unregistered_msg_type.assert_called_once_with(msg)
assert reply == mgr._handle_unregistered_msg_type()
assert next_workers == []
assert not mgr._log_received_msg.called
def test_one_next_worker_no_race_condition_mgmt(self, m_importlib):
mgr = manager.NowcastManager()
mgr._update_checklist = Mock(name="_update_checklist")
mgr._next_workers_module = Mock(
name="nowcast.next_workers",
after_test_worker=Mock(
name="after_test_worker",
return_value=[NextWorker("another_test_worker")],
),
)
msg = Message(source="test_worker", type="success", payload=None)
reply, next_workers = mgr._handle_continue_msg(msg)
assert next_workers == [NextWorker("another_test_worker")]
def test_need_msg(self):
mgr = manager.NowcastManager()
mgr._msg_registry = {"workers": {"test_worker": {"need": "Need info"}}}
mgr._handle_need_msg = Mock(
name="_handle_need_msg",
return_value=("ack msg w/ requested info in payload"),
)
mgr._log_received_msg = Mock(name="_log_received_msg")
msg = Message(source="test_worker", type="need", payload="info")
reply, next_workers = mgr._message_handler(msg.serialize())
assert mgr._log_received_msg.called
mgr._handle_need_msg.assert_called_once_with(msg)
assert reply == "ack msg w/ requested info in payload"
assert next_workers == []
def test_with_checklist_logging(self):
mgr = manager.NowcastManager()
mgr.checklist = {"foo": "bar"}
mgr.logger = Mock(name="logger")
mgr._write_checklist_to_disk = Mock(name="_write_checklist_to_disk")
with patch("nemo_nowcast.manager.logging") as m_logging:
m_checklist_logger = m_logging.getLogger("checklist")
m_checklist_logger.handlers = [Mock(name="checklist", level=1000)]
m_checklist_logger.handlers[0].name = "checklist"
mgr._clear_checklist()
m_checklist_logger.log.assert_called_once_with(
1000, "checklist:\n{'foo': 'bar'}")
assert mgr.checklist == {}
assert mgr.logger.info.call_count == 2
def test_config_load(self, m_importlib, m_logging):
mgr = manager.NowcastManager()
mgr.config._dict = {
"logging": {
"handlers": {}
},
"message registry": {
"next workers module": "nowcast.next_workers",
"workers": {},
},
}
mgr.config.load = Mock()
mgr._cli = Mock(name="_cli")
mgr.setup()
mgr.config.load.assert_called_once_with(mgr._parsed_args.config_file)
def test_parsed_args(self, m_importlib, m_logging):
test_config = """
checklist file: nowcast_checklist.yaml
python: python
logging:
handlers: []
message registry:
next workers module: nowcast.next_workers
"""
mgr = manager.NowcastManager()
mgr._cli = Mock(name="_cli")
with patch("nemo_nowcast.config.open",
mock_open(read_data=test_config)):
mgr.setup()
assert mgr._parsed_args == mgr._cli()
def test_yaml_dump_checklist_to_disk(self):
mgr = manager.NowcastManager()
mgr.logger = Mock(name="logger")
mgr.checklist = {"foo": "bar"}
mgr._write_checklist_to_disk = Mock(name="_write_checklist_to_disk")
mgr._msg_registry = {
"workers": {
"test_worker": {
"checklist key": "foo"
}
}
}
msg = Message(source="test_worker", type="success", payload="baz")
mgr._update_checklist(msg)
mgr._write_checklist_to_disk.assert_called_once_with()
def test_keyerror_adds_key_and_value(self):
mgr = manager.NowcastManager()
mgr.logger = Mock(name="logger")
mgr.checklist = {"foo": "bar"}
mgr._write_checklist_to_disk = Mock(name="_write_checklist_to_disk")
mgr._msg_registry = {
"workers": {
"test_worker": {
"checklist key": "fop"
}
}
}
msg = Message(source="test_worker", type="success", payload="baz")
mgr._update_checklist(msg)
assert mgr.checklist["fop"] == "baz"
def test_next_workers_module_import_error(self, m_logging):
mgr = manager.NowcastManager()
mgr.config._dict = {
"logging": {
"handlers": {}
},
"message registry": {
"next workers module": "nowcast.next_workers",
"workers": {},
},
}
mgr.config.load = Mock()
mgr._cli = Mock(name="_cli")
with pytest.raises(ImportError):
mgr.setup()
def test_logging_info(self, m_importlib, m_logging):
mgr = manager.NowcastManager()
mgr.config._dict = {
"logging": {
"handlers": {}
},
"message registry": {
"next workers module": "nowcast.next_workers",
"workers": {},
},
}
mgr.config.load = Mock()
mgr._cli = Mock(name="_cli")
mgr.setup()
assert mgr.logger.info.call_count == 4
