def test_main_loop(db):
config = {
"remote_conns": {
"test1db": db.connection_string("testuser"),
"test2db": db.connection_string("otheruser"),
},
}
cluster_state = {}
observer_state = {}
def create_alert_file(arg):
raise Exception(arg)
cluster_monitor_check_queue = Queue()
failover_decision_queue = Queue()
cm = ClusterMonitor(
config=config,
cluster_state=cluster_state,
observer_state=observer_state,
create_alert_file=create_alert_file,
cluster_monitor_check_queue=cluster_monitor_check_queue,
failover_decision_queue=failover_decision_queue,
stats=statsd.StatsClient(host=None),
)
cm.main_monitoring_loop(requested_check=True)
assert len(cm.cluster_state) == 2
assert "test1db" in cm.cluster_state
assert "test2db" in cm.cluster_state
assert failover_decision_queue.get(timeout=5) == "Completed requested monitoring loop"
def test_main_loop(db):
config = {
"remote_conns": {
"test1db": db.connection_string("testuser"),
"test2db": db.connection_string("otheruser"),
},
}
cluster_state = {}
observer_state = {}
def create_alert_file(arg):
raise Exception(arg)
trigger_check_queue = Queue()
trigger_check_queue.put(
"test entry so we don't wait five seconds to get one")
cm = ClusterMonitor(
config=config,
cluster_state=cluster_state,
observer_state=observer_state,
create_alert_file=create_alert_file,
trigger_check_queue=trigger_check_queue,
stats=statsd.StatsClient(host=None),
)
cm.main_monitoring_loop()
assert len(cm.cluster_state) == 2
assert "test1db" in cm.cluster_state
assert "test2db" in cm.cluster_state
def test_main_loop(db):
config = {
"remote_conns": {
"test1db": db.connection_string("testuser"),
"test2db": db.connection_string("otheruser"),
},
"observers": {
"local": "URL"
},
"poll_observers_on_warning_only": True
}
cluster_state = {}
observer_state = {}
def create_alert_file(arg):
raise Exception(arg)
cluster_monitor_check_queue = Queue()
failover_decision_queue = Queue()
cm = ClusterMonitor(
config=config,
cluster_state=cluster_state,
observer_state=observer_state,
create_alert_file=create_alert_file,
cluster_monitor_check_queue=cluster_monitor_check_queue,
failover_decision_queue=failover_decision_queue,
stats=statsd.StatsClient(host=None),
is_replication_lag_over_warning_limit=lambda: False)
cm.main_monitoring_loop(requested_check=True)
assert len(cm.cluster_state) == 2
assert "test1db" in cm.cluster_state
assert "test2db" in cm.cluster_state
assert failover_decision_queue.get(
timeout=5) == "Completed requested monitoring loop"
with patch.object(cm, "fetch_observer_state") as fetch_observer_state:
cm.main_monitoring_loop(requested_check=True)
fetch_observer_state.assert_not_called()
with patch.object(cm, "fetch_observer_state") as fetch_observer_state:
with patch.object(cm, "is_replication_lag_over_warning_limit",
lambda: True):
cm.main_monitoring_loop(requested_check=True)
fetch_observer_state.assert_called_once_with("local", "URL")
with patch.object(cm, "fetch_observer_state") as fetch_observer_state:
with patch.dict(cm.config, {"poll_observers_on_warning_only": False}):
cm.main_monitoring_loop(requested_check=True)
fetch_observer_state.assert_called_once_with("local", "URL")
def test_replication_lag():
# pylint: disable=protected-access
now = datetime.now()
status = {
"db_time": now,
"pg_is_in_recovery": True,
"pg_last_xact_replay_timestamp": now,
"pg_last_xlog_receive_location": "0/0000001",
"pg_last_xlog_replay_location": "0/0000002",
}
result = ClusterMonitor._parse_status_query_result(status.copy())
assert result["replication_time_lag"] == 0.0
status["db_time"] += timedelta(seconds=50, microseconds=42)
result = ClusterMonitor._parse_status_query_result(status.copy())
assert result["replication_time_lag"] == 50.000042
status["db_time"] = now + timedelta(hours=42)
result = ClusterMonitor._parse_status_query_result(status.copy())
assert result["replication_time_lag"] == 151200.0
def test_main_loop(psycopg2_connect):
config = {"remote_conns":
{"foo": "host=1.2.3.4 dbname=postgres user=pglookout password=fake_pass",
"bar": "host=2.3.4.5 dbname=postgres user=pglookout password=fake_pass"}}
cluster_state = {}
observer_state = {}
def alert_file_func(arg): # pylint: disable=unused-argument
pass
create_alert_file = alert_file_func
trigger_check_queue = Queue()
trigger_check_queue.put("test entry so we don't wait five seconds to get one")
class FakeCursor(MagicMock): # pylint: disable=too-many-ancestors
def execute(self, query): # pylint: disable=no-self-use,unused-argument
return
def fetchone(self): # pylint: disable=no-self-use
return {"pg_is_in_recovery": False, "pg_last_xact_replay_timestamp": datetime.utcnow(),
"db_time": datetime.utcnow()}
class FakeConn(Mock): # pylint: disable=too-many-ancestors
def cursor(self, cursor_factory): # pylint: disable=unused-argument
f = FakeCursor()
f.connection = self # pylint: disable=attribute-defined-outside-init
return f
def poll(self): # pylint: disable=no-self-use
return POLL_OK
psycopg2_connect.return_value = FakeConn()
cm = ClusterMonitor(
config=config,
cluster_state=cluster_state,
observer_state=observer_state,
create_alert_file=create_alert_file,
trigger_check_queue=trigger_check_queue)
cm.main_monitoring_loop()
assert len(cm.cluster_state) == 2
assert "foo" in cm.cluster_state
assert "bar" in cm.cluster_state
