def test_rate_control_auto_default(
self, celery, redis, metricsmock, key, default, value
):
"""Some Redis values are initialized to defaults."""
# TODO: check log entry when switched to structlog
rc_params = {
"rate_controller_target": 1000,
"rate_controller_kp": 1,
"rate_controller_ki": 0.002,
"rate_controller_kd": 0.2,
"rate_controller_enabled": 1,
}
rc_params[key] = value
for name, init in rc_params.items():
if init is not None:
redis.set(name, init)
monitor_queue_size_and_rate_control.delay().get()
assert int(redis.get(key)) == default
raw_state = redis.get("rate_controller_state")
assert raw_state
if value is None:
# Set to default and loaded
assert int(redis.get("rate_controller_enabled")) == 1
assert json.loads(raw_state)
else:
# Set to 0 and disabled
assert int(redis.get("rate_controller_enabled")) == 0
metricsmock.assert_gauge_once("rate_control.locate", value=100.0)
assert raw_state.decode("utf8") == "{}"
def test_empty_queues(self, celery, redis, metricsmock):
"""Empty queues emit metrics and have a rate of 100%"""
monitor_queue_size_and_rate_control.delay().get()
for name in celery.all_queues:
spec = self.expected_queues[name]
expected_tags = [f"queue:{name}", f"queue_type:{spec[0]}"]
if spec[0] == "data":
expected_tags.append(f"data_type:{spec[1]}")
metricsmock.assert_gauge_once("queue", value=0, tags=expected_tags)
metricsmock.assert_gauge_once("rate_control.locate", value=100.0)
def test_trx_history_metric(self, metricsmock, monitor_session, monitor_redis):
"""The MySQL transaxtion history length is converted to a metric."""
monitor_queue_size_and_rate_control.delay().get()
monitor_session.scalar.assert_called_once_with(
"SELECT count FROM information_schema.innodb_metrics"
" WHERE name = 'trx_rseg_history_len';"
)
metricsmock.assert_gauge_once("trx_history.length", value=123)
metricsmock.assert_gauge_once("trx_history.min", 1000)
metricsmock.assert_gauge_once("trx_history.max", 1000000)
metricsmock.assert_gauge_once("trx_history.purging", 0)
def test_emit_metrics_rc_disabled(
self, celery, monitor_redis, monitor_session, metricsmock
):
"""Metrics are emitted even when the rate controller is disabled."""
monitor_redis.set("rate_controller_enabled", 0)
monitor_redis.set("global_locate_sample_rate", "56.4")
monitor_queue_size_and_rate_control.delay().get()
assert monitor_redis.get("global_locate_sample_rate") == b"56.4"
metricsmock.assert_gauge_once("trx_history.length", value=123)
metricsmock.assert_gauge_once("rate_control.locate", value=56.4)
metricsmock.assert_not_gauge("trx_history.purging")
metricsmock.assert_not_gauge("rate_control.locate.target")
def test_trx_history_not_allowed(self, metricsmock, monitor_redis, monitor_session):
"""If MySQL connection does not have PROCESS privilege, then do not send metric."""
monitor_session.scalar.side_effect = OperationalError(
statement="SELECT count FROM information_schema...",
params=[],
orig=Exception(1227),
)
monitor_queue_size_and_rate_control.delay().get()
monitor_session.scalar.assert_called_once()
metricsmock.assert_not_gauge("trx_history.length")
metricsmock.assert_not_gauge("trx_history.min")
metricsmock.assert_not_gauge("trx_history.max")
metricsmock.assert_not_gauge("trx_history.purging")
def test_trx_history_other_error_raised(
self, metricsmock, monitor_session, raven_client
):
"""If a different error is raised, the task raises the exception."""
monitor_session.scalar.side_effect = OperationalError(
statement="SELECT count FROM information_schema...",
params=[],
orig=Exception(1234),
)
with pytest.raises(OperationalError):
monitor_queue_size_and_rate_control.delay().get()
monitor_session.scalar.assert_called_once()
metricsmock.assert_not_gauge("trx_history.length", value=123)
assert len(raven_client.msgs) == 1
raven_client._clear()
def test_trx_history_complete(
self, celery, monitor_redis, monitor_session, metricsmock
):
"""The rate controller exits purging mode below min transaction history."""
monitor_redis.set("rate_controller_trx_min", 124)
monitor_redis.set("rate_controller_trx_max", 200)
monitor_redis.set("rate_controller_trx_purging", 1)
monitor_redis.set("global_locate_sample_rate", 0)
monitor_queue_size_and_rate_control.delay().get()
assert float(monitor_redis.get("global_locate_sample_rate")) == 100.0
assert int(monitor_redis.get("rate_controller_trx_purging")) == 0
metricsmock.assert_gauge_once("trx_history.length", value=123)
metricsmock.assert_gauge_once("trx_history.min", value=124)
metricsmock.assert_gauge_once("trx_history.max", value=200)
metricsmock.assert_gauge_once("trx_history.purging", value=0)
def test_nonempty(self, celery, redis, metricsmock):
"""Non-empty but low queues emit metrics, have a rate of 100%"""
data = {}
for name in celery.all_queues:
data[name] = random.randint(1, 10)
for key, val in data.items():
redis.lpush(key, *range(val))
monitor_queue_size_and_rate_control.delay().get()
for key, val in data.items():
spec = self.expected_queues[key]
expected_tags = [f"queue:{key}", f"queue_type:{spec[0]}"]
if spec[0] == "data":
expected_tags.append(f"data_type:{spec[1]}")
metricsmock.assert_gauge_once("queue", value=val, tags=expected_tags)
metricsmock.assert_gauge_once("rate_control.locate", value=100.0)
def test_rate_control_reload(self, celery, redis):
redis.set("rate_controller_kp", 1.0)
redis.set("rate_controller_ki", 0.002)
redis.set("rate_controller_kd", 0.2)
redis.set("rate_controller_target", 1000)
redis.set("rate_controller_enabled", 1)
redis.lpush("update_wifi_b", *range(10))
old_state = {
"state": "running",
"p_term": 1000,
"i_term": 0.0001,
"d_term": 0.0,
"last_input": 0,
"last_output": 1000,
"last_time": time.monotonic() - 60.0,
}
redis.set("rate_controller_state", json.dumps(old_state))
monitor_queue_size_and_rate_control.delay().get()
assert int(redis.get("rate_controller_enabled")) == 1
# Backlog is well below target, sample rate is 100%
assert float(redis.get("global_locate_sample_rate")) == 100.0
raw_state = redis.get("rate_controller_state")
assert raw_state
state = json.loads(raw_state.decode("utf8"))
assert state == {
"state": "running",
"p_term": 990.0,
"i_term": state["i_term"],
"d_term": state["d_term"],
"last_input": 10,
"last_output": 1000,
"last_time": state["last_time"],
}
assert state["i_term"] != old_state["i_term"]
assert state["d_term"] != old_state["d_term"]
assert state["last_time"] > old_state["last_time"]
def test_rate_control(self, celery, monitor_redis, monitor_session, metricsmock):
"""Rate controller runs and emits metrics when enabled."""
redis = monitor_redis
monitor_queue_size_and_rate_control.delay().get()
assert int(redis.get("rate_controller_enabled")) == 1
assert int(redis.get("rate_controller_trx_purging")) == 0
assert float(redis.get("global_locate_sample_rate")) == 100.0
raw_state = redis.get("rate_controller_state")
assert raw_state
state = json.loads(raw_state.decode("utf8"))
assert state == {
"state": "running",
"p_term": 1000.0,
"i_term": state["i_term"],
"d_term": -0.0,
"last_input": 0,
"last_output": 1000,
"last_time": state["last_time"],
}
metricsmock.assert_gauge_once("rate_control.locate.target", value=1000)
metricsmock.assert_gauge_once("rate_control.locate.kp", value=1)
metricsmock.assert_gauge_once("rate_control.locate.ki", value=0.002)
metricsmock.assert_gauge_once("rate_control.locate.kd", value=0.2)
metricsmock.assert_gauge_once("rate_control.locate.pterm", value=1000)
metricsmock.assert_gauge_once(
"rate_control.locate.iterm", value=state["i_term"]
)
metricsmock.assert_gauge_once(
"rate_control.locate.dterm", value=state["d_term"]
)
metricsmock.assert_gauge_once("rate_control.locate", value=100.0)
metricsmock.assert_gauge_once("trx_history.length", value=123)
metricsmock.assert_gauge_once("trx_history.min", value=1000)
metricsmock.assert_gauge_once("trx_history.max", value=1000000)
metricsmock.assert_gauge_once("trx_history.purging", value=0)
def test_rate_control_auto_disable(self, celery, redis, metricsmock, key, value):
"""When some Redis values fail to validate, it disables the rate controller."""
# TODO: check log entry when switched to structlog
rc_params = {
"rate_controller_target": 1000,
"rate_controller_kp": 1,
"rate_controller_ki": 0.002,
"rate_controller_kd": 0.2,
"rate_controller_enabled": 1,
}
rc_params[key] = value
for name, init in rc_params.items():
if init is not None:
redis.set(name, init)
monitor_queue_size_and_rate_control.delay().get()
assert int(redis.get("rate_controller_enabled")) == 0
assert int(redis.get(key)) == 0
raw_state = redis.get("rate_controller_state")
assert raw_state
assert raw_state.decode("utf8") == "{}"
metricsmock.assert_gauge_once("rate_control.locate", value=100.0)
def test_rate_control_invalid_state(self, celery, redis):
redis.set("rate_controller_kp", 1.0)
redis.set("rate_controller_ki", 0.002)
redis.set("rate_controller_kd", 0.2)
redis.set("rate_controller_target", 9)
redis.set("rate_controller_enabled", 1)
redis.lpush("update_wifi_b", *range(10))
old_state = {
"state": "running",
"p_term": 990.0,
"i_term": 0.0001,
"d_term": 0.0,
"last_input": 10,
"last_output": 100,
# Missing last_time
}
redis.set("rate_controller_state", json.dumps(old_state))
monitor_queue_size_and_rate_control.delay().get()
assert int(redis.get("rate_controller_enabled")) == 1
# Queue size is above target, sample rate is 0%
assert float(redis.get("global_locate_sample_rate")) == 0.0
raw_state = redis.get("rate_controller_state")
assert raw_state
state = json.loads(raw_state.decode("utf8"))
assert state == {
"state": "running",
"p_term": -1.0,
"i_term": 0,
"d_term": -0.0,
"last_input": 10,
"last_output": 0,
"last_time": state["last_time"],
}
assert state["i_term"] != old_state["i_term"]