def test_rate_control_auto_default( self, celery, redis, metricsmock, key, default, value ): """Some Redis values are initialized to defaults.""" # TODO: check log entry when switched to structlog rc_params = { "rate_controller_target": 1000, "rate_controller_kp": 1, "rate_controller_ki": 0.002, "rate_controller_kd": 0.2, "rate_controller_enabled": 1, } rc_params[key] = value for name, init in rc_params.items(): if init is not None: redis.set(name, init) monitor_queue_size_and_rate_control.delay().get() assert int(redis.get(key)) == default raw_state = redis.get("rate_controller_state") assert raw_state if value is None: # Set to default and loaded assert int(redis.get("rate_controller_enabled")) == 1 assert json.loads(raw_state) else: # Set to 0 and disabled assert int(redis.get("rate_controller_enabled")) == 0 metricsmock.assert_gauge_once("rate_control.locate", value=100.0) assert raw_state.decode("utf8") == "{}"
def test_empty_queues(self, celery, redis, metricsmock): """Empty queues emit metrics and have a rate of 100%""" monitor_queue_size_and_rate_control.delay().get() for name in celery.all_queues: spec = self.expected_queues[name] expected_tags = [f"queue:{name}", f"queue_type:{spec[0]}"] if spec[0] == "data": expected_tags.append(f"data_type:{spec[1]}") metricsmock.assert_gauge_once("queue", value=0, tags=expected_tags) metricsmock.assert_gauge_once("rate_control.locate", value=100.0)
def test_trx_history_metric(self, metricsmock, monitor_session, monitor_redis): """The MySQL transaxtion history length is converted to a metric.""" monitor_queue_size_and_rate_control.delay().get() monitor_session.scalar.assert_called_once_with( "SELECT count FROM information_schema.innodb_metrics" " WHERE name = 'trx_rseg_history_len';" ) metricsmock.assert_gauge_once("trx_history.length", value=123) metricsmock.assert_gauge_once("trx_history.min", 1000) metricsmock.assert_gauge_once("trx_history.max", 1000000) metricsmock.assert_gauge_once("trx_history.purging", 0)
def test_emit_metrics_rc_disabled( self, celery, monitor_redis, monitor_session, metricsmock ): """Metrics are emitted even when the rate controller is disabled.""" monitor_redis.set("rate_controller_enabled", 0) monitor_redis.set("global_locate_sample_rate", "56.4") monitor_queue_size_and_rate_control.delay().get() assert monitor_redis.get("global_locate_sample_rate") == b"56.4" metricsmock.assert_gauge_once("trx_history.length", value=123) metricsmock.assert_gauge_once("rate_control.locate", value=56.4) metricsmock.assert_not_gauge("trx_history.purging") metricsmock.assert_not_gauge("rate_control.locate.target")
def test_trx_history_not_allowed(self, metricsmock, monitor_redis, monitor_session): """If MySQL connection does not have PROCESS privilege, then do not send metric.""" monitor_session.scalar.side_effect = OperationalError( statement="SELECT count FROM information_schema...", params=[], orig=Exception(1227), ) monitor_queue_size_and_rate_control.delay().get() monitor_session.scalar.assert_called_once() metricsmock.assert_not_gauge("trx_history.length") metricsmock.assert_not_gauge("trx_history.min") metricsmock.assert_not_gauge("trx_history.max") metricsmock.assert_not_gauge("trx_history.purging")
def test_trx_history_other_error_raised( self, metricsmock, monitor_session, raven_client ): """If a different error is raised, the task raises the exception.""" monitor_session.scalar.side_effect = OperationalError( statement="SELECT count FROM information_schema...", params=[], orig=Exception(1234), ) with pytest.raises(OperationalError): monitor_queue_size_and_rate_control.delay().get() monitor_session.scalar.assert_called_once() metricsmock.assert_not_gauge("trx_history.length", value=123) assert len(raven_client.msgs) == 1 raven_client._clear()
def test_trx_history_complete( self, celery, monitor_redis, monitor_session, metricsmock ): """The rate controller exits purging mode below min transaction history.""" monitor_redis.set("rate_controller_trx_min", 124) monitor_redis.set("rate_controller_trx_max", 200) monitor_redis.set("rate_controller_trx_purging", 1) monitor_redis.set("global_locate_sample_rate", 0) monitor_queue_size_and_rate_control.delay().get() assert float(monitor_redis.get("global_locate_sample_rate")) == 100.0 assert int(monitor_redis.get("rate_controller_trx_purging")) == 0 metricsmock.assert_gauge_once("trx_history.length", value=123) metricsmock.assert_gauge_once("trx_history.min", value=124) metricsmock.assert_gauge_once("trx_history.max", value=200) metricsmock.assert_gauge_once("trx_history.purging", value=0)
def test_nonempty(self, celery, redis, metricsmock): """Non-empty but low queues emit metrics, have a rate of 100%""" data = {} for name in celery.all_queues: data[name] = random.randint(1, 10) for key, val in data.items(): redis.lpush(key, *range(val)) monitor_queue_size_and_rate_control.delay().get() for key, val in data.items(): spec = self.expected_queues[key] expected_tags = [f"queue:{key}", f"queue_type:{spec[0]}"] if spec[0] == "data": expected_tags.append(f"data_type:{spec[1]}") metricsmock.assert_gauge_once("queue", value=val, tags=expected_tags) metricsmock.assert_gauge_once("rate_control.locate", value=100.0)
def test_rate_control_reload(self, celery, redis): redis.set("rate_controller_kp", 1.0) redis.set("rate_controller_ki", 0.002) redis.set("rate_controller_kd", 0.2) redis.set("rate_controller_target", 1000) redis.set("rate_controller_enabled", 1) redis.lpush("update_wifi_b", *range(10)) old_state = { "state": "running", "p_term": 1000, "i_term": 0.0001, "d_term": 0.0, "last_input": 0, "last_output": 1000, "last_time": time.monotonic() - 60.0, } redis.set("rate_controller_state", json.dumps(old_state)) monitor_queue_size_and_rate_control.delay().get() assert int(redis.get("rate_controller_enabled")) == 1 # Backlog is well below target, sample rate is 100% assert float(redis.get("global_locate_sample_rate")) == 100.0 raw_state = redis.get("rate_controller_state") assert raw_state state = json.loads(raw_state.decode("utf8")) assert state == { "state": "running", "p_term": 990.0, "i_term": state["i_term"], "d_term": state["d_term"], "last_input": 10, "last_output": 1000, "last_time": state["last_time"], } assert state["i_term"] != old_state["i_term"] assert state["d_term"] != old_state["d_term"] assert state["last_time"] > old_state["last_time"]
def test_rate_control(self, celery, monitor_redis, monitor_session, metricsmock): """Rate controller runs and emits metrics when enabled.""" redis = monitor_redis monitor_queue_size_and_rate_control.delay().get() assert int(redis.get("rate_controller_enabled")) == 1 assert int(redis.get("rate_controller_trx_purging")) == 0 assert float(redis.get("global_locate_sample_rate")) == 100.0 raw_state = redis.get("rate_controller_state") assert raw_state state = json.loads(raw_state.decode("utf8")) assert state == { "state": "running", "p_term": 1000.0, "i_term": state["i_term"], "d_term": -0.0, "last_input": 0, "last_output": 1000, "last_time": state["last_time"], } metricsmock.assert_gauge_once("rate_control.locate.target", value=1000) metricsmock.assert_gauge_once("rate_control.locate.kp", value=1) metricsmock.assert_gauge_once("rate_control.locate.ki", value=0.002) metricsmock.assert_gauge_once("rate_control.locate.kd", value=0.2) metricsmock.assert_gauge_once("rate_control.locate.pterm", value=1000) metricsmock.assert_gauge_once( "rate_control.locate.iterm", value=state["i_term"] ) metricsmock.assert_gauge_once( "rate_control.locate.dterm", value=state["d_term"] ) metricsmock.assert_gauge_once("rate_control.locate", value=100.0) metricsmock.assert_gauge_once("trx_history.length", value=123) metricsmock.assert_gauge_once("trx_history.min", value=1000) metricsmock.assert_gauge_once("trx_history.max", value=1000000) metricsmock.assert_gauge_once("trx_history.purging", value=0)
def test_rate_control_auto_disable(self, celery, redis, metricsmock, key, value): """When some Redis values fail to validate, it disables the rate controller.""" # TODO: check log entry when switched to structlog rc_params = { "rate_controller_target": 1000, "rate_controller_kp": 1, "rate_controller_ki": 0.002, "rate_controller_kd": 0.2, "rate_controller_enabled": 1, } rc_params[key] = value for name, init in rc_params.items(): if init is not None: redis.set(name, init) monitor_queue_size_and_rate_control.delay().get() assert int(redis.get("rate_controller_enabled")) == 0 assert int(redis.get(key)) == 0 raw_state = redis.get("rate_controller_state") assert raw_state assert raw_state.decode("utf8") == "{}" metricsmock.assert_gauge_once("rate_control.locate", value=100.0)
def test_rate_control_invalid_state(self, celery, redis): redis.set("rate_controller_kp", 1.0) redis.set("rate_controller_ki", 0.002) redis.set("rate_controller_kd", 0.2) redis.set("rate_controller_target", 9) redis.set("rate_controller_enabled", 1) redis.lpush("update_wifi_b", *range(10)) old_state = { "state": "running", "p_term": 990.0, "i_term": 0.0001, "d_term": 0.0, "last_input": 10, "last_output": 100, # Missing last_time } redis.set("rate_controller_state", json.dumps(old_state)) monitor_queue_size_and_rate_control.delay().get() assert int(redis.get("rate_controller_enabled")) == 1 # Queue size is above target, sample rate is 0% assert float(redis.get("global_locate_sample_rate")) == 0.0 raw_state = redis.get("rate_controller_state") assert raw_state state = json.loads(raw_state.decode("utf8")) assert state == { "state": "running", "p_term": -1.0, "i_term": 0, "d_term": -0.0, "last_input": 10, "last_output": 0, "last_time": state["last_time"], } assert state["i_term"] != old_state["i_term"]