def test_models_process(self, events):
reporter = PrometheusReporter("prometheus")
for event in events:
logger.info(f"Sending event to reporter: {event}")
reporter.process(event=event)
# Make sure Prometheus events counter was incremented
assert (
REGISTRY.get_sample_value(
"karrot_reporter_events_count_total",
labels={"reporter": reporter._name},
)
> 0.0
)
# Make sure Prometheus lag is matching totallag
assert (
REGISTRY.get_sample_value(
"karrot_consumer_lag",
labels={
"cluster": reporter._event.Result.cluster,
"consumer": reporter._event.Result.group,
},
)
== reporter._event.Result.totallag
)
def test_enabled(self):
self.assertEqual(1, REGISTRY.get_sample_value('python_gc_enabled'))
try:
gc.disable()
self.assertEqual(0, REGISTRY.get_sample_value('python_gc_enabled'))
finally:
gc.enable()
def test_app_manager_rolls_queries_and_populates_metrics(self):
app_mgr = ApplicationQueryManager(self.fw_query)
app_mgr.validate_query_definitions()
app_mgr.collect_application_query_results()
after = REGISTRY.get_sample_value('extra_metrics_application_version',
labels={
"query_name":
"Adobe Acrobat Reader Win",
"application_version": "12.1",
"query_id": "101"
})
self.assertEqual(1, after)
after = REGISTRY.get_sample_value('extra_metrics_application_version',
labels={
"query_name":
"Adobe Acrobat Reader Win",
"application_version":
"20.009.20067",
"query_id": "101"
})
self.assertEqual(2, after)
after = REGISTRY.get_sample_value('extra_metrics_application_version',
labels={
"query_name": "Zoom Win",
"application_version": "20.1",
"query_id": "105"
})
self.assertEqual(1, after)
def _assert_task_states(self, states, cnt):
for state in states:
assert REGISTRY.get_sample_value('celery_tasks',
labels=dict(state=state)) == cnt
task_by_name_label = dict(state=state, name=self.task)
assert REGISTRY.get_sample_value('celery_tasks_by_name',
labels=task_by_name_label) == cnt
def test_tasks_events(self):
task_uuid = uuid()
hostname = 'myhost'
local_received = time()
latency_before_started = 123.45
runtime = 234.5
m = MonitorThread(app=self.app)
self._assert_task_states(celery.states.ALL_STATES, 0)
assert REGISTRY.get_sample_value('celery_task_latency_count') == 0
assert REGISTRY.get_sample_value('celery_task_latency_sum') == 0
m._process_event(Event(
'task-received', uuid=task_uuid, name=self.task,
args='()', kwargs='{}', retries=0, eta=None, hostname=hostname,
clock=0,
local_received=local_received))
self._assert_all_states({celery.states.RECEIVED})
m._process_event(Event(
'task-started', uuid=task_uuid, hostname=hostname,
clock=1, name=self.task,
local_received=local_received + latency_before_started))
self._assert_all_states({celery.states.STARTED})
m._process_event(Event(
'task-succeeded', uuid=task_uuid, result='42',
runtime=runtime, hostname=hostname, clock=2,
local_received=local_received + latency_before_started + runtime))
self._assert_all_states({celery.states.SUCCESS})
assert REGISTRY.get_sample_value('celery_task_latency_count') == 1
self.assertAlmostEqual(REGISTRY.get_sample_value(
'celery_task_latency_sum'), latency_before_started)
def test_default():
app = create_app()
Instrumentator().add(metrics.default()).instrument(app).expose(app)
client = TestClient(app)
client.get("/", data="fefeef")
client.get("/")
_ = get_response(client, "/metrics")
assert (REGISTRY.get_sample_value(
"http_requests_total",
{
"handler": "/",
"method": "GET",
"status": "2xx"
},
) > 0)
assert (REGISTRY.get_sample_value(
"http_request_size_bytes_sum",
{"handler": "/"},
) > 0)
assert (REGISTRY.get_sample_value(
"http_response_size_bytes_sum",
{"handler": "/"},
) > 0)
assert (REGISTRY.get_sample_value(
"http_request_duration_highr_seconds_sum",
{},
) > 0)
assert (REGISTRY.get_sample_value(
"http_request_duration_seconds_sum",
{"handler": "/"},
) > 0)
def test_histo_tween_factory(self):
prom.includeme(self.conf)
tween = tweenview.histo_tween_factory(view, self.conf.registry)
req = testing.DummyRequest(matched_route=DummyRoute())
tween(req)
inf = REGISTRY.get_sample_value(
'pyramid_request_bucket',
{
'path_info_pattern': '/bars/{id}',
'method': 'GET',
'le': '+Inf',
'status': '200'
},
)
self.assertEqual(inf, 1.)
ingress = REGISTRY.get_sample_value(
'pyramid_request_ingress',
{
'path_info_pattern': '/bars/{id}',
'method': 'GET',
},
)
self.assertEqual(ingress, 0.)
def test_no_buildinfo(self):
print("test_no_buildinfo")
try:
os.remove(PROM_BUILD_FILE)
except OSError as err:
print("Nothing to cleanup")
# Unregister all collectors.
collectors = list(REGISTRY._collector_to_names.keys())
print(f"before unregister collectors={collectors}")
for collector in collectors:
REGISTRY.unregister(collector)
print(f"after unregister collectors={list(REGISTRY._collector_to_names.keys())}")
python_version_info = sys.version_info
python_version = "{}.{}.{}".format(python_version_info.major, python_version_info.minor,
python_version_info.micro)
labels = {"branch": "master",
"revision": "abcdef",
"pythonversion": python_version,
"version": "1.0.0"}
before = REGISTRY.get_sample_value("test_app_build_info", labels)
print("before: {}".format(before))
from prometheus_build_info import metrics
after = REGISTRY.get_sample_value("test_app_build_info", labels)
print("after: {}".format(after))
self.assertEqual(before, None)
self.assertEqual(after, None)
def test_metric(self):
print("test_metric")
# Setup build_info
runner = CliRunner()
runner.invoke(builder.make_build_info, ["test_app", "master", "abcdef", "1.0.0"])
python_version_info = sys.version_info
python_version = "{}.{}.{}".format(python_version_info.major, python_version_info.minor,
python_version_info.micro)
# Unregister all collectors.
collectors = list(REGISTRY._collector_to_names.keys())
print(f"before unregister collectors={collectors}")
for collector in collectors:
REGISTRY.unregister(collector)
print(f"after unregister collectors={list(REGISTRY._collector_to_names.keys())}")
labels = {"branch": "master",
"revision": "abcdef",
"pythonversion": python_version,
"version": "1.0.0"}
# Test
before = REGISTRY.get_sample_value("test_app_build_info", labels)
from prometheus_build_info import metrics
after = REGISTRY.get_sample_value("test_app_build_info", labels)
self.assertEqual(before, None)
self.assertEqual(after, 1.0)
print("before: {}".format(before))
print("after: {}".format(after))
# Cleanup
os.remove(PROM_BUILD_FILE)
def _assert_task_states(self, states, cnt):
for state in states:
assert REGISTRY.get_sample_value(
'celery_tasks', labels=dict(state=state)) == cnt
task_by_name_label = dict(state=state, name=self.task)
assert REGISTRY.get_sample_value(
'celery_tasks_by_name', labels=task_by_name_label) == cnt
def test_include():
cls = vaping.plugin.get_plugin_class("prometheus")
config = dict(
port=12345,
)
prom = cls(config, None)
emit_data = dict(
data=[
dict(
host="host0",
min=1.2,
max=2.5,
avg=1.6,
cnt=5,
loss=1.0,
)
]
)
# need to find where this gets put in the registry
REGISTRY.get_sample_value("number_of_packets_sent")
prom.emit(emit_data)
"""
def test_debug(self):
self.assertEqual(0, REGISTRY.get_sample_value('python_gc_debug'))
try:
gc.set_debug(gc.DEBUG_STATS)
self.assertEqual(gc.DEBUG_STATS, REGISTRY.get_sample_value(
'python_gc_enabled'))
finally:
gc.set_debug(0)
def test_add_filtered_to_empty_set_metric(prometheus_noop_scenario):
before = REGISTRY.get_sample_value(FILTERED_TO_EMPTY_SET_METRIC_NAME)
assert before == 0
prometheus_noop_scenario.execute()
after = REGISTRY.get_sample_value(FILTERED_TO_EMPTY_SET_METRIC_NAME)
assert after == 1
def test_db_is_not_connected(self, mock_connection):
"""Test that db is not connected, log at ERROR level and counter increments."""
mock_connection.cursor.side_effect = OperationalError("test exception")
logging.disable(logging.NOTSET)
before = REGISTRY.get_sample_value("db_connection_errors_total")
with self.assertLogs(logger="koku.metrics", level=logging.ERROR):
DatabaseStatus().connection_check()
after = REGISTRY.get_sample_value("db_connection_errors_total")
self.assertEqual(1, after - before)
def test_celery_task(self, mock_collect):
"""Test celery task to increment prometheus counter."""
before = REGISTRY.get_sample_value('db_connection_errors_total')
with mock.patch('django.db.backends.utils.CursorWrapper') as mock_cursor:
mock_cursor.side_effect = OperationalError('test exception')
mock_collect.return_value = []
task = collect_metrics.s().apply()
self.assertTrue(task.successful())
after = REGISTRY.get_sample_value('db_connection_errors_total')
self.assertEqual(1, after - before)
def test_get_except(self, mock_get):
"""Test handling of request with ConnectionError."""
before = REGISTRY.get_sample_value("rbac_connection_errors_total")
rbac = RbacService()
url = f"{rbac.protocol}://{rbac.host}:{rbac.port}{rbac.path}"
with self.assertRaises(RbacConnectionError):
rbac._request_user_access(url, headers={}) # pylint: disable=protected-access
after = REGISTRY.get_sample_value("rbac_connection_errors_total")
self.assertEqual(1, after - before)
mock_get.assert_called()
def test_state_to_metric(self):
"""Test the state_to_metric function."""
self.processor.state = Report.FAILED_DOWNLOAD
self.processor.account_number = '1234'
failed_download_before = \
REGISTRY.get_sample_value(
'failed_download', {'account_number': '1234'}) or 0.0
self.processor.record_failed_state_metrics()
failed_download_after = REGISTRY.get_sample_value(
'failed_download', {'account_number': '1234'})
self.assertEqual(1.0,
float(failed_download_after) - failed_download_before)
def wrapper(*args, **kw):
if pano_processsed_status:
before = REGISTRY.get_sample_value('pano_process_success')
else:
before = REGISTRY.get_sample_value('pano_process_failure')
f(*args, **kw)
if pano_processsed_status:
after = REGISTRY.get_sample_value('pano_process_success')
else:
after = REGISTRY.get_sample_value('pano_process_failure')
assert 1 == after - before
def test_prometheus_metrics_metric_without_service_name_label():
metrics = PrometheusMetricsFactory()
gauge = metrics.create_gauge(
name='jaeger:test_gauge_without_service_name_label')
gauge(1)
gauge_after = REGISTRY.get_sample_value(
'jaeger:test_gauge_without_service_name_label')
counter = metrics.create_counter(
name='jaeger:test_counter_without_service_name_label')
counter(1)
counter_after = REGISTRY.get_sample_value(
'jaeger:test_counter_without_service_name_label')
assert 1 == counter_after
assert 1 == gauge_after
def test_requests_made_counter(self):
method, path = 'TEST', 'PATH'
before = REGISTRY.get_sample_value('http_requests_total', {
'method': method,
'path': path
}) or 0
trigger_metrics(method, path)
after = REGISTRY.get_sample_value('http_requests_total', {
'method': method,
'path': path
})
assert after is not None
assert after - before == 1
def test_rounding():
app = create_app()
Instrumentator(should_round_latency_decimals=True).add(
metrics.latency(buckets=(
1,
2,
3,
))).instrument(app).expose(app)
client = TestClient(app)
get_response(client, "/")
get_response(client, "/")
get_response(client, "/")
_ = get_response(client, "/metrics")
result = REGISTRY.get_sample_value(
"http_request_duration_seconds_sum",
{
"handler": "/",
"method": "GET",
"status": "2xx"
},
)
entropy = calc_entropy(str(result).split(".")[1][4:])
assert entropy < 10
def test_set_zero_on_queue_length_when_an_channel_layer_error_occurs_during_queue_read(self):
instance = QueueLenghtMonitoringThread(app=self.app, queue_list=['noqueue'])
instance.measure_queues_length()
sample = REGISTRY.get_sample_value('celery_queue_length', {'queue_name':'noqueue'})
self.assertEqual(0.0, sample)
def test_rootLoggerExports(self):
logging.error('There was an error.')
self.assertEquals(
1, REGISTRY.get_sample_value('python_logging_messages_total',
labels={'logger': 'test_levels',
'level': 'ERROR'}))
def test_requests_exceptions_counter(self):
method, path = 'TEST', 'PATH'
before = REGISTRY.get_sample_value('http_exceptions_total', {
'method': method,
'path': path
}) or 0
with pytest.raises(ValueError):
trigger_metrics(method, path, raise_exc=True)
after = REGISTRY.get_sample_value('http_exceptions_total', {
'method': method,
'path': path
})
assert after is not None
assert after - before == 1
def test_set_zero_on_queue_length_when_an_channel_layer_error_occurs_during_queue_read(self):
instance = QueueLengthMonitoringThread(app=self.app, queue_list=['noqueue'])
instance.measure_queues_length()
sample = REGISTRY.get_sample_value('celery_queue_length', {'queue_name':'noqueue'})
self.assertEqual(0.0, sample)
def test_prometheus_metrics_guage():
metrics = PrometheusMetricsFactory(namespace='test')
gauge = metrics.create_gauge(name='jaeger:test_gauge',
tags={'result': 'ok'})
gauge(1)
after = REGISTRY.get_sample_value('test_jaeger:test_gauge',
{'result': 'ok'})
assert 1 == after
def test_prometheus_metrics_counter():
metrics = PrometheusMetricsFactory(namespace='test')
counter1 = metrics.create_counter(name='jaeger:test_counter', tags={'result': 'ok'})
counter1(1)
counter2 = metrics.create_counter(name='jaeger:test_counter', tags={'result': 'ok'})
counter2(1)
after = REGISTRY.get_sample_value('test_jaeger:test_counter', {'result': 'ok'})
assert 2 == after
def test_prometheus_metrics_metric_with_service_name_label():
metrics = PrometheusMetricsFactory(service_name_label='test')
gauge = metrics.create_gauge(
name='jaeger:test_gauge_with_service_name_label')
gauge(1)
gauge_after = REGISTRY.get_sample_value(
'jaeger:test_gauge_with_service_name_label', {'service': 'test'})
counter = metrics.create_counter(
name='jaeger:test_counter_with_service_name_label', tags={'x': 'y'})
counter(1)
counter_after = REGISTRY.get_sample_value(
'jaeger:test_counter_with_service_name_label', {
'service': 'test',
'x': 'y'
})
assert 1 == counter_after
assert 1 == gauge_after
def test_workers_count(self, ping_result, workers_count, mocker, app):
monitor = WorkerMonitoringThread(app=app)
mock_ping = mocker.patch.object(app.control, 'ping')
mock_ping.return_value = ping_result
monitor.update_workers_count()
assert REGISTRY.get_sample_value('celery_workers') == workers_count
def test_prometheus_metrics_guage():
metrics = PrometheusMetricsFactory(namespace='test')
gauge = metrics.create_gauge(name='jaeger:test_gauge',
tags={'result': 'ok'})
gauge(1)
after = REGISTRY.get_sample_value('test_jaeger:test_gauge',
{'result': 'ok'})
assert 1 == after
def test_exportsPython(self):
self.assertEqual(
1,
REGISTRY.get_sample_value('pip_system_components_map_total',
labels={
'component': 'python',
'version': sys.version,
}))
def test_add_probability_filter_passed_no_nodes_metric(prometheus_noop_scenario):
"""
Ensures that add_probability_filter_passed_no_nodes_metric is called when
the filter decides to pass no nodes based on a probability
"""
assert prometheus_noop_scenario.name == "test scenario"
random.seed(6) # make the tests deterministic
candidates = [make_dummy_object()]
before = REGISTRY.get_sample_value(PROBABILITY_FILTER_NOT_PASSED_METRIC_NAME)
assert before == 0
criterion = {"probabilityPassAll": 0.00000001}
prometheus_noop_scenario.filter_probability(candidates, criterion)
after = REGISTRY.get_sample_value(PROBABILITY_FILTER_NOT_PASSED_METRIC_NAME)
assert after == 1
def test_request_latency_count(self):
method, path = 'TEST', 'PATH'
before = REGISTRY.get_sample_value(
'http_requests_latency_seconds_count', {
'method': method,
'path': path
}) or 0
trigger_metrics(method, path, wait=True)
after = REGISTRY.get_sample_value(
'http_requests_latency_seconds_count', {
'method': method,
'path': path
})
assert after is not None
assert after - before == 1
def test_setLevel(self):
logger = logging.getLogger('test_setLevel')
logger.setLevel(logging.CRITICAL)
logger.debug('debug message')
logger.info('info message')
logger.warn('warn message')
logger.error('error message')
logger.critical('critical message')
for level in ('DEBUG', 'INFO', 'WARNING', 'ERROR'):
self.assertEquals(
None,
REGISTRY.get_sample_value('python_logging_messages_total',
labels={'logger': 'test_setLevel',
'level': level}))
self.assertEquals(
1, REGISTRY.get_sample_value('python_logging_messages_total',
labels={'logger': 'test_levels',
'level': 'CRITICAL'}))
def test_assign_report_slice_new(self):
"""Test the assign report slice function with only a new report slice."""
self.report_slice.state = ReportSlice.NEW
self.report_slice.save()
self.processor.report_or_slice = None
self.processor.assign_object()
self.assertEqual(self.processor.report_or_slice, self.report_slice)
queued_slices = REGISTRY.get_sample_value('queued_report_slices')
self.assertEqual(queued_slices, 1)
def test_prometheus_metrics_counter():
metrics = PrometheusMetricsFactory(namespace='test')
counter1 = metrics.create_counter(name='jaeger:test_counter',
tags={'result': 'ok'})
counter1(1)
counter2 = metrics.create_counter(name='jaeger:test_counter',
tags={'result': 'ok'})
counter2(1)
after = REGISTRY.get_sample_value('test_jaeger:test_counter',
{'result': 'ok'})
assert 2 == after
def test_can_measure_queue_length(self):
celery_app = get_celery_app(queue='realqueue')
sample_task = SampleTask()
sample_task.app = celery_app
monitoring_thread_instance = QueueLenghtMonitoringThread(celery_app, queue_list=['realqueue'])
sample_task.delay()
monitoring_thread_instance.measure_queues_length()
sample = REGISTRY.get_sample_value('celery_queue_length', {'queue_name':'realqueue'})
self.assertEqual(1.0, sample)
def test_workers_count(self):
assert REGISTRY.get_sample_value('celery_workers') == 0
with patch.object(self.app.control, 'ping') as mock_ping:
w = WorkerMonitoringThread(app=self.app)
mock_ping.return_value = []
w.update_workers_count()
assert REGISTRY.get_sample_value('celery_workers') == 0
mock_ping.return_value = [0] # 1 worker
w.update_workers_count()
assert REGISTRY.get_sample_value('celery_workers') == 1
mock_ping.return_value = [0, 0] # 2 workers
w.update_workers_count()
assert REGISTRY.get_sample_value('celery_workers') == 2
mock_ping.return_value = []
w.update_workers_count()
assert REGISTRY.get_sample_value('celery_workers') == 0
def test_thresholds(self):
self.assertTrue(REGISTRY.get_sample_value(
'python_gc_threshold', labels={'generation': '0'}) is not None)
self.assertTrue(REGISTRY.get_sample_value(
'python_gc_threshold', labels={'generation': '1'}) is not None)
self.assertTrue(REGISTRY.get_sample_value(
'python_gc_threshold', labels={'generation': '2'}) is not None)
original_thresholds = gc.get_threshold()
try:
gc.disable()
gc.set_threshold(42, 43, 44)
self.assertEqual(42, REGISTRY.get_sample_value(
'python_gc_threshold', labels={'generation': '0'}))
self.assertEqual(43, REGISTRY.get_sample_value(
'python_gc_threshold', labels={'generation': '1'}))
self.assertEqual(44, REGISTRY.get_sample_value(
'python_gc_threshold', labels={'generation': '2'}))
finally:
gc.set_threshold(*original_thresholds)
gc.enable()
def test_tasks_events(self):
task_uuid = uuid()
hostname = 'myhost'
local_received = time()
latency_before_started = 123.45
runtime = 234.5
m = MonitorThread(app=self.app)
self._assert_task_states(celery.states.ALL_STATES, 0)
assert REGISTRY.get_sample_value('celery_task_latency_count') == 0
assert REGISTRY.get_sample_value('celery_task_latency_sum') == 0
m._process_event(Event(
'task-received', uuid=task_uuid, name=self.task,
args='()', kwargs='{}', retries=0, eta=None, hostname=hostname,
clock=0,
local_received=local_received))
self._assert_all_states({celery.states.RECEIVED})
m._process_event(Event(
'task-started', uuid=task_uuid, hostname=hostname,
clock=1, name=self.task,
local_received=local_received + latency_before_started))
self._assert_all_states({celery.states.STARTED})
m._process_event(Event(
'task-succeeded', uuid=task_uuid, result='42',
runtime=runtime, hostname=hostname, clock=2,
local_received=local_received + latency_before_started + runtime))
self._assert_all_states({celery.states.SUCCESS})
assert REGISTRY.get_sample_value('celery_task_latency_count') == 1
self.assertAlmostEqual(REGISTRY.get_sample_value(
'celery_task_latency_sum'), latency_before_started)
assert REGISTRY.get_sample_value(
'celery_tasks_runtime_seconds_count',
labels=dict(name=self.task)) == 1
assert REGISTRY.get_sample_value(
'celery_tasks_runtime_seconds_sum',
labels=dict(name=self.task)) == 234.5
def getMetric(self, metric_name, **labels):
return REGISTRY.get_sample_value(metric_name, labels=labels)
def test_prometheus_metrics_counter_without_tags():
metrics = PrometheusMetricsFactory()
counter = metrics.create_counter(name='jaeger:test_counter_no_tags')
counter(1)
after = REGISTRY.get_sample_value('jaeger:test_counter_no_tags')
assert 1 == after
def test_prometheus_metrics_gauge_without_tags():
metrics = PrometheusMetricsFactory()
gauge = metrics.create_gauge(name='jaeger:test_gauge_no_tags')
gauge(1)
after = REGISTRY.get_sample_value('jaeger:test_gauge_no_tags')
assert 1 == after
def test_initial_metric_values(self):
self._assert_task_states(celery.states.ALL_STATES, 0)
assert REGISTRY.get_sample_value('celery_workers') == 0
assert REGISTRY.get_sample_value('celery_task_latency_count') == 0
assert REGISTRY.get_sample_value('celery_task_latency_sum') == 0
def test_collection_time(self):
current_total = REGISTRY.get_sample_value(
'python_gc_collection_process_time_total_s')
gc.collect()
self.assertTrue(current_total < REGISTRY.get_sample_value(
'python_gc_collection_process_time_total_s'))
