def setUp(self):
super(StatsStoreTest, self).setUp()
self.process_id = "some_pid"
self.stats_store = aff4.FACTORY.Create(None,
stats_store.StatsStore,
mode="w",
token=self.token)
fake_stats_collector = default_stats_collector.DefaultStatsCollector([
stats_utils.CreateCounterMetadata("counter"),
stats_utils.CreateCounterMetadata("counter_with_fields",
fields=[("source", str)]),
stats_utils.CreateEventMetadata("events"),
stats_utils.CreateEventMetadata("events_with_fields",
fields=[("source", str)]),
stats_utils.CreateGaugeMetadata("int_gauge", int),
stats_utils.CreateGaugeMetadata("str_gauge", str),
stats_utils.CreateGaugeMetadata("str_gauge_with_fields",
str,
fields=[("task", int)])
])
fake_stats_context = stats_test_utils.FakeStatsContext(
fake_stats_collector)
fake_stats_context.start()
self.addCleanup(fake_stats_context.stop)
def GetMetadata():
"""Returns a list of MetricMetadata for the client's metrics."""
return [
stats_utils.CreateCounterMetadata("grr_client_received_bytes"),
stats_utils.CreateCounterMetadata("grr_client_sent_bytes"),
stats_utils.CreateGaugeMetadata("grr_client_cpu_usage", str),
stats_utils.CreateGaugeMetadata("grr_client_io_usage", str)
]
def _CreateFakeStatsCollector():
"""Returns a stats-collector for use by tests in this file."""
return default_stats_collector.DefaultStatsCollector([
stats_utils.CreateCounterMetadata("counter"),
stats_utils.CreateCounterMetadata(
"counter_with_fields", fields=[("source", str)]),
stats_utils.CreateEventMetadata("events"),
stats_utils.CreateEventMetadata(
"events_with_fields", fields=[("source", str)]),
stats_utils.CreateGaugeMetadata("int_gauge", int),
stats_utils.CreateGaugeMetadata("str_gauge", str),
stats_utils.CreateGaugeMetadata(
"str_gauge_with_fields", str, fields=[("task", int)])
])
def testGetAllMetricsMetadataWorksCorrectlyOnSimpleMetrics(self):
counter_name = "testGAMM_SimpleMetrics_counter"
int_gauge_name = "testGAMM_SimpleMetrics_int_gauge"
event_metric_name = "testGAMM_SimpleMetrics_event_metric"
collector = self._CreateStatsCollector([
stats_utils.CreateCounterMetadata(counter_name),
stats_utils.CreateGaugeMetadata(int_gauge_name,
int,
fields=[("dimension", str)]),
stats_utils.CreateEventMetadata(event_metric_name)
])
metrics = collector.GetAllMetricsMetadata()
self.assertEqual(metrics[counter_name].metric_type,
rdf_stats.MetricMetadata.MetricType.COUNTER)
self.assertFalse(metrics[counter_name].fields_defs)
self.assertEqual(metrics[int_gauge_name].metric_type,
rdf_stats.MetricMetadata.MetricType.GAUGE)
self.assertEqual(metrics[int_gauge_name].fields_defs, [
rdf_stats.MetricFieldDefinition(
field_name="dimension",
field_type=rdf_stats.MetricFieldDefinition.FieldType.STR)
])
self.assertEqual(metrics[event_metric_name].metric_type,
rdf_stats.MetricMetadata.MetricType.EVENT)
self.assertFalse(metrics[event_metric_name].fields_defs)
def testGaugeWithFields(self):
int_gauge_name = "testGaugeWithFields_int_gauge"
collector = self._CreateStatsCollector([
stats_utils.CreateGaugeMetadata(int_gauge_name,
int,
fields=[("dimension", str)])
])
self.assertEqual(
0,
collector.GetMetricValue(int_gauge_name,
fields=["dimension_value_1"]))
self.assertEqual(
0,
collector.GetMetricValue(int_gauge_name,
fields=["dimesnioN_value_2"]))
collector.SetGaugeValue(int_gauge_name,
1,
fields=["dimension_value_1"])
collector.SetGaugeValue(int_gauge_name,
2,
fields=["dimension_value_2"])
self.assertEqual(
1,
collector.GetMetricValue(int_gauge_name,
fields=["dimension_value_1"]))
self.assertEqual(
2,
collector.GetMetricValue(int_gauge_name,
fields=["dimension_value_2"]))
def testGetMetricFieldsWorksCorrectly(self):
counter_name = "testGetMetricFieldsWorksCorrectly_counter"
int_gauge_name = "testGetMetricFieldsWorksCorrectly_int_gauge"
event_metric_name = "testGetMetricFieldsWorksCorrectly_event_metric"
collector = self._CreateStatsCollector([
stats_utils.CreateCounterMetadata(
counter_name, fields=[("dimension1", str), ("dimension2", str)]),
stats_utils.CreateGaugeMetadata(
int_gauge_name, int, fields=[("dimension", str)]),
stats_utils.CreateEventMetadata(
event_metric_name, fields=[("dimension", str)]),
])
collector.IncrementCounter(counter_name, fields=["b", "b"])
collector.IncrementCounter(counter_name, fields=["a", "c"])
collector.SetGaugeValue(int_gauge_name, 20, fields=["a"])
collector.SetGaugeValue(int_gauge_name, 30, fields=["b"])
collector.RecordEvent(event_metric_name, 0.1, fields=["a"])
collector.RecordEvent(event_metric_name, 0.1, fields=["b"])
fields = sorted(collector.GetMetricFields(counter_name), key=lambda t: t[0])
self.assertEqual([("a", "c"), ("b", "b")], fields)
fields = sorted(
collector.GetMetricFields(int_gauge_name), key=lambda t: t[0])
self.assertEqual([("a",), ("b",)], fields)
fields = sorted(
collector.GetMetricFields(event_metric_name), key=lambda t: t[0])
self.assertEqual([("a",), ("b",)], fields)
def testRaisesOnImproperFieldsUsage2(self):
counter_name = "testRaisesOnImproperFieldsUsage2_counter"
int_gauge_name = "testRaisesOnImproperFieldsUsage2_int_gauge"
event_metric_name = "testRaisesOnImproperFieldsUsage2_event_metric"
collector = self._CreateStatsCollector([
stats_utils.CreateCounterMetadata(
counter_name, fields=[("dimension", str)]),
stats_utils.CreateGaugeMetadata(
int_gauge_name, int, fields=[("dimension", str)]),
stats_utils.CreateEventMetadata(
event_metric_name, fields=[("dimension", str)])
])
# Check for counters
self.assertRaises(ValueError, collector.GetMetricValue, counter_name)
self.assertRaises(
ValueError, collector.GetMetricValue, counter_name, fields=["a", "b"])
# Check for gauges
self.assertRaises(ValueError, collector.GetMetricValue, int_gauge_name)
self.assertRaises(
ValueError, collector.GetMetricValue, int_gauge_name, fields=["a", "b"])
# Check for event metrics
self.assertRaises(ValueError, collector.GetMetricValue, event_metric_name)
self.assertRaises(
ValueError,
collector.GetMetricValue,
event_metric_name,
fields=["a", "b"])
def testRaisesOnImproperFieldsUsage1(self):
counter_name = "testRaisesOnImproperFieldsUsage1_counter"
int_gauge_name = "testRaisesOnImproperFieldsUsage1_int_gauge"
event_metric_name = "testRaisesOnImproperFieldsUsage1_event_metric"
collector = self._CreateStatsCollector([
stats_utils.CreateCounterMetadata(counter_name),
stats_utils.CreateGaugeMetadata(int_gauge_name, int),
stats_utils.CreateEventMetadata(event_metric_name)
])
# Check for counters
with self.assertRaises(ValueError):
collector.GetMetricValue(counter_name, fields=["a"])
# Check for gauges
with self.assertRaises(ValueError):
collector.GetMetricValue(int_gauge_name, fields=["a"])
# Check for event metrics
self.assertRaises(
ValueError,
collector.GetMetricValue,
event_metric_name,
fields=["a", "b"])
def Run(self):
# We have to include all server metadata in the test context since server
# code that uses the metrics runs within the context.
non_test_metadata = list(
itervalues(stats_collector_instance.Get().GetAllMetricsMetadata()))
test_metadata = non_test_metadata + [
stats_utils.CreateCounterMetadata(
_TEST_COUNTER, docstring="Sample counter metric."),
stats_utils.CreateGaugeMetadata(
_TEST_GAUGE_METRIC, str, docstring="Sample gauge metric."),
stats_utils.CreateEventMetadata(
_TEST_EVENT_METRIC, docstring="Sample event metric."),
]
stats_collector = default_stats_collector.DefaultStatsCollector(
test_metadata)
with stats_test_utils.FakeStatsContext(stats_collector):
with aff4.FACTORY.Create(
None, aff4_stats_store.StatsStore, mode="w",
token=self.token) as stats_store:
stats_store.WriteStats(process_id="worker_1")
# We use mixins to run the same tests against multiple APIs.
# Result-filtering is only needed for HTTP API tests.
if isinstance(self, api_regression_http.HttpApiRegressionTestMixinBase):
api_post_process_fn = self._PostProcessApiResult
else:
api_post_process_fn = None
self.Check(
"ListStatsStoreMetricsMetadata",
args=stats_plugin.ApiListStatsStoreMetricsMetadataArgs(
component="WORKER"),
api_post_process_fn=api_post_process_fn)
def testGaugeWithCallback(self):
int_gauge_name = "testGaugeWithCallback_int_gauge"
string_gauge_name = "testGaugeWithCallback_string_gauge"
collector = self._CreateStatsCollector([
stats_utils.CreateGaugeMetadata(int_gauge_name, int),
stats_utils.CreateGaugeMetadata(string_gauge_name, str)
])
self.assertEqual(0, collector.GetMetricValue(int_gauge_name))
self.assertEqual("", collector.GetMetricValue(string_gauge_name))
collector.SetGaugeCallback(int_gauge_name, lambda: 42)
collector.SetGaugeCallback(string_gauge_name, lambda: "some")
self.assertEqual(42, collector.GetMetricValue(int_gauge_name))
self.assertEqual("some", collector.GetMetricValue(string_gauge_name))
def testGaugeWithCallback(self):
int_gauge_name = "testGaugeWithCallback_int_gauge"
float_gauge_name = "testGaugeWithCallback_float_gauge"
collector = self._CreateStatsCollector([
stats_utils.CreateGaugeMetadata(int_gauge_name, int),
stats_utils.CreateGaugeMetadata(float_gauge_name, float)
])
self.assertEqual(0, collector.GetMetricValue(int_gauge_name))
self.assertEqual(0.0, collector.GetMetricValue(float_gauge_name))
collector.SetGaugeCallback(int_gauge_name, lambda: 42)
collector.SetGaugeCallback(float_gauge_name, lambda: 42.3)
self.assertEqual(42, collector.GetMetricValue(int_gauge_name))
self.assertAlmostEqual(42.3, collector.GetMetricValue(float_gauge_name))
def testSimpleGauge(self):
int_gauge_name = "testSimpleGauge_int_gauge"
float_gauge_name = "testSimpleGauge_float_gauge"
collector = self._CreateStatsCollector([
stats_utils.CreateGaugeMetadata(int_gauge_name, int),
stats_utils.CreateGaugeMetadata(float_gauge_name, float)
])
self.assertEqual(0, collector.GetMetricValue(int_gauge_name))
self.assertEqual(0.0, collector.GetMetricValue(float_gauge_name))
collector.SetGaugeValue(int_gauge_name, 42)
collector.SetGaugeValue(float_gauge_name, 42.3)
self.assertEqual(42, collector.GetMetricValue(int_gauge_name))
self.assertAlmostEqual(42.3, collector.GetMetricValue(float_gauge_name))
# At least default Python type checking is enforced in gauges:
# we can't assign string to int
with self.assertRaises(ValueError):
collector.SetGaugeValue(int_gauge_name, "some")
def Run(self):
real_metric_metadata = list(
itervalues(stats_collector_instance.Get().GetAllMetricsMetadata()))
test_metadata = real_metric_metadata + [
stats_utils.CreateCounterMetadata(
_TEST_COUNTER, docstring="Sample counter metric."),
stats_utils.CreateGaugeMetadata(
_TEST_GAUGE_METRIC, float, docstring="Sample gauge metric."),
stats_utils.CreateEventMetadata(
_TEST_EVENT_METRIC, docstring="Sample event metric."),
]
stats_collector = default_stats_collector.DefaultStatsCollector(
test_metadata)
with stats_test_utils.FakeStatsContext(stats_collector):
for i in range(10):
with test_lib.FakeTime(42 + i * 60):
stats_collector.IncrementCounter(_TEST_COUNTER)
stats_collector.SetGaugeValue(_TEST_GAUGE_METRIC, i * 0.5)
stats_collector.RecordEvent(_TEST_EVENT_METRIC, 0.42 + 0.5 * i)
with aff4.FACTORY.Create(
None, aff4_stats_store.StatsStore, mode="w",
token=self.token) as stats_store:
stats_store.WriteStats(process_id="worker_1")
range_start = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(42)
range_end = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(3600)
self.Check(
"GetStatsStoreMetric",
args=stats_plugin.ApiGetStatsStoreMetricArgs(
component="WORKER",
metric_name=_TEST_COUNTER,
start=range_start,
end=range_end))
self.Check(
"GetStatsStoreMetric",
args=stats_plugin.ApiGetStatsStoreMetricArgs(
component="WORKER",
metric_name=_TEST_COUNTER,
start=range_start,
end=range_end,
rate="1m"))
self.Check(
"GetStatsStoreMetric",
args=stats_plugin.ApiGetStatsStoreMetricArgs(
component="WORKER",
metric_name=_TEST_GAUGE_METRIC,
start=range_start,
end=range_end))
self.Check(
"GetStatsStoreMetric",
args=stats_plugin.ApiGetStatsStoreMetricArgs(
component="WORKER",
metric_name=_TEST_EVENT_METRIC,
start=range_start,
end=range_end))
self.Check(
"GetStatsStoreMetric",
args=stats_plugin.ApiGetStatsStoreMetricArgs(
component="WORKER",
metric_name=_TEST_EVENT_METRIC,
start=range_start,
end=range_end,
distribution_handling_mode="DH_COUNT"))
def GetMetadata():
"""Returns a list of MetricMetadata for GRR server components."""
return [
# GRR user-management metrics.
stats_utils.CreateEventMetadata("acl_check_time",
fields=[("check_type", str)]),
stats_utils.CreateCounterMetadata("approval_searches",
fields=[("reason_presence", str),
("source", str)]),
# Cronjob metrics.
stats_utils.CreateCounterMetadata("cron_internal_error"),
stats_utils.CreateCounterMetadata("cron_job_failure",
fields=[("cron_job_id", str)]),
stats_utils.CreateCounterMetadata("cron_job_timeout",
fields=[("cron_job_id", str)]),
stats_utils.CreateEventMetadata("cron_job_latency",
fields=[("cron_job_id", str)]),
# Access-control metrics.
stats_utils.CreateCounterMetadata("grr_expired_tokens"),
# Datastore metrics.
stats_utils.CreateCounterMetadata("grr_commit_failure"),
stats_utils.CreateCounterMetadata("datastore_retries"),
stats_utils.CreateGaugeMetadata(
"datastore_size",
int,
docstring="Size of data store in bytes",
units="BYTES"),
stats_utils.CreateCounterMetadata("grr_task_retransmission_count"),
stats_utils.CreateCounterMetadata("grr_task_ttl_expired_count"),
stats_utils.CreateEventMetadata(
"db_request_latency",
fields=[("call", str)],
bins=[0.05 * 1.2**x for x in range(30)]), # 50ms to ~10 secs
stats_utils.CreateCounterMetadata("db_request_errors",
fields=[("call", str),
("type", str)]),
stats_utils.CreateEventMetadata(
"blob_store_poll_hit_latency",
bins=[0.05, 0.1, 0.2, 0.5, 1, 2, 5, 10, 20, 50]),
stats_utils.CreateEventMetadata("blob_store_poll_hit_iteration",
bins=[1, 2, 5, 10, 20, 50]),
stats_utils.CreateEventMetadata(
"dual_blob_store_write_latency",
fields=[("backend", str), ("backend_class", str)],
bins=[0.05, 0.1, 0.2, 0.5, 1, 2, 5, 10, 20, 50]),
stats_utils.CreateCounterMetadata("dual_blob_store_success_count",
fields=[("backend", str),
("backend_class", str)]),
stats_utils.CreateCounterMetadata("dual_blob_store_error_count",
fields=[("backend", str),
("backend_class", str)]),
stats_utils.CreateCounterMetadata("dual_blob_store_discard_count",
fields=[("backend", str),
("backend_class", str)]),
# Threadpool metrics.
stats_utils.CreateGaugeMetadata("threadpool_outstanding_tasks",
int,
fields=[("pool_name", str)]),
stats_utils.CreateGaugeMetadata("threadpool_threads",
int,
fields=[("pool_name", str)]),
stats_utils.CreateGaugeMetadata("threadpool_cpu_use",
float,
fields=[("pool_name", str)]),
stats_utils.CreateCounterMetadata("threadpool_task_exceptions",
fields=[("pool_name", str)]),
stats_utils.CreateEventMetadata("threadpool_working_time",
fields=[("pool_name", str)]),
stats_utils.CreateEventMetadata("threadpool_queueing_time",
fields=[("pool_name", str)]),
# Worker and flow-related metrics.
stats_utils.CreateCounterMetadata("grr_flows_stuck"),
stats_utils.CreateCounterMetadata("worker_bad_flow_objects",
fields=[("type", str)]),
stats_utils.CreateCounterMetadata("worker_session_errors",
fields=[("type", str)]),
stats_utils.CreateCounterMetadata(
"worker_flow_lock_error",
docstring=
"Worker lock failures. We expect these to be high when the "
"systemis idle."),
stats_utils.CreateEventMetadata("worker_flow_processing_time",
fields=[("flow", str)]),
stats_utils.CreateEventMetadata(
"worker_time_to_retrieve_notifications"),
stats_utils.CreateCounterMetadata("grr_flow_completed_count"),
stats_utils.CreateCounterMetadata("grr_flow_errors"),
stats_utils.CreateCounterMetadata("grr_flow_invalid_flow_count"),
stats_utils.CreateCounterMetadata("grr_request_retransmission_count"),
stats_utils.CreateCounterMetadata("grr_response_out_of_order"),
stats_utils.CreateCounterMetadata("grr_unique_clients"),
stats_utils.CreateCounterMetadata("grr_worker_states_run"),
stats_utils.CreateCounterMetadata("grr_well_known_flow_requests"),
stats_utils.CreateCounterMetadata("flow_starts",
fields=[("flow", str)]),
stats_utils.CreateCounterMetadata("flow_errors",
fields=[("flow", str)]),
stats_utils.CreateCounterMetadata("flow_completions",
fields=[("flow", str)]),
stats_utils.CreateCounterMetadata("well_known_flow_requests",
fields=[("flow", str)]),
stats_utils.CreateCounterMetadata("well_known_flow_errors",
fields=[("flow", str)]),
stats_utils.CreateEventMetadata("fleetspeak_last_ping_latency_millis"),
# Hunt-related metrics.
stats_utils.CreateCounterMetadata("hunt_output_plugin_verifications",
fields=[("status", str)]),
stats_utils.CreateCounterMetadata(
"hunt_output_plugin_verification_errors"),
stats_utils.CreateCounterMetadata("hunt_output_plugin_errors",
fields=[("plugin", str)]),
stats_utils.CreateCounterMetadata("hunt_results_ran_through_plugin",
fields=[("plugin", str)]),
stats_utils.CreateCounterMetadata("hunt_results_compacted"),
stats_utils.CreateCounterMetadata(
"hunt_results_compaction_locking_errors"),
stats_utils.CreateCounterMetadata("hunt_results_added"),
# GRR-API metrics.
stats_utils.CreateEventMetadata("api_method_latency",
fields=[("method_name", str),
("protocol", str),
("status", str)]),
stats_utils.CreateEventMetadata("api_access_probe_latency",
fields=[("method_name", str),
("protocol", str),
("status", str)]),
# Client-related metrics.
stats_utils.CreateCounterMetadata("grr_client_crashes"),
stats_utils.CreateCounterMetadata("client_pings_by_label",
fields=[("label", str)]),
# Metrics specific to GRR frontends.
stats_utils.CreateGaugeMetadata("frontend_active_count",
int,
fields=[("source", str)]),
stats_utils.CreateGaugeMetadata("frontend_max_active_count", int),
stats_utils.CreateCounterMetadata("frontend_http_requests",
fields=[("action", str),
("protocol", str)]),
stats_utils.CreateCounterMetadata("frontend_in_bytes",
fields=[("source", str)]),
stats_utils.CreateCounterMetadata("frontend_out_bytes",
fields=[("source", str)]),
stats_utils.CreateCounterMetadata("frontend_request_count",
fields=[("source", str)]),
stats_utils.CreateCounterMetadata("frontend_inactive_request_count",
fields=[("source", str)]),
stats_utils.CreateEventMetadata("frontend_request_latency",
fields=[("source", str)]),
stats_utils.CreateEventMetadata("grr_frontendserver_handle_time"),
stats_utils.CreateCounterMetadata("grr_frontendserver_handle_num"),
stats_utils.CreateGaugeMetadata("grr_frontendserver_client_cache_size",
int),
stats_utils.CreateCounterMetadata("grr_messages_sent"),
stats_utils.CreateCounterMetadata("grr_pub_key_cache",
fields=[("type", str)]),
]