def testRaisesOnImproperFieldsUsage2(self): with self.SetUpStatsCollector(self._CreateStatsCollector): counter = metrics.Counter( "testRaisesOnImproperFieldsUsage2_counter", fields=[("dimension", str)]) int_gauge = metrics.Gauge( "testRaisesOnImproperFieldsUsage2_int_gauge", int, fields=[("dimension", str)]) event_metric = metrics.Event( "testRaisesOnImproperFieldsUsage2_event_metric", fields=[("dimension", str)]) # Check for counters with self.assertRaises(ValueError): counter.GetValue() with self.assertRaises(ValueError): counter.GetValue(fields=["a", "b"]) # Check for gauges with self.assertRaises(ValueError): int_gauge.GetValue() with self.assertRaises(ValueError): int_gauge.GetValue(fields=["a", "b"]) # Check for event metrics with self.assertRaises(ValueError): event_metric.GetValue() with self.assertRaises(ValueError): event_metric.GetValue(fields=["a", "b"])
def testSimpleEventMetric(self): with self.SetUpStatsCollector(self._CreateStatsCollector): event_metric = metrics.Event("testSimpleEventMetric_event_metric", bins=[0.0, 0.1, 0.2]) data = event_metric.GetValue() self.assertAlmostEqual(0, data.sum) self.assertEqual(0, data.count) self.assertEqual([-_INF, 0.0, 0.1, 0.2], list(data.bins)) self.assertEqual({-_INF: 0, 0.0: 0, 0.1: 0, 0.2: 0}, data.bins_heights) event_metric.RecordEvent(0.15) data = event_metric.GetValue() self.assertAlmostEqual(0.15, data.sum) self.assertEqual(1, data.count) self.assertEqual([-_INF, 0.0, 0.1, 0.2], list(data.bins)) self.assertEqual({-_INF: 0, 0.0: 0, 0.1: 1, 0.2: 0}, data.bins_heights) event_metric.RecordEvent(0.5) data = event_metric.GetValue() self.assertAlmostEqual(0.65, data.sum) self.assertEqual(2, data.count) self.assertEqual([-_INF, 0.0, 0.1, 0.2], list(data.bins)) self.assertEqual({-_INF: 0, 0.0: 0, 0.1: 1, 0.2: 1}, data.bins_heights) event_metric.RecordEvent(-0.1) data = event_metric.GetValue() self.assertAlmostEqual(0.55, data.sum) self.assertEqual(3, data.count) self.assertEqual([-_INF, 0.0, 0.1, 0.2], list(data.bins)) self.assertEqual({-_INF: 1, 0.0: 0, 0.1: 1, 0.2: 1}, data.bins_heights)
def testEventMetricWithFields(self): with self.SetUpStatsCollector(self._CreateStatsCollector): event_metric = metrics.Event( "testEventMetricWithFields_event_metric", bins=[0.0, 0.1, 0.2], fields=[("dimension", str)]) data = event_metric.GetValue(fields=["dimension_value_1"]) self.assertAlmostEqual(0, data.sum) self.assertEqual(0, data.count) self.assertEqual([-_INF, 0.0, 0.1, 0.2], list(data.bins)) self.assertEqual({-_INF: 0, 0.0: 0, 0.1: 0, 0.2: 0}, data.bins_heights) event_metric.RecordEvent(0.15, fields=["dimension_value_1"]) event_metric.RecordEvent(0.25, fields=["dimension_value_2"]) data = event_metric.GetValue(fields=["dimension_value_1"]) self.assertAlmostEqual(0.15, data.sum) self.assertEqual(1, data.count) self.assertEqual([-_INF, 0.0, 0.1, 0.2], list(data.bins)) self.assertEqual({-_INF: 0, 0.0: 0, 0.1: 1, 0.2: 0}, data.bins_heights) data = event_metric.GetValue(fields=["dimension_value_2"]) self.assertAlmostEqual(0.25, data.sum) self.assertEqual(1, data.count) self.assertEqual([-_INF, 0.0, 0.1, 0.2], list(data.bins)) self.assertEqual({-_INF: 0, 0.0: 0, 0.1: 0, 0.2: 1}, data.bins_heights)
def testMultipleFuncs(self): """Tests if multiple decorators produce aggregate stats.""" with self.SetUpStatsCollector(self._CreateStatsCollector): counter = metrics.Counter("testMultipleFuncs_counter") event_metric = metrics.Event("testMultipleFuncs_event_metric", bins=[0, 1, 2]) @counter.Counted() def Func1(n): self._Sleep(n) @counter.Counted() def Func2(n): self._Sleep(n) @event_metric.Timed() def Func3(n): self._Sleep(n) @event_metric.Timed() def Func4(n): self._Sleep(n) Func1(0.1) Func2(0.1) self.assertEqual(counter.GetValue(), 2) Func3(0.1) Func4(1.1) m = event_metric.GetValue() self.assertEqual(m.bins_heights, {-_INF: 0, 0: 1, 1: 1, 2: 0})
def testGetAllMetricsMetadataWorksCorrectlyOnSimpleMetrics(self): counter_name = "testGAMM_SimpleMetrics_counter" int_gauge_name = "testGAMM_SimpleMetrics_int_gauge" event_metric_name = "testGAMM_SimpleMetrics_event_metric" with self.SetUpStatsCollector(self._CreateStatsCollector): metrics.Counter(counter_name) metrics.Gauge(int_gauge_name, int, fields=[("dimension", str)]) metrics.Event(event_metric_name) metadatas = self.collector.GetAllMetricsMetadata() self.assertEqual(metadatas[counter_name].metric_type, rdf_stats.MetricMetadata.MetricType.COUNTER) self.assertFalse(metadatas[counter_name].fields_defs) self.assertEqual(metadatas[int_gauge_name].metric_type, rdf_stats.MetricMetadata.MetricType.GAUGE) self.assertEqual(metadatas[int_gauge_name].fields_defs, [ rdf_stats.MetricFieldDefinition( field_name="dimension", field_type=rdf_stats.MetricFieldDefinition.FieldType.STR) ]) self.assertEqual(metadatas[event_metric_name].metric_type, rdf_stats.MetricMetadata.MetricType.EVENT) self.assertFalse(metadatas[event_metric_name].fields_defs)
def testTimedDecorator(self): with self.SetUpStatsCollector( default_stats_collector.DefaultStatsCollector()): event = metrics.Event("efoo", fields=[("bar", str)]) @event.Timed(fields=["baz"]) def Foo(): pass with self.assertStatsCounterDelta(1, event, fields=["baz"]): Foo()
def testCombiningDecorators(self): with self.SetUpStatsCollector(self._CreateStatsCollector): counter = metrics.Counter("testCombiningDecorators_counter") event_metric = metrics.Event( "testCombiningDecorators_event_metric", bins=[0.0, 0.1, 0.2]) @event_metric.Timed() @counter.Counted() def OverdecoratedFunc(n): self._Sleep(n) OverdecoratedFunc(0.02) # Check if all vars get updated m = event_metric.GetValue() self.assertEqual(m.bins_heights, {-_INF: 0, 0: 1, 0.1: 0, 0.2: 0}) self.assertEqual(counter.GetValue(), 1)
def testRaisesOnImproperFieldsUsage1(self): with self.SetUpStatsCollector(self._CreateStatsCollector()): counter = metrics.Counter("testRaisesOnImproperFieldsUsage1_counter") int_gauge = metrics.Gauge("testRaisesOnImproperFieldsUsage1_int_gauge", int) event_metric = metrics.Event( "testRaisesOnImproperFieldsUsage1_event_metric") # Check for counters with self.assertRaises(ValueError): counter.GetValue(fields=["a"]) # Check for gauges with self.assertRaises(ValueError): int_gauge.GetValue(fields=["a"]) # Check for event metrics with self.assertRaises(ValueError): event_metric.GetValue(fields=["a", "b"])
def testExceptionHandling(self): with self.SetUpStatsCollector(self._CreateStatsCollector): counter = metrics.Counter("testExceptionHandling_counter") event_metric = metrics.Event("testExceptionHandling_event_metric", bins=[0, 0.1, 0.2]) @event_metric.Timed() @counter.Counted() def RaiseFunc(n): self._Sleep(n) raise Exception() with self.assertRaises(Exception): RaiseFunc(0.11) # Check if all vars get updated m = event_metric.GetValue() self.assertEqual(m.bins_heights, {-_INF: 0, 0: 0, 0.1: 1, 0.2: 0}) self.assertEqual(counter.GetValue(), 1)
def testBinnedTimings(self): event_metric_name = "testMaps_event_metric" with self.SetUpStatsCollector(self._CreateStatsCollector): event_metric = metrics.Event(event_metric_name, bins=[0, 0.1, 0.2]) @event_metric.Timed() def TimedFunc(n): self._Sleep(n) m = event_metric.GetValue() self.assertEqual(m.bins_heights, {-_INF: 0, 0: 0, 0.1: 0, 0.2: 0}) for _ in range(3): TimedFunc(0.01) m = event_metric.GetValue() self.assertEqual(m.bins_heights, {-_INF: 0, 0: 3, 0.1: 0, 0.2: 0}) TimedFunc(0.11) m = event_metric.GetValue() self.assertEqual(m.bins_heights, {-_INF: 0, 0: 3, 0.1: 1, 0.2: 0})
def testGetMetricFieldsWorksCorrectly(self): with self.SetUpStatsCollector(self._CreateStatsCollector): counter = metrics.Counter( "testGetMetricFieldsWorksCorrectly_counter", fields=[("dimension1", str), ("dimension2", str)]) int_gauge = metrics.Gauge( "testGetMetricFieldsWorksCorrectly_int_gauge", int, fields=[("dimension", str)]) event_metric = metrics.Event( "testGetMetricFieldsWorksCorrectly_event_metric", fields=[("dimension", str)]) counter.Increment(fields=["b", "b"]) counter.Increment(fields=["a", "c"]) self.assertCountEqual([("a", "c"), ("b", "b")], counter.GetFields()) int_gauge.SetValue(20, fields=["a"]) int_gauge.SetValue(30, fields=["b"]) self.assertCountEqual([("a", ), ("b", )], int_gauge.GetFields()) event_metric.RecordEvent(0.1, fields=["a"]) event_metric.RecordEvent(0.1, fields=["b"]) self.assertCountEqual([("a", ), ("b", )], event_metric.GetFields())
def testEventRegistration(self): with self.SetUpStatsCollector( default_stats_collector.DefaultStatsCollector()): metrics.Event("efoo") self.assertIsNotNone(self.collector.GetMetricMetadata("efoo"))
from google.protobuf import timestamp_pb2 from grr_response_core import config from grr_response_core.lib import rdfvalue from grr_response_core.lib.rdfvalues import flows as rdf_flows from grr_response_core.lib.util import text from grr_response_core.stats import metrics from grr_response_server import data_store from grr_response_server import fleetspeak_connector from fleetspeak.src.common.proto.fleetspeak import common_pb2 as fs_common_pb2 from fleetspeak.src.common.proto.fleetspeak import system_pb2 as fs_system_pb2 from fleetspeak.src.server.proto.fleetspeak_server import admin_pb2 from fleetspeak.src.server.proto.fleetspeak_server import resource_pb2 FLEETSPEAK_CALL_LATENCY = metrics.Event("fleetspeak_call_latency", fields=[("call", str)]) WRITE_SINGLE_TRY_TIMEOUT = datetime.timedelta(seconds=30) WRITE_TOTAL_TIMEOUT = datetime.timedelta(seconds=300) READ_SINGLE_TRY_TIMEOUT = datetime.timedelta(seconds=60) READ_TOTAL_TIMEOUT = datetime.timedelta(seconds=120) def IsFleetspeakEnabledClient(grr_id): """Returns whether the provided GRR id is a Fleetspeak client.""" if grr_id is None: return False md = data_store.REL_DB.ReadClientMetadata(grr_id) if not md:
def testRecordEvent(self): with self.SetUpStatsCollector( default_stats_collector.DefaultStatsCollector()): event = metrics.Event("efoo", fields=[("bar", str)]) with self.assertStatsCounterDelta(1, event, fields=["baz"]): event.RecordEvent(42, fields=["baz"])
import time from typing import Dict, Iterable, List, Optional from grr_response_core.lib import rdfvalue from grr_response_core.lib.util import precondition from grr_response_core.stats import metrics from grr_response_server.rdfvalues import objects as rdf_objects # Global blob stores registry. # # NOTE: this is a rudimentary registry that will be migrated to the uniform # registry approach by hanuszczak@ (currently in the works). REGISTRY = {} BLOB_STORE_POLL_HIT_LATENCY = metrics.Event( "blob_store_poll_hit_latency", bins=[0.05, 0.1, 0.2, 0.5, 1, 2, 5, 10, 20, 50]) BLOB_STORE_POLL_HIT_ITERATION = metrics.Event("blob_store_poll_hit_iteration", bins=[1, 2, 5, 10, 20, 50]) class BlobStoreTimeoutError(Exception): """An exception class raised when certain blob store operation times out.""" class BlobStore(metaclass=abc.ABCMeta): """The blob store base class.""" def WriteBlobsWithUnknownHashes(self, blobs_data): """Writes the contents of the given blobs, using their hash as BlobID. Args:
from __future__ import division from __future__ import unicode_literals import functools import logging import time from typing import Text from grr_response_core.lib import utils from grr_response_core.lib.util import precondition from grr_response_core.stats import metrics from grr_response_server.databases import db DB_REQUEST_LATENCY = metrics.Event("db_request_latency", fields=[("call", str)], bins=[0.05 * 1.2**x for x in range(30) ]) # 50ms to ~10 secs DB_REQUEST_ERRORS = metrics.Counter("db_request_errors", fields=[("call", str), ("type", str)]) class Error(Exception): pass class FlowIDIsNotAnIntegerError(Error): pass class OutputPluginIDIsNotAnIntegerError(Error): pass
from grr_response_core import config from grr_response_core.lib.util import compatibility from grr_response_core.lib.util import precondition from grr_response_core.stats import metrics from grr_response_server import blob_store from grr_response_server.rdfvalues import objects as rdf_objects # Maximum queue length, where each queue entry can consist of multiple blobs. # Thus the number of enqueued blobs can be considerably bigger. This only # serves as a basic measure to prevent unbounded memory growth. _SECONDARY_WRITE_QUEUE_MAX_LENGTH = 30 DUAL_BLOB_STORE_LATENCY = metrics.Event( "dual_blob_store_latency", fields=[("backend_class", str), ("method", str)], bins=[0.05, 0.1, 0.2, 0.5, 1, 2, 5, 10, 20, 50]) DUAL_BLOB_STORE_OP_SIZE = metrics.Event( "dual_blob_store_op_size", fields=[("backend_class", str), ("method", str)], bins=[0, 1, 2, 5, 10, 20, 50, 100, 200, 500]) DUAL_BLOB_STORE_SUCCESS_COUNT = metrics.Counter( "dual_blob_store_success_count", fields=[("backend_class", str), ("method", str)]) DUAL_BLOB_STORE_ERROR_COUNT = metrics.Counter( "dual_blob_store_error_count", fields=[("backend_class", str), ("method", str)]) DUAL_BLOB_STORE_DISCARD_COUNT = metrics.Counter( "dual_blob_store_discard_count", fields=[("backend_class", str), ("method", str)])
from grr_response_core.lib.util import compatibility from grr_response_core.lib.util import precondition from grr_response_core.lib.util.compat import json from grr_response_core.stats import metrics from grr_response_server import access_control from grr_response_server import data_store from grr_response_server.gui import api_auth_manager from grr_response_server.gui import api_call_context from grr_response_server.gui import api_call_handler_base from grr_response_server.gui import api_call_router from grr_response_server.gui import api_value_renderers from grr_response_server.gui import http_response API_METHOD_LATENCY = metrics.Event( "api_method_latency", fields=[("method_name", str), ("protocol", str), ("status", str)]) API_ACCESS_PROBE_LATENCY = metrics.Event( "api_access_probe_latency", fields=[("method_name", str), ("protocol", str), ("status", str)]) class Error(Exception): pass class PostRequestParsingError(Error): pass class UnsupportedHttpMethod(Error):
from grr_response_core.stats import metrics from grr_response_server import data_store from grr_response_server import hunt from grr_response_server import threadpool from grr_response_server.rdfvalues import cronjobs as rdf_cronjobs # The maximum number of log-messages to store in the DB for a given cron-job # run. _MAX_LOG_MESSAGES = 20 CRON_JOB_FAILURE = metrics.Counter( "cron_job_failure", fields=[("cron_job_id", str)]) CRON_JOB_TIMEOUT = metrics.Counter( "cron_job_timeout", fields=[("cron_job_id", str)]) CRON_JOB_LATENCY = metrics.Event( "cron_job_latency", fields=[("cron_job_id", str)]) CRON_JOB_USERNAME = "******" class Error(Exception): pass class OneOrMoreCronJobsFailedError(Error): def __init__(self, failure_map): message = "One or more cron jobs failed unexpectedly: " + ", ".join( "%s=%s" % (k, v) for k, v in failure_map.items()) super().__init__(message) self.failure_map = failure_map
from grr_response_server import access_control from grr_response_server import data_store from grr_response_server.gui import api_auth_manager from grr_response_server.gui import api_call_context from grr_response_server.gui import api_call_handler_base from grr_response_server.gui import api_call_router from grr_response_server.gui import api_value_renderers from grr_response_server.gui import http_response _FIELDS = ( ("method_name", str), ("protocol", str), ("status", str), ("origin", str), ) API_METHOD_LATENCY = metrics.Event("api_method_latency", fields=_FIELDS) API_ACCESS_PROBE_LATENCY = metrics.Event("api_access_probe_latency", fields=_FIELDS) class Error(Exception): pass class PostRequestParsingError(Error): pass class UnsupportedHttpMethod(Error): pass
FRONTEND_ACTIVE_COUNT = metrics.Gauge("frontend_active_count", int, fields=[("source", str)]) FRONTEND_MAX_ACTIVE_COUNT = metrics.Gauge("frontend_max_active_count", int) FRONTEND_HTTP_REQUESTS = metrics.Counter("frontend_http_requests", fields=[("action", str), ("protocol", str)]) FRONTEND_IN_BYTES = metrics.Counter("frontend_in_bytes", fields=[("source", str)]) FRONTEND_OUT_BYTES = metrics.Counter("frontend_out_bytes", fields=[("source", str)]) FRONTEND_REQUEST_COUNT = metrics.Counter("frontend_request_count", fields=[("source", str)]) FRONTEND_INACTIVE_REQUEST_COUNT = metrics.Counter( "frontend_inactive_request_count", fields=[("source", str)]) FRONTEND_REQUEST_LATENCY = metrics.Event("frontend_request_latency", fields=[("source", str)]) GRR_FRONTENDSERVER_HANDLE_TIME = metrics.Event( "grr_frontendserver_handle_time") GRR_FRONTENDSERVER_HANDLE_NUM = metrics.Counter( "grr_frontendserver_handle_num") GRR_MESSAGES_SENT = metrics.Counter("grr_messages_sent") GRR_UNIQUE_CLIENTS = metrics.Counter("grr_unique_clients") class ServerCommunicator(communicator.Communicator): """A communicator which stores certificates using the relational db.""" def __init__(self, certificate, private_key): super().__init__(certificate=certificate, private_key=private_key) self.common_name = self.certificate.GetCN() def _GetRemotePublicKey(self, common_name):
from grr_response_core.stats import metrics STOP_MESSAGE = "Stop message" THREADPOOL_OUTSTANDING_TASKS = metrics.Gauge("threadpool_outstanding_tasks", int, fields=[("pool_name", str)]) THREADPOOL_THREADS = metrics.Gauge("threadpool_threads", int, fields=[("pool_name", str)]) THREADPOOL_CPU_USE = metrics.Gauge("threadpool_cpu_use", float, fields=[("pool_name", str)]) THREADPOOL_TASK_EXCEPTIONS = metrics.Counter("threadpool_task_exceptions", fields=[("pool_name", str)]) THREADPOOL_WORKING_TIME = metrics.Event("threadpool_working_time", fields=[("pool_name", str)]) THREADPOOL_QUEUEING_TIME = metrics.Event("threadpool_queueing_time", fields=[("pool_name", str)]) class Error(Exception): pass class ThreadPoolNotStartedError(Error): """Raised when a task is added to a not-yet-started pool.""" class DuplicateThreadpoolError(Error): """Raised when a thread pool with the same name already exists."""