def testDeleteStatsFromLegacyDB(self): with test_lib.ConfigOverrider({"StatsStore.stats_ttl_hours": 1}): timestamp1 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(1) timestamp2 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(3600) timestamp3 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(4800) with test_lib.FakeTime(timestamp1): stats_collector_instance.Get().IncrementCounter(_SINGLE_DIM_COUNTER) stats_store._WriteStats(process_id="fake_process_id") expected_results = { "fake_process_id": { _SINGLE_DIM_COUNTER: [(1, timestamp1)] } } self.assertDictEqual( stats_store.ReadStats("f", _SINGLE_DIM_COUNTER), expected_results) with test_lib.FakeTime(timestamp2): stats_store._WriteStats(process_id="fake_process_id") expected_results = { "fake_process_id": { _SINGLE_DIM_COUNTER: [(1, timestamp1), (1, timestamp2)] } } self.assertDictEqual( stats_store.ReadStats("f", _SINGLE_DIM_COUNTER), expected_results) with test_lib.FakeTime(timestamp3): stats_store._DeleteStatsFromLegacyDB("fake_process_id") # timestamp1 is older than 1h, so it should get deleted. expected_results = { "fake_process_id": { _SINGLE_DIM_COUNTER: [(1, timestamp2)] } } self.assertDictEqual( stats_store.ReadStats("f", _SINGLE_DIM_COUNTER), expected_results)
def testPurgeServerStats(self): if not data_store.RelationalDBReadEnabled(): self.skipTest("Test is only for the relational DB. Skipping...") fake_stats_collector = default_stats_collector.DefaultStatsCollector([ stats_utils.CreateCounterMetadata("fake_counter"), ]) timestamp1 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(1) timestamp2 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(3600) timestamp3 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(4800) config_overrides = { "Database.useForReads.stats": True, "StatsStore.stats_ttl_hours": 1 } with test_lib.ConfigOverrider(config_overrides): with stats_test_utils.FakeStatsContext(fake_stats_collector): with test_lib.FakeTime(rdfvalue.RDFDatetime(timestamp1)): stats_collector_instance.Get().IncrementCounter( "fake_counter") stats_store._WriteStats(process_id="fake_process_id") expected_results = { "fake_process_id": { "fake_counter": [(1, timestamp1)] } } self.assertDictEqual( stats_store.ReadStats("f", "fake_counter"), expected_results) with test_lib.FakeTime(timestamp2): stats_store._WriteStats(process_id="fake_process_id") expected_results = { "fake_process_id": { "fake_counter": [(1, timestamp1), (1, timestamp2)] } } self.assertDictEqual( stats_store.ReadStats("f", "fake_counter"), expected_results) with test_lib.FakeTime(timestamp3): system.PurgeServerStatsCronJob( rdf_cronjobs.CronJobRun(), rdf_cronjobs.CronJob()).Run() # timestamp1 is older than 1h, so it should get deleted. expected_results = { "fake_process_id": { "fake_counter": [(1, timestamp2)] } } self.assertDictEqual( stats_store.ReadStats("f", "fake_counter"), expected_results)
def testReadStats(self): with test_lib.FakeTime(rdfvalue.RDFDatetime(1000)): stats_collector_instance.Get().IncrementCounter( _SINGLE_DIM_COUNTER) stats_collector_instance.Get().IncrementCounter( _COUNTER_WITH_ONE_FIELD, fields=["fieldval1"]) stats_collector_instance.Get().IncrementCounter( _COUNTER_WITH_TWO_FIELDS, fields=["fieldval2", 3]) stats_store._WriteStats(process_id="fake_process_id") with test_lib.FakeTime(rdfvalue.RDFDatetime(2000)): stats_collector_instance.Get().IncrementCounter( _COUNTER_WITH_TWO_FIELDS, fields=["fieldval2", 3]) stats_collector_instance.Get().IncrementCounter( _COUNTER_WITH_TWO_FIELDS, fields=["fieldval2", 4]) stats_store._WriteStats(process_id="fake_process_id") expected_single_dim_results = { "fake_process_id": { _SINGLE_DIM_COUNTER: [(1, 1000), (1, 2000)] } } expected_multi_dim1_results = { "fake_process_id": { _COUNTER_WITH_ONE_FIELD: { "fieldval1": [(1, 1000), (1, 2000)] } } } expected_multi_dim2_results = { "fake_process_id": { _COUNTER_WITH_TWO_FIELDS: { "fieldval2": { 3: [(1, 1000), (2, 2000)], 4: [(1, 2000)] } } } } self.assertDictEqual(stats_store.ReadStats("f", _SINGLE_DIM_COUNTER), expected_single_dim_results) self.assertDictEqual( stats_store.ReadStats("fake", _COUNTER_WITH_ONE_FIELD), expected_multi_dim1_results) self.assertEqual( stats_store.ReadStats("fake_process_id", _COUNTER_WITH_TWO_FIELDS), expected_multi_dim2_results)
def SetupSampleMetrics(token=None): now = rdfvalue.RDFDatetime.Now() handle_data = [(3, now - rdfvalue.Duration("50m")), (0, now - rdfvalue.Duration("45m")), (1, now - rdfvalue.Duration("40m")), (0, now - rdfvalue.Duration("35m")), (0, now - rdfvalue.Duration("30m")), (1, now - rdfvalue.Duration("25m")), (0, now - rdfvalue.Duration("20m")), (0, now - rdfvalue.Duration("15m")), (0, now - rdfvalue.Duration("10m")), (5, now - rdfvalue.Duration("5m")), (0, now)] # pyformat: disable handle_data = [(value, timestamp.AsMicrosecondsSinceEpoch()) for value, timestamp in handle_data] for value, timestamp in handle_data: with test_lib.FakeTime(timestamp / 1e6): stats_collector_instance.Get().IncrementCounter( "grr_frontendserver_handle_num", value) stats_store._WriteStats(process_id="frontend")
def testPurgeServerStats(self): if not data_store.RelationalDBReadEnabled(): self.skipTest("Test is only for the relational DB. Skipping...") fake_stats_collector = default_stats_collector.DefaultStatsCollector([ stats_utils.CreateCounterMetadata("fake_counter"), ]) timestamp0 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(1) timestamp1 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(2) timestamp2 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(3600) timestamp3 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(4800) config_overrides = { "Database.useForReads.stats": True, "StatsStore.stats_ttl_hours": 1 } with test_lib.ConfigOverrider(config_overrides), \ stats_test_utils.FakeStatsContext(fake_stats_collector), \ mock.patch.object(system, "_STATS_DELETION_BATCH_SIZE", 1): with test_lib.FakeTime(rdfvalue.RDFDatetime(timestamp0)): stats_store._WriteStats(process_id="fake_process_id") with test_lib.FakeTime(rdfvalue.RDFDatetime(timestamp1)): stats_collector_instance.Get().IncrementCounter("fake_counter") stats_store._WriteStats(process_id="fake_process_id") expected_results = { "fake_process_id": { "fake_counter": [(0, timestamp0), (1, timestamp1)] } } self.assertDictEqual( stats_store.ReadStats("f", "fake_counter"), expected_results) with test_lib.FakeTime(timestamp2): stats_store._WriteStats(process_id="fake_process_id") expected_results = { "fake_process_id": { "fake_counter": [(0, timestamp0), (1, timestamp1), (1, timestamp2)] } } self.assertDictEqual( stats_store.ReadStats("f", "fake_counter"), expected_results) with test_lib.FakeTime(timestamp3): cron = system.PurgeServerStatsCronJob( rdf_cronjobs.CronJobRun(), rdf_cronjobs.CronJob()) cron.Run() # timestamp0 and timestamp1 are older than 1h, so they should get # deleted. expected_results = { "fake_process_id": { "fake_counter": [(1, timestamp2)] } } self.assertDictEqual( stats_store.ReadStats("f", "fake_counter"), expected_results) self.assertIn("Deleted 2 stats entries.", cron.run_state.log_message)
def Run(self): with test_lib.ConfigOverrider({"Database.useForReads.stats": True}): real_metric_metadata = list( itervalues( stats_collector_instance.Get().GetAllMetricsMetadata())) test_metadata = real_metric_metadata + [ stats_utils.CreateCounterMetadata( _TEST_COUNTER, docstring="Sample counter metric."), stats_utils.CreateGaugeMetadata( _TEST_GAUGE_METRIC, float, docstring="Sample gauge metric."), stats_utils.CreateEventMetadata( _TEST_EVENT_METRIC, docstring="Sample event metric."), ] stats_collector = default_stats_collector.DefaultStatsCollector( test_metadata) with stats_test_utils.FakeStatsContext(stats_collector): for i in range(10): with test_lib.FakeTime(42 + i * 60): stats_collector.IncrementCounter(_TEST_COUNTER) stats_collector.SetGaugeValue(_TEST_GAUGE_METRIC, i * 0.5) stats_collector.RecordEvent(_TEST_EVENT_METRIC, 0.42 + 0.5 * i) stats_store._WriteStats(process_id="worker_1") range_start = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(42) range_end = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(3600) self.Check("GetStatsStoreMetric", args=stats_plugin.ApiGetStatsStoreMetricArgs( component="WORKER", metric_name=_TEST_COUNTER, start=range_start, end=range_end)) self.Check("GetStatsStoreMetric", args=stats_plugin.ApiGetStatsStoreMetricArgs( component="WORKER", metric_name=_TEST_COUNTER, start=range_start, end=range_end, rate="1m")) self.Check("GetStatsStoreMetric", args=stats_plugin.ApiGetStatsStoreMetricArgs( component="WORKER", metric_name=_TEST_GAUGE_METRIC, start=range_start, end=range_end)) self.Check("GetStatsStoreMetric", args=stats_plugin.ApiGetStatsStoreMetricArgs( component="WORKER", metric_name=_TEST_EVENT_METRIC, start=range_start, end=range_end)) self.Check("GetStatsStoreMetric", args=stats_plugin.ApiGetStatsStoreMetricArgs( component="WORKER", metric_name=_TEST_EVENT_METRIC, start=range_start, end=range_end, distribution_handling_mode="DH_COUNT"))
def testPurgeServerStats(self): if not data_store.RelationalDBReadEnabled(): self.skipTest("Test is only for the relational DB. Skipping...") fake_stats_collector = prometheus_stats_collector.PrometheusStatsCollector( [ stats_utils.CreateCounterMetadata("fake_counter"), ]) timestamp0 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(1) timestamp1 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(2) timestamp2 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(3600) timestamp3 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(4800) config_overrides = { "Database.useForReads.stats": True, "StatsStore.stats_ttl_hours": 1 } zero_duration = rdfvalue.Duration(0) # Backslash continuation is explicitly allowed by Google's style guide for # nested context manager expressions spanning 3 or more lines. # pylint: disable=g-backslash-continuation with test_lib.ConfigOverrider(config_overrides), \ stats_test_utils.FakeStatsContext(fake_stats_collector), \ mock.patch.object(system, "_STATS_DELETION_BATCH_SIZE", 1), \ mock.patch.object(system, "_stats_checkpoint_period", zero_duration): with test_lib.FakeTime(rdfvalue.RDFDatetime(timestamp0)): stats_store._WriteStats(process_id="fake_process_id") with test_lib.FakeTime(rdfvalue.RDFDatetime(timestamp1)): stats_collector_instance.Get().IncrementCounter("fake_counter") stats_store._WriteStats(process_id="fake_process_id") expected_results = { "fake_process_id": { "fake_counter": [(0, timestamp0), (1, timestamp1)] } } self.assertDictEqual( stats_store.ReadStats("f", "fake_counter"), expected_results) with test_lib.FakeTime(timestamp2): stats_store._WriteStats(process_id="fake_process_id") expected_results = { "fake_process_id": { "fake_counter": [(0, timestamp0), (1, timestamp1), (1, timestamp2)] } } self.assertDictEqual( stats_store.ReadStats("f", "fake_counter"), expected_results) with test_lib.FakeTime(timestamp3): cron_name = compatibility.GetName( system.PurgeServerStatsCronJob) cronjobs.ScheduleSystemCronJobs(names=[cron_name]) job_data = data_store.REL_DB.ReadCronJobs([cron_name])[0] cron_run = rdf_cronjobs.CronJobRun(cron_job_id=cron_name) cron_run.GenerateRunId() cron_run.started_at = rdfvalue.RDFDatetime.Now() cron = system.PurgeServerStatsCronJob(cron_run, job_data) cron.Run() # timestamp0 and timestamp1 are older than 1h, so they should get # deleted. expected_results = { "fake_process_id": { "fake_counter": [(1, timestamp2)] } } self.assertDictEqual( stats_store.ReadStats("f", "fake_counter"), expected_results) self.assertEqual( "Deleted 2 stats entries.\nDeleted 1 stats entries.", cron.run_state.log_message)