def testDeleteStatsFromLegacyDB(self):
with test_lib.ConfigOverrider({"StatsStore.stats_ttl_hours": 1}):
timestamp1 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(1)
timestamp2 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(3600)
timestamp3 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(4800)
with test_lib.FakeTime(timestamp1):
stats_collector_instance.Get().IncrementCounter(_SINGLE_DIM_COUNTER)
stats_store._WriteStats(process_id="fake_process_id")
expected_results = {
"fake_process_id": {
_SINGLE_DIM_COUNTER: [(1, timestamp1)]
}
}
self.assertDictEqual(
stats_store.ReadStats("f", _SINGLE_DIM_COUNTER), expected_results)
with test_lib.FakeTime(timestamp2):
stats_store._WriteStats(process_id="fake_process_id")
expected_results = {
"fake_process_id": {
_SINGLE_DIM_COUNTER: [(1, timestamp1), (1, timestamp2)]
}
}
self.assertDictEqual(
stats_store.ReadStats("f", _SINGLE_DIM_COUNTER), expected_results)
with test_lib.FakeTime(timestamp3):
stats_store._DeleteStatsFromLegacyDB("fake_process_id")
# timestamp1 is older than 1h, so it should get deleted.
expected_results = {
"fake_process_id": {
_SINGLE_DIM_COUNTER: [(1, timestamp2)]
}
}
self.assertDictEqual(
stats_store.ReadStats("f", _SINGLE_DIM_COUNTER), expected_results)
def testPurgeServerStats(self):
if not data_store.RelationalDBReadEnabled():
self.skipTest("Test is only for the relational DB. Skipping...")
fake_stats_collector = default_stats_collector.DefaultStatsCollector([
stats_utils.CreateCounterMetadata("fake_counter"),
])
timestamp0 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(1)
timestamp1 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(2)
timestamp2 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(3600)
timestamp3 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(4800)
config_overrides = {
"Database.useForReads.stats": True,
"StatsStore.stats_ttl_hours": 1
}
with test_lib.ConfigOverrider(config_overrides), \
stats_test_utils.FakeStatsContext(fake_stats_collector), \
mock.patch.object(system, "_STATS_DELETION_BATCH_SIZE", 1):
with test_lib.FakeTime(rdfvalue.RDFDatetime(timestamp0)):
stats_store._WriteStats(process_id="fake_process_id")
with test_lib.FakeTime(rdfvalue.RDFDatetime(timestamp1)):
stats_collector_instance.Get().IncrementCounter("fake_counter")
stats_store._WriteStats(process_id="fake_process_id")
expected_results = {
"fake_process_id": {
"fake_counter": [(0, timestamp0), (1, timestamp1)]
}
}
self.assertDictEqual(
stats_store.ReadStats("f", "fake_counter"),
expected_results)
with test_lib.FakeTime(timestamp2):
stats_store._WriteStats(process_id="fake_process_id")
expected_results = {
"fake_process_id": {
"fake_counter": [(0, timestamp0), (1, timestamp1),
(1, timestamp2)]
}
}
self.assertDictEqual(
stats_store.ReadStats("f", "fake_counter"),
expected_results)
with test_lib.FakeTime(timestamp3):
cron = system.PurgeServerStatsCronJob(
rdf_cronjobs.CronJobRun(), rdf_cronjobs.CronJob())
cron.Run()
# timestamp0 and timestamp1 are older than 1h, so they should get
# deleted.
expected_results = {
"fake_process_id": {
"fake_counter": [(1, timestamp2)]
}
}
self.assertDictEqual(
stats_store.ReadStats("f", "fake_counter"),
expected_results)
self.assertIn("Deleted 2 stats entries.",
cron.run_state.log_message)
def testPurgeServerStats(self):
if not data_store.RelationalDBReadEnabled():
self.skipTest("Test is only for the relational DB. Skipping...")
fake_stats_collector = default_stats_collector.DefaultStatsCollector([
stats_utils.CreateCounterMetadata("fake_counter"),
])
timestamp1 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(1)
timestamp2 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(3600)
timestamp3 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(4800)
config_overrides = {
"Database.useForReads.stats": True,
"StatsStore.stats_ttl_hours": 1
}
with test_lib.ConfigOverrider(config_overrides):
with stats_test_utils.FakeStatsContext(fake_stats_collector):
with test_lib.FakeTime(rdfvalue.RDFDatetime(timestamp1)):
stats_collector_instance.Get().IncrementCounter(
"fake_counter")
stats_store._WriteStats(process_id="fake_process_id")
expected_results = {
"fake_process_id": {
"fake_counter": [(1, timestamp1)]
}
}
self.assertDictEqual(
stats_store.ReadStats("f", "fake_counter"),
expected_results)
with test_lib.FakeTime(timestamp2):
stats_store._WriteStats(process_id="fake_process_id")
expected_results = {
"fake_process_id": {
"fake_counter": [(1, timestamp1), (1, timestamp2)]
}
}
self.assertDictEqual(
stats_store.ReadStats("f", "fake_counter"),
expected_results)
with test_lib.FakeTime(timestamp3):
system.PurgeServerStatsCronJob(
rdf_cronjobs.CronJobRun(),
rdf_cronjobs.CronJob()).Run()
# timestamp1 is older than 1h, so it should get deleted.
expected_results = {
"fake_process_id": {
"fake_counter": [(1, timestamp2)]
}
}
self.assertDictEqual(
stats_store.ReadStats("f", "fake_counter"),
expected_results)
def testReadStats(self):
with test_lib.FakeTime(rdfvalue.RDFDatetime(1000)):
stats_collector_instance.Get().IncrementCounter(
_SINGLE_DIM_COUNTER)
stats_collector_instance.Get().IncrementCounter(
_COUNTER_WITH_ONE_FIELD, fields=["fieldval1"])
stats_collector_instance.Get().IncrementCounter(
_COUNTER_WITH_TWO_FIELDS, fields=["fieldval2", 3])
stats_store._WriteStats(process_id="fake_process_id")
with test_lib.FakeTime(rdfvalue.RDFDatetime(2000)):
stats_collector_instance.Get().IncrementCounter(
_COUNTER_WITH_TWO_FIELDS, fields=["fieldval2", 3])
stats_collector_instance.Get().IncrementCounter(
_COUNTER_WITH_TWO_FIELDS, fields=["fieldval2", 4])
stats_store._WriteStats(process_id="fake_process_id")
expected_single_dim_results = {
"fake_process_id": {
_SINGLE_DIM_COUNTER: [(1, 1000), (1, 2000)]
}
}
expected_multi_dim1_results = {
"fake_process_id": {
_COUNTER_WITH_ONE_FIELD: {
"fieldval1": [(1, 1000), (1, 2000)]
}
}
}
expected_multi_dim2_results = {
"fake_process_id": {
_COUNTER_WITH_TWO_FIELDS: {
"fieldval2": {
3: [(1, 1000), (2, 2000)],
4: [(1, 2000)]
}
}
}
}
self.assertDictEqual(stats_store.ReadStats("f", _SINGLE_DIM_COUNTER),
expected_single_dim_results)
self.assertDictEqual(
stats_store.ReadStats("fake", _COUNTER_WITH_ONE_FIELD),
expected_multi_dim1_results)
self.assertEqual(
stats_store.ReadStats("fake_process_id", _COUNTER_WITH_TWO_FIELDS),
expected_multi_dim2_results)
def Handle(self, args, token):
start_time = args.start
end_time = args.end
if not end_time:
end_time = rdfvalue.RDFDatetime.Now()
if not start_time:
start_time = end_time - rdfvalue.Duration("1h")
# Run for a little extra time at the start. This improves the quality of the
# first data points of counter metrics which don't appear in every interval.
base_start_time = start_time
# pylint: disable=g-no-augmented-assignment
start_time = start_time - rdfvalue.Duration("10m")
# pylint: enable=g-no-augmented-assignment
if end_time <= start_time:
raise ValueError("End time can't be less than start time.")
result = ApiStatsStoreMetric(start=base_start_time,
end=end_time,
metric_name=args.metric_name)
data = stats_store.ReadStats(unicode(args.component.name.lower()),
args.metric_name,
time_range=(start_time, end_time),
token=token)
if not data:
return result
metric_metadata = stats_collector_instance.Get().GetMetricMetadata(
args.metric_name)
query = stats_store.StatsStoreDataQuery(data)
query.In(args.component.name.lower() + ".*").In(args.metric_name)
if metric_metadata.fields_defs:
query.InAll()
requested_duration = end_time - start_time
if requested_duration >= rdfvalue.Duration("1d"):
sampling_duration = rdfvalue.Duration("5m")
elif requested_duration >= rdfvalue.Duration("6h"):
sampling_duration = rdfvalue.Duration("1m")
else:
sampling_duration = rdfvalue.Duration("30s")
if metric_metadata.metric_type == metric_metadata.MetricType.COUNTER:
query.TakeValue().MakeIncreasing().Normalize(
sampling_duration,
start_time,
end_time,
mode=timeseries.NORMALIZE_MODE_COUNTER)
elif metric_metadata.metric_type == metric_metadata.MetricType.EVENT:
if args.distribution_handling_mode == "DH_SUM":
query.TakeDistributionSum()
elif args.distribution_handling_mode == "DH_COUNT":
query.TakeDistributionCount()
else:
raise ValueError(
"Unexpected request.distribution_handling_mode "
"value: %s." % args.distribution_handling_mode)
query.MakeIncreasing()
query.Normalize(sampling_duration,
start_time,
end_time,
mode=timeseries.NORMALIZE_MODE_COUNTER)
elif metric_metadata.metric_type == metric_metadata.MetricType.GAUGE:
query.TakeValue().Normalize(sampling_duration, start_time,
end_time)
else:
raise RuntimeError("Unsupported metric type.")
if args.aggregation_mode == "AGG_SUM":
query.AggregateViaSum()
elif args.aggregation_mode == "AGG_MEAN":
query.AggregateViaMean()
elif args.aggregation_mode == "AGG_NONE":
pass
else:
raise ValueError("Unexpected request.aggregation value: %s." %
args.aggregation)
if (args.rate and metric_metadata.metric_type !=
metric_metadata.MetricType.GAUGE):
query.Rate()
query.InTimeRange(base_start_time, end_time)
for value, timestamp in query.ts.data:
if value is not None:
result.data_points.append(
ApiStatsStoreMetricDataPoint(timestamp=timestamp,
value=value))
return result
def testPurgeServerStats(self):
if not data_store.RelationalDBReadEnabled():
self.skipTest("Test is only for the relational DB. Skipping...")
fake_stats_collector = prometheus_stats_collector.PrometheusStatsCollector(
[
stats_utils.CreateCounterMetadata("fake_counter"),
])
timestamp0 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(1)
timestamp1 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(2)
timestamp2 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(3600)
timestamp3 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(4800)
config_overrides = {
"Database.useForReads.stats": True,
"StatsStore.stats_ttl_hours": 1
}
zero_duration = rdfvalue.Duration(0)
# Backslash continuation is explicitly allowed by Google's style guide for
# nested context manager expressions spanning 3 or more lines.
# pylint: disable=g-backslash-continuation
with test_lib.ConfigOverrider(config_overrides), \
stats_test_utils.FakeStatsContext(fake_stats_collector), \
mock.patch.object(system, "_STATS_DELETION_BATCH_SIZE", 1), \
mock.patch.object(system, "_stats_checkpoint_period", zero_duration):
with test_lib.FakeTime(rdfvalue.RDFDatetime(timestamp0)):
stats_store._WriteStats(process_id="fake_process_id")
with test_lib.FakeTime(rdfvalue.RDFDatetime(timestamp1)):
stats_collector_instance.Get().IncrementCounter("fake_counter")
stats_store._WriteStats(process_id="fake_process_id")
expected_results = {
"fake_process_id": {
"fake_counter": [(0, timestamp0), (1, timestamp1)]
}
}
self.assertDictEqual(
stats_store.ReadStats("f", "fake_counter"),
expected_results)
with test_lib.FakeTime(timestamp2):
stats_store._WriteStats(process_id="fake_process_id")
expected_results = {
"fake_process_id": {
"fake_counter": [(0, timestamp0), (1, timestamp1),
(1, timestamp2)]
}
}
self.assertDictEqual(
stats_store.ReadStats("f", "fake_counter"),
expected_results)
with test_lib.FakeTime(timestamp3):
cron_name = compatibility.GetName(
system.PurgeServerStatsCronJob)
cronjobs.ScheduleSystemCronJobs(names=[cron_name])
job_data = data_store.REL_DB.ReadCronJobs([cron_name])[0]
cron_run = rdf_cronjobs.CronJobRun(cron_job_id=cron_name)
cron_run.GenerateRunId()
cron_run.started_at = rdfvalue.RDFDatetime.Now()
cron = system.PurgeServerStatsCronJob(cron_run, job_data)
cron.Run()
# timestamp0 and timestamp1 are older than 1h, so they should get
# deleted.
expected_results = {
"fake_process_id": {
"fake_counter": [(1, timestamp2)]
}
}
self.assertDictEqual(
stats_store.ReadStats("f", "fake_counter"),
expected_results)
self.assertEqual(
"Deleted 2 stats entries.\nDeleted 1 stats entries.",
cron.run_state.log_message)
