def testInCallAcceptsRegularExpressions(self):
# Initialize and write test data.
stats_collector_instance.Get().IncrementCounter(_SINGLE_DIM_COUNTER)
self.stats_store.WriteStats(
process_id="pid1",
timestamp=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(0))
stats_collector_instance.Get().IncrementCounter(_SINGLE_DIM_COUNTER)
self.stats_store.WriteStats(
process_id="pid1",
timestamp=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(90))
self.stats_store.WriteStats(
process_id="pid2",
timestamp=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(90))
stats_data = self.stats_store.MultiReadStats(
process_ids=["pid1", "pid2"])
query = stats_store.StatsStoreDataQuery(stats_data)
self.assertEqual(
query.In("pid1").In(_SINGLE_DIM_COUNTER).SeriesCount(), 1)
query = stats_store.StatsStoreDataQuery(stats_data)
self.assertEqual(
query.In("pid2").In(_SINGLE_DIM_COUNTER).SeriesCount(), 1)
query = stats_store.StatsStoreDataQuery(stats_data)
self.assertEqual(
query.In("pid.*").In(_SINGLE_DIM_COUNTER).SeriesCount(), 2)
def testInTimeRangeRaisesIfAppliedBeforeTakeMethod(self):
stats_data = self.stats_store.ReadStats(process_id=self.process_id)
query = stats_store.StatsStoreDataQuery(stats_data)
with self.assertRaises(RuntimeError):
query.In(_SINGLE_DIM_COUNTER).InTimeRange(
rdfvalue.RDFDatetime.FromSecondsSinceEpoch(80),
rdfvalue.RDFDatetime.FromSecondsSinceEpoch(120))
def testInTimeRangeLimitsQueriesByTime(self):
# Write test data.
stats_collector_instance.Get().IncrementCounter(_SINGLE_DIM_COUNTER)
self.stats_store.WriteStats(
process_id=self.process_id,
timestamp=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(42))
stats_collector_instance.Get().IncrementCounter(_SINGLE_DIM_COUNTER)
self.stats_store.WriteStats(
process_id=self.process_id,
timestamp=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(100))
stats_collector_instance.Get().IncrementCounter(_SINGLE_DIM_COUNTER)
self.stats_store.WriteStats(
process_id=self.process_id,
timestamp=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(140))
# Read data back.
stats_data = self.stats_store.ReadStats(process_id=self.process_id)
# Check that InTimeRange works as expected.
query = stats_store.StatsStoreDataQuery(stats_data)
ts = query.In(_SINGLE_DIM_COUNTER).TakeValue().InTimeRange(
rdfvalue.RDFDatetime.FromSecondsSinceEpoch(80),
rdfvalue.RDFDatetime.FromSecondsSinceEpoch(120)).ts
self.assertListEqual(ts.data, [[2, 100 * 1e6]])
def testRate(self):
# Write test data.
for i in range(5):
for _ in range(i):
stats_collector_instance.Get().IncrementCounter(
_SINGLE_DIM_COUNTER)
self.stats_store.WriteStats(
process_id=self.process_id,
timestamp=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(10 * i))
# Read data back.
stats_data = self.stats_store.ReadStats(process_id=self.process_id)
# Get time series generated with TakeValue().
query = stats_store.StatsStoreDataQuery(stats_data)
ts = query.In(_SINGLE_DIM_COUNTER).TakeValue().Normalize(
rdfvalue.Duration("10s"), 0, rdfvalue.Duration("50s")).Rate().ts
# We expect following time serie:
# 1970-01-01 00:00:00 0
# 1970-01-01 00:00:10 1
# 1970-01-01 00:00:20 3
# 1970-01-01 00:00:30 6
# 1970-01-01 00:00:40 10
#
# Therefore we expect the following after applying Rate():
# 1970-01-01 00:00:00 0
# 1970-01-01 00:00:10 0.1
# 1970-01-01 00:00:20 0.2
# 1970-01-01 00:00:30 0.3
# 1970-01-01 00:00:40 0.4
self.assertListEqual(
ts.data, [[0.1, 0], [0.2, 10 * 1e6],
[0.30000000000000004, 20 * 1e6], [0.4, 30 * 1e6]])
def testMeanReturnsZeroIfQueryHasNoTimeSeries(self):
# Read data back.
stats_data = self.stats_store.ReadStats(process_id=self.process_id)
# Get time series generated with TakeValue().
query = stats_store.StatsStoreDataQuery(stats_data)
self.assertEqual(query.In(_SINGLE_DIM_COUNTER).TakeValue().Mean(), 0)
def testSeriesCountReturnsNumberOfDataSeriesInCurrentQuery(self):
# Write test data.
stats_collector_instance.Get().IncrementCounter(_SINGLE_DIM_COUNTER)
self.stats_store.WriteStats(
process_id="pid1",
timestamp=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(0))
self.stats_store.WriteStats(
process_id="pid2",
timestamp=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(90))
stats_data = self.stats_store.MultiReadStats(process_ids=["pid1", "pid2"])
query = stats_store.StatsStoreDataQuery(stats_data)
self.assertEqual(query.In("pid.*").SeriesCount(), 2)
query = stats_store.StatsStoreDataQuery(stats_data)
self.assertEqual(query.In("pid1").In(_SINGLE_DIM_COUNTER).SeriesCount(), 1)
query = stats_store.StatsStoreDataQuery(stats_data)
self.assertEqual(query.In("pid.*").In(_SINGLE_DIM_COUNTER).SeriesCount(), 2)
def testTakeValueRaisesIfDistributionIsEncountered(self):
# Write test data.
stats_collector_instance.Get().RecordEvent(_EVENT_METRIC, 42)
self.stats_store.WriteStats(
process_id=self.process_id,
timestamp=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(42))
# Read data back.
stats_data = self.stats_store.ReadStats(process_id=self.process_id)
query = stats_store.StatsStoreDataQuery(stats_data)
with self.assertRaises(ValueError):
query.In(_EVENT_METRIC).TakeValue()
def testAggregateViaSumAggregatesMultipleTimeSeriesIntoOne(self):
# Write test data.
stats_collector_instance.Get().IncrementCounter(_SINGLE_DIM_COUNTER)
self.stats_store.WriteStats(
process_id="pid1",
timestamp=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(0))
stats_collector_instance.Get().IncrementCounter(_SINGLE_DIM_COUNTER)
self.stats_store.WriteStats(
process_id="pid2",
timestamp=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(0))
stats_collector_instance.Get().IncrementCounter(_SINGLE_DIM_COUNTER)
self.stats_store.WriteStats(
process_id="pid1",
timestamp=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(90))
self.stats_store.WriteStats(
process_id="pid2",
timestamp=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(90))
stats_data = self.stats_store.MultiReadStats(
process_ids=["pid1", "pid2"])
query = stats_store.StatsStoreDataQuery(stats_data)
ts = query.In("pid.*").In(_SINGLE_DIM_COUNTER).TakeValue().Normalize(
rdfvalue.Duration("30s"),
0,
rdfvalue.Duration("2m"),
mode=timeseries.NORMALIZE_MODE_COUNTER).AggregateViaSum().ts
# We expect 2 time series in the query:
# 1970-01-01 00:00:00 1
# 1970-01-01 00:00:30 1
# 1970-01-01 00:01:00 1
# 1970-01-01 00:01:30 3
#
# and:
# 1970-01-01 00:00:00 2
# 1970-01-01 00:00:30 2
# 1970-01-01 00:01:00 2
# 1970-01-01 00:01:30 3
#
# Therefore we expect the sum to look like:
# 1970-01-01 00:00:00 3
# 1970-01-01 00:00:30 3
# 1970-01-01 00:01:00 3
# 1970-01-01 00:01:30 6
self.assertAlmostEqual(ts.data[0][0], 3)
self.assertAlmostEqual(ts.data[1][0], 3)
self.assertAlmostEqual(ts.data[2][0], 3)
self.assertAlmostEqual(ts.data[3][0], 6)
self.assertListEqual([t for _, t in ts.data],
[0.0 * 1e6, 30.0 * 1e6, 60.0 * 1e6, 90.0 * 1e6])
def testTakeDistributionSumRaisesIfPlainValueIsEncountered(self):
# Write test data.
stats_collector_instance.Get().IncrementCounter(_SINGLE_DIM_COUNTER)
self.stats_store.WriteStats(
process_id=self.process_id,
timestamp=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(42))
# Read data back.
stats_data = self.stats_store.ReadStats(process_id=self.process_id)
query = stats_store.StatsStoreDataQuery(stats_data)
with self.assertRaises(ValueError):
query.In(_SINGLE_DIM_COUNTER).TakeDistributionSum()
def testMeanReducesTimeSerieToSingleNumber(self):
# Write test data.
for i in range(5):
stats_collector_instance.Get().IncrementCounter(_SINGLE_DIM_COUNTER)
self.stats_store.WriteStats(
process_id=self.process_id,
timestamp=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(10 * i))
# Read data back.
stats_data = self.stats_store.ReadStats(process_id=self.process_id)
# Get time series generated with TakeValue().
query = stats_store.StatsStoreDataQuery(stats_data)
self.assertAlmostEqual(query.In(_SINGLE_DIM_COUNTER).TakeValue().Mean(), 3)
def testMeanRaisesIfCalledOnMultipleTimeSeries(self):
# Write test data.
stats_collector_instance.Get().IncrementCounter(_SINGLE_DIM_COUNTER)
self.stats_store.WriteStats(
process_id="pid1",
timestamp=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(0))
self.stats_store.WriteStats(
process_id="pid2",
timestamp=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(90))
stats_data = self.stats_store.MultiReadStats(process_ids=["pid1", "pid2"])
query = stats_store.StatsStoreDataQuery(stats_data)
with self.assertRaises(RuntimeError):
query.In("pid.*").In(_SINGLE_DIM_COUNTER).TakeValue().Mean()
def testUsingInCallNarrowsQuerySpace(self):
# Create sample data.
stats_collector_instance.Get().IncrementCounter(_SINGLE_DIM_COUNTER)
stats_collector_instance.Get().IncrementCounter(
_COUNTER_WITH_ONE_FIELD, fields=["http"])
stats_collector_instance.Get().IncrementCounter(
_COUNTER_WITH_ONE_FIELD, fields=["rpc"])
# Write to data store.
self.stats_store.WriteStats(process_id=self.process_id, timestamp=42)
# Read them back and apply queries with In() and InAll() calls.
stats_data = self.stats_store.ReadStats(process_id=self.process_id)
query = stats_store.StatsStoreDataQuery(stats_data)
self.assertEqual(query.In(_SINGLE_DIM_COUNTER).SeriesCount(), 1)
query = stats_store.StatsStoreDataQuery(stats_data)
self.assertEqual(query.In(_COUNTER_WITH_ONE_FIELD).InAll().SeriesCount(), 2)
query = stats_store.StatsStoreDataQuery(stats_data)
self.assertEqual(
query.In(_COUNTER_WITH_ONE_FIELD).In("http").SeriesCount(), 1)
def testMakeIncreasingHandlesValuesResets(self):
# Write test data.
self.stats_store.WriteStats(
process_id="pid1",
timestamp=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(0))
stats_collector_instance.Get().IncrementCounter(_SINGLE_DIM_COUNTER)
self.stats_store.WriteStats(
process_id="pid1",
timestamp=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(30))
stats_collector_instance.Get().IncrementCounter(_SINGLE_DIM_COUNTER)
self.stats_store.WriteStats(
process_id="pid1",
timestamp=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(60))
# Simulate process restart by reseting the stats-collector.
with stats_test_utils.FakeStatsContext(_CreateFakeStatsCollector()):
self.stats_store.WriteStats(
process_id="pid1",
timestamp=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(90))
# We've reset the counter on 60th second, so we get following time series:
# 1970-01-01 00:00:00 0
# 1970-01-01 00:00:30 1
# 1970-01-01 00:01:00 2
# 1970-01-01 00:01:30 0
stats_data = self.stats_store.ReadStats(process_id="pid1")
query = stats_store.StatsStoreDataQuery(stats_data)
ts = query.In(_SINGLE_DIM_COUNTER).TakeValue().ts
self.assertAlmostEqual(ts.data[0][0], 0)
self.assertAlmostEqual(ts.data[1][0], 1)
self.assertAlmostEqual(ts.data[2][0], 2)
self.assertAlmostEqual(ts.data[3][0], 0)
# EnsureIsIncremental detects the reset and increments values that follow
# the reset point:
# 1970-01-01 00:00:00 0
# 1970-01-01 00:00:30 1
# 1970-01-01 00:01:00 2
# 1970-01-01 00:01:30 2
ts = query.MakeIncreasing().ts
self.assertAlmostEqual(ts.data[0][0], 0)
self.assertAlmostEqual(ts.data[1][0], 1)
self.assertAlmostEqual(ts.data[2][0], 2)
self.assertAlmostEqual(ts.data[3][0], 2)
def testTakeDistributionSumUsesDistributionSumsToBuildTimeSeries(self):
# Write test data.
stats_collector_instance.Get().RecordEvent(_EVENT_METRIC, 42)
self.stats_store.WriteStats(
process_id=self.process_id,
timestamp=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(42))
stats_collector_instance.Get().RecordEvent(_EVENT_METRIC, 43)
self.stats_store.WriteStats(
process_id=self.process_id,
timestamp=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(100))
# Read data back.
stats_data = self.stats_store.ReadStats(process_id=self.process_id)
query = stats_store.StatsStoreDataQuery(stats_data)
ts = query.In(_EVENT_METRIC).TakeDistributionSum().ts
self.assertListEqual(ts.data, [[42, 42 * 1e6], [85, 100 * 1e6]])
def testTakeValueUsesPlainValuesToBuildTimeSeries(self):
# Write test data.
stats_collector_instance.Get().IncrementCounter(_SINGLE_DIM_COUNTER)
self.stats_store.WriteStats(
process_id=self.process_id,
timestamp=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(42))
stats_collector_instance.Get().IncrementCounter(_SINGLE_DIM_COUNTER)
self.stats_store.WriteStats(
process_id=self.process_id,
timestamp=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(100))
# Read data back.
stats_data = self.stats_store.ReadStats(process_id=self.process_id)
# Get time series generated with TakeValue().
query = stats_store.StatsStoreDataQuery(stats_data)
ts = query.In(_SINGLE_DIM_COUNTER).TakeValue().ts
self.assertListEqual(ts.data, [[1, 42 * 1e6], [2, 100 * 1e6]])
def testNormalizeFillsGapsInTimeSeries(self):
# Write test data.
stats_collector_instance.Get().IncrementCounter(_SINGLE_DIM_COUNTER)
self.stats_store.WriteStats(
process_id=self.process_id,
timestamp=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(0))
stats_collector_instance.Get().IncrementCounter(_SINGLE_DIM_COUNTER)
self.stats_store.WriteStats(
process_id=self.process_id,
timestamp=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(120))
# Read data back.
stats_data = self.stats_store.ReadStats(process_id=self.process_id)
query = stats_store.StatsStoreDataQuery(stats_data)
ts = query.In(_SINGLE_DIM_COUNTER).TakeValue().Normalize(
rdfvalue.Duration("30s"), 0, rdfvalue.Duration("130s")).ts
self.assertListEqual(ts.data, [[1.0, 0], [None, 30 * 1e6], [None, 60 * 1e6],
[None, 90 * 1e6], [2.0, 120 * 1e6]])
def Handle(self, args, token):
start_time = args.start
end_time = args.end
if not end_time:
end_time = rdfvalue.RDFDatetime.Now()
if not start_time:
start_time = end_time - rdfvalue.Duration("1h")
# Run for a little extra time at the start. This improves the quality of the
# first data points of counter metrics which don't appear in every interval.
base_start_time = start_time
# pylint: disable=g-no-augmented-assignment
start_time = start_time - rdfvalue.Duration("10m")
# pylint: enable=g-no-augmented-assignment
if end_time <= start_time:
raise ValueError("End time can't be less than start time.")
result = ApiStatsStoreMetric(start=base_start_time,
end=end_time,
metric_name=args.metric_name)
data = stats_store.ReadStats(unicode(args.component.name.lower()),
args.metric_name,
time_range=(start_time, end_time),
token=token)
if not data:
return result
metric_metadata = stats_collector_instance.Get().GetMetricMetadata(
args.metric_name)
query = stats_store.StatsStoreDataQuery(data)
query.In(args.component.name.lower() + ".*").In(args.metric_name)
if metric_metadata.fields_defs:
query.InAll()
requested_duration = end_time - start_time
if requested_duration >= rdfvalue.Duration("1d"):
sampling_duration = rdfvalue.Duration("5m")
elif requested_duration >= rdfvalue.Duration("6h"):
sampling_duration = rdfvalue.Duration("1m")
else:
sampling_duration = rdfvalue.Duration("30s")
if metric_metadata.metric_type == metric_metadata.MetricType.COUNTER:
query.TakeValue().MakeIncreasing().Normalize(
sampling_duration,
start_time,
end_time,
mode=timeseries.NORMALIZE_MODE_COUNTER)
elif metric_metadata.metric_type == metric_metadata.MetricType.EVENT:
if args.distribution_handling_mode == "DH_SUM":
query.TakeDistributionSum()
elif args.distribution_handling_mode == "DH_COUNT":
query.TakeDistributionCount()
else:
raise ValueError(
"Unexpected request.distribution_handling_mode "
"value: %s." % args.distribution_handling_mode)
query.MakeIncreasing()
query.Normalize(sampling_duration,
start_time,
end_time,
mode=timeseries.NORMALIZE_MODE_COUNTER)
elif metric_metadata.metric_type == metric_metadata.MetricType.GAUGE:
query.TakeValue().Normalize(sampling_duration, start_time,
end_time)
else:
raise RuntimeError("Unsupported metric type.")
if args.aggregation_mode == "AGG_SUM":
query.AggregateViaSum()
elif args.aggregation_mode == "AGG_MEAN":
query.AggregateViaMean()
elif args.aggregation_mode == "AGG_NONE":
pass
else:
raise ValueError("Unexpected request.aggregation value: %s." %
args.aggregation)
if (args.rate and metric_metadata.metric_type !=
metric_metadata.MetricType.GAUGE):
query.Rate()
query.InTimeRange(base_start_time, end_time)
for value, timestamp in query.ts.data:
if value is not None:
result.data_points.append(
ApiStatsStoreMetricDataPoint(timestamp=timestamp,
value=value))
return result
def testNormalizeRaisesIfAppliedBeforeTakeMethod(self):
stats_data = self.stats_store.ReadStats(process_id=self.process_id)
query = stats_store.StatsStoreDataQuery(stats_data)
with self.assertRaises(RuntimeError):
query.In(_SINGLE_DIM_COUNTER).Normalize(15, 0, 60)
