def testLaterValuesDoNotOverridePrevious(self):
stats.STATS.RegisterCounterMetric("counter")
stats.STATS.IncrementCounter("counter")
self.stats_store.WriteStats(process_id=self.process_id,
timestamp=42,
sync=True)
stats.STATS.IncrementCounter("counter")
self.stats_store.WriteStats(process_id=self.process_id,
timestamp=43,
sync=True)
row = data_store.DB.ResolveRegex("aff4:/stats_store/some_pid",
".*",
token=self.token)
counters = [x for x in row if x[0] == "aff4:stats_store/counter"]
self.assertEqual(len(counters), 2)
counters = sorted(counters, key=lambda x: x[2])
stored_value = rdfvalue.StatsStoreValue(
value_type=rdfvalue.MetricMetadata.ValueType.INT, int_value=1)
self.assertEqual(
counters[0],
("aff4:stats_store/counter", stored_value.SerializeToString(), 42))
stored_value = rdfvalue.StatsStoreValue(
value_type=rdfvalue.MetricMetadata.ValueType.INT, int_value=2)
self.assertEqual(
counters[1],
("aff4:stats_store/counter", stored_value.SerializeToString(), 43))
def WriteStats(self, timestamp=None, sync=False):
to_set = {}
metrics_metadata = stats.STATS.GetAllMetricsMetadata()
self.WriteMetadataDescriptors(metrics_metadata,
timestamp=timestamp,
sync=sync)
for name, metadata in metrics_metadata.iteritems():
if metadata.fields_defs:
for fields_values in stats.STATS.GetMetricFields(name):
value = stats.STATS.GetMetricValue(name,
fields=fields_values)
store_value = rdfvalue.StatsStoreValue()
store_fields_values = []
for field_def, field_value in zip(metadata.fields_defs,
fields_values):
store_field_value = rdfvalue.StatsStoreFieldValue()
store_field_value.SetValue(field_value,
field_def.field_type)
store_fields_values.append(store_field_value)
store_value.fields_values = store_fields_values
store_value.SetValue(value, metadata.value_type)
to_set.setdefault(self.STATS_STORE_PREFIX + name,
[]).append(store_value)
else:
value = stats.STATS.GetMetricValue(name)
store_value = rdfvalue.StatsStoreValue()
store_value.SetValue(value, metadata.value_type)
to_set[self.STATS_STORE_PREFIX + name] = [store_value]
# Write actual data
data_store.DB.MultiSet(self.urn,
to_set,
replace=False,
token=self.token,
timestamp=timestamp,
sync=sync)
def testEventMetricsWithFieldsAreWrittenToDataStore(self):
stats.STATS.RegisterEventMetric("foo_event", fields=[("source", str)])
stats.STATS.RecordEvent("foo_event", 5, fields=["http"])
stats.STATS.RecordEvent("foo_event", 15, fields=["rpc"])
self.stats_store.WriteStats(process_id=self.process_id,
timestamp=42,
sync=True)
row = data_store.DB.ResolveRegex("aff4:/stats_store/some_pid",
".*",
token=self.token)
values = [
rdfvalue.StatsStoreValue(x[1]) for x in row
if x[0] == "aff4:stats_store/foo_event"
]
self.assertEqual(len(values), 2)
http_field_value = rdfvalue.StatsStoreFieldValue(
field_type=rdfvalue.MetricFieldDefinition.FieldType.STR,
str_value="http")
rpc_field_value = rdfvalue.StatsStoreFieldValue(
field_type=rdfvalue.MetricFieldDefinition.FieldType.STR,
str_value="rpc")
# Check that distribution with source=http is written.
http_events = [
x for x in values if x.fields_values == [http_field_value]
]
self.assertTrue(http_events)
self.assertEqual(http_events[0].value_type,
rdfvalue.MetricMetadata.ValueType.DISTRIBUTION)
self.assertEqual(http_events[0].distribution_value.count, 1)
self.assertEqual(http_events[0].distribution_value.sum, 5)
# Check that distribution with source=rpc is written.
rpc_events = [
x for x in values if x.fields_values == [rpc_field_value]
]
self.assertTrue(rpc_events)
self.assertEqual(rpc_events[0].value_type,
rdfvalue.MetricMetadata.ValueType.DISTRIBUTION)
self.assertEqual(rpc_events[0].distribution_value.count, 1)
self.assertEqual(rpc_events[0].distribution_value.sum, 15)
def testIntGaugeValuesAreWrittenToDataStore(self):
stats.STATS.RegisterGaugeMetric("int_gauge", int)
stats.STATS.SetGaugeValue("int_gauge", 4242)
self.stats_store.WriteStats(process_id=self.process_id,
timestamp=42,
sync=True)
row = data_store.DB.ResolveRegex("aff4:/stats_store/some_pid",
".*",
token=self.token)
counter = [x for x in row if x[0] == "aff4:stats_store/int_gauge"]
self.assertTrue(counter)
stored_value = rdfvalue.StatsStoreValue(
value_type=rdfvalue.MetricMetadata.ValueType.INT, int_value=4242)
self.assertEqual(counter[0], ("aff4:stats_store/int_gauge",
stored_value.SerializeToString(), 42))
def testCountersWithFieldsAreWrittenToDataStore(self):
stats.STATS.RegisterCounterMetric("counter", fields=[("source", str)])
stats.STATS.IncrementCounter("counter", fields=["http"])
stats.STATS.IncrementCounter("counter", delta=2, fields=["rpc"])
self.stats_store.WriteStats(process_id=self.process_id,
timestamp=42,
sync=True)
row = data_store.DB.ResolveRegex("aff4:/stats_store/some_pid",
".*",
token=self.token)
# Check that no plain counter is written.
values = [
rdfvalue.StatsStoreValue(x[1]) for x in row
if x[0] == "aff4:stats_store/counter"
]
self.assertEqual(len(values), 2)
http_field_value = rdfvalue.StatsStoreFieldValue(
field_type=rdfvalue.MetricFieldDefinition.FieldType.STR,
str_value="http")
rpc_field_value = rdfvalue.StatsStoreFieldValue(
field_type=rdfvalue.MetricFieldDefinition.FieldType.STR,
str_value="rpc")
# Check that counter with source=http is written.
http_counter = [
x for x in values if x.fields_values == [http_field_value]
]
self.assertTrue(http_counter)
self.assertEqual(http_counter[0].value_type,
rdfvalue.MetricMetadata.ValueType.INT)
self.assertEqual(http_counter[0].int_value, 1)
# Check that counter with source=rpc is written.
rpc_counter = [
x for x in values if x.fields_values == [rpc_field_value]
]
self.assertTrue(rpc_counter)
self.assertEqual(rpc_counter[0].value_type,
rdfvalue.MetricMetadata.ValueType.INT)
self.assertEqual(rpc_counter[0].int_value, 2)
def testEventMetricsAreWrittenToDataStore(self):
stats.STATS.RegisterEventMetric("foo_event")
stats.STATS.RecordEvent("foo_event", 5)
stats.STATS.RecordEvent("foo_event", 15)
self.stats_store.WriteStats(process_id=self.process_id, timestamp=42,
sync=True)
row = data_store.DB.ResolveRegex("aff4:/stats_store/some_pid", ".*",
token=self.token)
values = [rdfvalue.StatsStoreValue(x[1]) for x in row
if x[0] == "aff4:stats_store/foo_event"]
self.assertEqual(len(values), 1)
stored_value = values[0]
self.assertEqual(stored_value.value_type,
rdfvalue.MetricMetadata.ValueType.DISTRIBUTION)
self.assertEqual(stored_value.distribution_value.count, 2)
self.assertEqual(stored_value.distribution_value.sum, 20)
def MultiReadStats(self,
process_ids=None,
predicate_regex=".*",
timestamp=ALL_TIMESTAMPS,
limit=10000):
"""Reads historical data for multiple process ids at once."""
if not process_ids:
process_ids = self.ListUsedProcessIds()
multi_metadata = self.MultiReadMetadata(process_ids=process_ids)
subjects = [
self.DATA_STORE_ROOT.Add(process_id) for process_id in process_ids
]
multi_query_results = data_store.DB.MultiResolveRegex(
subjects,
StatsStoreProcessData.STATS_STORE_PREFIX + predicate_regex,
token=self.token,
timestamp=timestamp,
limit=limit)
results = {}
for subject, subject_results in multi_query_results:
subject = rdfvalue.RDFURN(subject)
subject_results = sorted(subject_results, key=lambda x: x[2])
subject_metadata = multi_metadata.get(subject.Basename(), {})
part_results = {}
for predicate, value_string, timestamp in subject_results:
metric_name = predicate[len(StatsStoreProcessData.
STATS_STORE_PREFIX):]
try:
metadata = subject_metadata[metric_name]
except KeyError:
continue
stored_value = rdfvalue.StatsStoreValue(value_string)
fields_values = []
if metadata.fields_defs:
for stored_field_value in stored_value.fields_values:
fields_values.append(stored_field_value.value)
current_dict = part_results.setdefault(metric_name, {})
for field_value in fields_values[:-1]:
new_dict = {}
current_dict.setdefault(field_value, new_dict)
current_dict = new_dict
result_values_list = current_dict.setdefault(
fields_values[-1], [])
else:
result_values_list = part_results.setdefault(
metric_name, [])
result_values_list.append((stored_value.value, timestamp))
results[subject.Basename()] = part_results
return results
