def testLaterValuesDoNotOverridePrevious(self): stats.STATS.RegisterCounterMetric("counter") stats.STATS.IncrementCounter("counter") self.stats_store.WriteStats(process_id=self.process_id, timestamp=42, sync=True) stats.STATS.IncrementCounter("counter") self.stats_store.WriteStats(process_id=self.process_id, timestamp=43, sync=True) row = data_store.DB.ResolveRegex("aff4:/stats_store/some_pid", ".*", token=self.token) counters = [x for x in row if x[0] == "aff4:stats_store/counter"] self.assertEqual(len(counters), 2) counters = sorted(counters, key=lambda x: x[2]) stored_value = rdfvalue.StatsStoreValue( value_type=rdfvalue.MetricMetadata.ValueType.INT, int_value=1) self.assertEqual( counters[0], ("aff4:stats_store/counter", stored_value.SerializeToString(), 42)) stored_value = rdfvalue.StatsStoreValue( value_type=rdfvalue.MetricMetadata.ValueType.INT, int_value=2) self.assertEqual( counters[1], ("aff4:stats_store/counter", stored_value.SerializeToString(), 43))
def WriteStats(self, timestamp=None, sync=False): to_set = {} metrics_metadata = stats.STATS.GetAllMetricsMetadata() self.WriteMetadataDescriptors(metrics_metadata, timestamp=timestamp, sync=sync) for name, metadata in metrics_metadata.iteritems(): if metadata.fields_defs: for fields_values in stats.STATS.GetMetricFields(name): value = stats.STATS.GetMetricValue(name, fields=fields_values) store_value = rdfvalue.StatsStoreValue() store_fields_values = [] for field_def, field_value in zip(metadata.fields_defs, fields_values): store_field_value = rdfvalue.StatsStoreFieldValue() store_field_value.SetValue(field_value, field_def.field_type) store_fields_values.append(store_field_value) store_value.fields_values = store_fields_values store_value.SetValue(value, metadata.value_type) to_set.setdefault(self.STATS_STORE_PREFIX + name, []).append(store_value) else: value = stats.STATS.GetMetricValue(name) store_value = rdfvalue.StatsStoreValue() store_value.SetValue(value, metadata.value_type) to_set[self.STATS_STORE_PREFIX + name] = [store_value] # Write actual data data_store.DB.MultiSet(self.urn, to_set, replace=False, token=self.token, timestamp=timestamp, sync=sync)
def testEventMetricsWithFieldsAreWrittenToDataStore(self): stats.STATS.RegisterEventMetric("foo_event", fields=[("source", str)]) stats.STATS.RecordEvent("foo_event", 5, fields=["http"]) stats.STATS.RecordEvent("foo_event", 15, fields=["rpc"]) self.stats_store.WriteStats(process_id=self.process_id, timestamp=42, sync=True) row = data_store.DB.ResolveRegex("aff4:/stats_store/some_pid", ".*", token=self.token) values = [ rdfvalue.StatsStoreValue(x[1]) for x in row if x[0] == "aff4:stats_store/foo_event" ] self.assertEqual(len(values), 2) http_field_value = rdfvalue.StatsStoreFieldValue( field_type=rdfvalue.MetricFieldDefinition.FieldType.STR, str_value="http") rpc_field_value = rdfvalue.StatsStoreFieldValue( field_type=rdfvalue.MetricFieldDefinition.FieldType.STR, str_value="rpc") # Check that distribution with source=http is written. http_events = [ x for x in values if x.fields_values == [http_field_value] ] self.assertTrue(http_events) self.assertEqual(http_events[0].value_type, rdfvalue.MetricMetadata.ValueType.DISTRIBUTION) self.assertEqual(http_events[0].distribution_value.count, 1) self.assertEqual(http_events[0].distribution_value.sum, 5) # Check that distribution with source=rpc is written. rpc_events = [ x for x in values if x.fields_values == [rpc_field_value] ] self.assertTrue(rpc_events) self.assertEqual(rpc_events[0].value_type, rdfvalue.MetricMetadata.ValueType.DISTRIBUTION) self.assertEqual(rpc_events[0].distribution_value.count, 1) self.assertEqual(rpc_events[0].distribution_value.sum, 15)
def testIntGaugeValuesAreWrittenToDataStore(self): stats.STATS.RegisterGaugeMetric("int_gauge", int) stats.STATS.SetGaugeValue("int_gauge", 4242) self.stats_store.WriteStats(process_id=self.process_id, timestamp=42, sync=True) row = data_store.DB.ResolveRegex("aff4:/stats_store/some_pid", ".*", token=self.token) counter = [x for x in row if x[0] == "aff4:stats_store/int_gauge"] self.assertTrue(counter) stored_value = rdfvalue.StatsStoreValue( value_type=rdfvalue.MetricMetadata.ValueType.INT, int_value=4242) self.assertEqual(counter[0], ("aff4:stats_store/int_gauge", stored_value.SerializeToString(), 42))
def testCountersWithFieldsAreWrittenToDataStore(self): stats.STATS.RegisterCounterMetric("counter", fields=[("source", str)]) stats.STATS.IncrementCounter("counter", fields=["http"]) stats.STATS.IncrementCounter("counter", delta=2, fields=["rpc"]) self.stats_store.WriteStats(process_id=self.process_id, timestamp=42, sync=True) row = data_store.DB.ResolveRegex("aff4:/stats_store/some_pid", ".*", token=self.token) # Check that no plain counter is written. values = [ rdfvalue.StatsStoreValue(x[1]) for x in row if x[0] == "aff4:stats_store/counter" ] self.assertEqual(len(values), 2) http_field_value = rdfvalue.StatsStoreFieldValue( field_type=rdfvalue.MetricFieldDefinition.FieldType.STR, str_value="http") rpc_field_value = rdfvalue.StatsStoreFieldValue( field_type=rdfvalue.MetricFieldDefinition.FieldType.STR, str_value="rpc") # Check that counter with source=http is written. http_counter = [ x for x in values if x.fields_values == [http_field_value] ] self.assertTrue(http_counter) self.assertEqual(http_counter[0].value_type, rdfvalue.MetricMetadata.ValueType.INT) self.assertEqual(http_counter[0].int_value, 1) # Check that counter with source=rpc is written. rpc_counter = [ x for x in values if x.fields_values == [rpc_field_value] ] self.assertTrue(rpc_counter) self.assertEqual(rpc_counter[0].value_type, rdfvalue.MetricMetadata.ValueType.INT) self.assertEqual(rpc_counter[0].int_value, 2)
def testEventMetricsAreWrittenToDataStore(self): stats.STATS.RegisterEventMetric("foo_event") stats.STATS.RecordEvent("foo_event", 5) stats.STATS.RecordEvent("foo_event", 15) self.stats_store.WriteStats(process_id=self.process_id, timestamp=42, sync=True) row = data_store.DB.ResolveRegex("aff4:/stats_store/some_pid", ".*", token=self.token) values = [rdfvalue.StatsStoreValue(x[1]) for x in row if x[0] == "aff4:stats_store/foo_event"] self.assertEqual(len(values), 1) stored_value = values[0] self.assertEqual(stored_value.value_type, rdfvalue.MetricMetadata.ValueType.DISTRIBUTION) self.assertEqual(stored_value.distribution_value.count, 2) self.assertEqual(stored_value.distribution_value.sum, 20)
def MultiReadStats(self, process_ids=None, predicate_regex=".*", timestamp=ALL_TIMESTAMPS, limit=10000): """Reads historical data for multiple process ids at once.""" if not process_ids: process_ids = self.ListUsedProcessIds() multi_metadata = self.MultiReadMetadata(process_ids=process_ids) subjects = [ self.DATA_STORE_ROOT.Add(process_id) for process_id in process_ids ] multi_query_results = data_store.DB.MultiResolveRegex( subjects, StatsStoreProcessData.STATS_STORE_PREFIX + predicate_regex, token=self.token, timestamp=timestamp, limit=limit) results = {} for subject, subject_results in multi_query_results: subject = rdfvalue.RDFURN(subject) subject_results = sorted(subject_results, key=lambda x: x[2]) subject_metadata = multi_metadata.get(subject.Basename(), {}) part_results = {} for predicate, value_string, timestamp in subject_results: metric_name = predicate[len(StatsStoreProcessData. STATS_STORE_PREFIX):] try: metadata = subject_metadata[metric_name] except KeyError: continue stored_value = rdfvalue.StatsStoreValue(value_string) fields_values = [] if metadata.fields_defs: for stored_field_value in stored_value.fields_values: fields_values.append(stored_field_value.value) current_dict = part_results.setdefault(metric_name, {}) for field_value in fields_values[:-1]: new_dict = {} current_dict.setdefault(field_value, new_dict) current_dict = new_dict result_values_list = current_dict.setdefault( fields_values[-1], []) else: result_values_list = part_results.setdefault( metric_name, []) result_values_list.append((stored_value.value, timestamp)) results[subject.Basename()] = part_results return results