def test_identity_default(self): spectator_response = EXAMPLE_MEMORY_USED_RESPONSE spec = {} options = {'default_is_identity': True} transformer = SpectatorMetricTransformer(options, spec) got_response = transformer.process_response(spectator_response) self.assertResponseEquals(spectator_response, got_response)
def do_test(self, spec_yaml, spectator_response, expect_response): spec = yaml.load(spec_yaml) transformer = SpectatorMetricTransformer( spec, default_namespace='stackdriver') got_response = transformer.process_response(spectator_response) for _, got_meter_data in got_response.items(): values = got_meter_data.get('values') if values: values.sort() self.assertResponseEquals(expect_response, got_response)
def test_snakeify(self): spec = {} transformer = SpectatorMetricTransformer({'use_snake_case': True}, spec) snakeify = lambda name: transformer.normalize_text_case(name) self.assertEquals('test', snakeify('test')) self.assertEquals('test', snakeify('Test')) self.assertEquals('test', snakeify('TEST')) self.assertEquals('camel_case', snakeify('camelCase')) self.assertEquals('title_case', snakeify('TitleCase')) self.assertEquals('snake_case', snakeify('Snake_Case')) self.assertEquals('http_response', snakeify('HTTPResponse')) self.assertEquals('upper_case', snakeify('UPPER_CASE'))
def do_test(self, spec_yaml, spectator_response, expect_response, options=None): spec = yaml.load(spec_yaml) options = options or {} transformer = SpectatorMetricTransformer(options, spec) got_response = transformer.process_response(spectator_response) for _, got_meter_data in got_response.items(): values = got_meter_data.get('values') if values: values.sort() self.assertResponseEquals(expect_response, got_response)
def test_stackdriver_timers(self): spec = {} transformer = SpectatorMetricTransformer( {'enforce_stackdriver_names': True}, spec) do_name = lambda name, kind: transformer.normalize_meter_name( name, kind) self.assertEquals('timers', do_name('timers', 'Gauge')) self.assertEquals('timers', do_name('timers', 'Counter')) self.assertEquals('timer_latency', do_name('timer', 'Timer')) self.assertEquals('timer_latency', do_name('timers', 'Timer')) self.assertEquals('timer_latency', do_name('timer_latency', 'Timer')) self.assertEquals('timer_latency', do_name('timerLatency', 'Timer'))
def test_summary_as_is(self): options = {} transformer = SpectatorMetricTransformer(options, {}) rule = TransformationRule( transformer, { 'rename': 'NewName', 'kind': 'DistributionSummary', 'tags': ['status'], }) builder = AggregatedMetricsBuilder(rule) self.do_test_compound_kind_as_is(builder)
def __init__(self, options): self.__spectator_options = normalize_options(options)['spectator'] self.__disable_filter = self.__spectator_options.get( 'disable_metric_filter', False) self.__disable_transform = self.__spectator_options.get( 'disable_metric_transform', False) self.__filter_dir = self.__spectator_options.get('metric_filter_dir') if self.__filter_dir: logging.info('Using explicit --metric_filter_dir=%s', self.__filter_dir) else: path = os.path.abspath( os.path.join(os.path.dirname(os.path.dirname(__file__)), 'filters')) if os.path.exists(path): self.__filter_dir = path logging.info('Using implicit --metric_filter_dir=%s', self.__filter_dir) elif os.path.exists(DEFAULT_FILTER_DIR): self.__filter_dir = DEFAULT_FILTER_DIR logging.info('Using implicit --metric_filter_dir=%s', self.__filter_dir) # responses are filtered with only highest precedence filter found. # service comes from daemon configuration for instrumented service # default comes from daemon global config self.__service_metric_filter = {} self.__service_metric_transformer = {} # These are bootstrap values so we can reference them in the # __load_service_filter_and_transform in general. We'll set # these real values later after we load the 'default'. self.__default_metric_filter = None self.__default_metric_transformer = None default_metric_filter, default_metric_transformer = ( self.__load_service_filter_and_transform('default')) if default_metric_filter is None: logging.info('default_metric_filter will be IDENTITY') default_metric_filter = lambda all: all if default_metric_transformer is None: logging.info('default_metric_transformer will be IDENTITY') copy_opts = dict(self.__spectator_options) copy_opts['default_is_identity'] = True spec = {} default_metric_transformer = SpectatorMetricTransformer( copy_opts, spec) self.__default_metric_filter = default_metric_filter self.__default_metric_transformer = default_metric_transformer
def _make_simple_rule_builder(self, options=None): # transformer isnt used for these tests, but is required to construct options = options or {} transformer = SpectatorMetricTransformer(options, {}) rule = TransformationRule(transformer, { 'rename': 'NewName', 'kind': 'Timer', 'tags': ['status'], }) # The builder only uses the kind part of the rule. # The other parts of the rule are used when it is applied # to preprocess the response before adding to the builder. return AggregatedMetricsBuilder(rule)
def test_stackdriver_timers(self): spec = {} transformer = SpectatorMetricTransformer( {'enforce_stackdriver_names': True}, spec) do_name = transformer.normalize_meter_name self.assertEquals('timers', do_name('timers', 'Gauge')) self.assertEquals('timers', do_name('timers', 'Counter')) self.assertEquals('timer_latencies', do_name('timer', 'Timer')) self.assertEquals('timer_latencies', do_name('timers', 'Timer')) self.assertEquals('timer_latencies', do_name('timer_latencies', 'Timer')) self.assertEquals('timer_latencies', do_name('timerLatencies', 'Timer')) do_label = transformer.normalize_label_name self.assertEquals('status_code', do_label('status_code')) self.assertEquals('status_code', do_label('statusCode')) self.assertEquals('status_code_class', do_label('status'))
def test_summary(self): transformer = SpectatorMetricTransformer({}, {}) rule = TransformationRule( transformer, { 'rename': 'NewName', 'kind': 'Summary', 'tags': ['status'], }) builder = AggregatedMetricsBuilder(rule) for measurement in self._make_timer_measurements(): builder.add(measurement['values'][0], measurement['tags']) self.assertEquals( [{ 'values': [{'v': {'count': 123, 'totalTime': 321}, 't': self.TIMESTAMP}], 'tags': [{'key': 'status', 'value': '2xx'}] }], sorted(builder.build()) )
def test_summary_summarize(self): options = {'summarize_compound_kinds': True} transformer = SpectatorMetricTransformer(options, {}) rule = TransformationRule( transformer, { 'rename': 'NewName', 'kind': 'DistributionSummary', 'tags': ['status'], }) builder = AggregatedMetricsBuilder(rule) for measurement in self._make_timer_measurements(): builder.add(measurement['values'][0], measurement['tags']) self.assertEquals( [{ 'values': [{'v': {'count': 123, 'totalTime': 321}, 't': self.TIMESTAMP}], 'tags': [{'key': 'status', 'value': '2xx'}] }], sort_dictionaries(builder.build()) )
def __load_service_filter_and_transform(self, service): metric_filter = None metric_transformer = None if self.__filter_dir: logging.debug('Loading transform for service=%s', service) path = os.path.join(self.__filter_dir, service + '.yml') if os.path.exists(path): default_path = os.path.join(self.__filter_dir, 'default.yml') default_spec = {} if default_path != path and os.path.exists(default_path): with open(default_path) as fd: default_spec = yaml.safe_load(fd) # pylint: disable=invalid-name with open(path) as fd: whole_spec = yaml.safe_load(fd) if default_spec: logging.info( 'Adding baseline filters from "default.yml"') whole_spec = merge_specifications( default_spec, whole_spec) filter_spec = whole_spec.get('monitoring', {}).get('filters') if self.__disable_filter: logging.info( 'Filtering is disabled -- no filter on %s.', service) elif filter_spec is not None: logging.info('Loading metric filter from "%s"', path) metric_filter = MetricFilter(service, filter_spec) else: logging.info( '"%s" has no monitoring.filters entry -- ignoring', path) transform_spec = whole_spec.get('monitoring', {}).get('transforms') if self.__disable_transform: logging.info( 'Transforming is disabled -- no transform on %s.', service) elif transform_spec is not None: logging.info('Loading metric transformer from "%s"', path) metric_transformer = SpectatorMetricTransformer( self.__spectator_options, transform_spec) else: logging.info( '"%s" has no monitoring.transforms entry -- ignoring', path) if metric_filter is None: logging.info('Using default metric filter for "%s"', service) metric_filter = self.__default_metric_filter if metric_transformer is None: logging.info('Using default metric transformer for "%s"', service) metric_transformer = self.__default_metric_transformer self.__service_metric_filter[service] = metric_filter self.__service_metric_transformer[service] = metric_transformer return metric_filter, metric_transformer
def test_identity_explicit(self): spectator_response = EXAMPLE_MEMORY_USED_RESPONSE spec = {'jvm.memory.used': None} transformer = SpectatorMetricTransformer({}, spec) got_response = transformer.process_response(spectator_response) self.assertResponseEquals(spectator_response, got_response)
def test_discard_default(self): spectator_response = EXAMPLE_MEMORY_USED_RESPONSE spec = {} transformer = SpectatorMetricTransformer({}, spec) got_response = transformer.process_response(spectator_response) self.assertResponseEquals({}, got_response)