def test_identity_default(self):
spectator_response = EXAMPLE_MEMORY_USED_RESPONSE
spec = {}
options = {'default_is_identity': True}
transformer = SpectatorMetricTransformer(options, spec)
got_response = transformer.process_response(spectator_response)
self.assertResponseEquals(spectator_response, got_response)
def do_test(self, spec_yaml, spectator_response, expect_response):
spec = yaml.load(spec_yaml)
transformer = SpectatorMetricTransformer(
spec, default_namespace='stackdriver')
got_response = transformer.process_response(spectator_response)
for _, got_meter_data in got_response.items():
values = got_meter_data.get('values')
if values:
values.sort()
self.assertResponseEquals(expect_response, got_response)
def test_snakeify(self):
spec = {}
transformer = SpectatorMetricTransformer({'use_snake_case': True}, spec)
snakeify = lambda name: transformer.normalize_text_case(name)
self.assertEquals('test', snakeify('test'))
self.assertEquals('test', snakeify('Test'))
self.assertEquals('test', snakeify('TEST'))
self.assertEquals('camel_case', snakeify('camelCase'))
self.assertEquals('title_case', snakeify('TitleCase'))
self.assertEquals('snake_case', snakeify('Snake_Case'))
self.assertEquals('http_response', snakeify('HTTPResponse'))
self.assertEquals('upper_case', snakeify('UPPER_CASE'))
def do_test(self, spec_yaml, spectator_response, expect_response,
options=None):
spec = yaml.load(spec_yaml)
options = options or {}
transformer = SpectatorMetricTransformer(options, spec)
got_response = transformer.process_response(spectator_response)
for _, got_meter_data in got_response.items():
values = got_meter_data.get('values')
if values:
values.sort()
self.assertResponseEquals(expect_response, got_response)
def test_stackdriver_timers(self):
spec = {}
transformer = SpectatorMetricTransformer(
{'enforce_stackdriver_names': True}, spec)
do_name = lambda name, kind: transformer.normalize_meter_name(
name, kind)
self.assertEquals('timers', do_name('timers', 'Gauge'))
self.assertEquals('timers', do_name('timers', 'Counter'))
self.assertEquals('timer_latency', do_name('timer', 'Timer'))
self.assertEquals('timer_latency', do_name('timers', 'Timer'))
self.assertEquals('timer_latency', do_name('timer_latency', 'Timer'))
self.assertEquals('timer_latency', do_name('timerLatency', 'Timer'))
def test_summary_as_is(self):
options = {}
transformer = SpectatorMetricTransformer(options, {})
rule = TransformationRule(
transformer,
{
'rename': 'NewName',
'kind': 'DistributionSummary',
'tags': ['status'],
})
builder = AggregatedMetricsBuilder(rule)
self.do_test_compound_kind_as_is(builder)
def __init__(self, options):
self.__spectator_options = normalize_options(options)['spectator']
self.__disable_filter = self.__spectator_options.get(
'disable_metric_filter', False)
self.__disable_transform = self.__spectator_options.get(
'disable_metric_transform', False)
self.__filter_dir = self.__spectator_options.get('metric_filter_dir')
if self.__filter_dir:
logging.info('Using explicit --metric_filter_dir=%s',
self.__filter_dir)
else:
path = os.path.abspath(
os.path.join(os.path.dirname(os.path.dirname(__file__)),
'filters'))
if os.path.exists(path):
self.__filter_dir = path
logging.info('Using implicit --metric_filter_dir=%s',
self.__filter_dir)
elif os.path.exists(DEFAULT_FILTER_DIR):
self.__filter_dir = DEFAULT_FILTER_DIR
logging.info('Using implicit --metric_filter_dir=%s',
self.__filter_dir)
# responses are filtered with only highest precedence filter found.
# service comes from daemon configuration for instrumented service
# default comes from daemon global config
self.__service_metric_filter = {}
self.__service_metric_transformer = {}
# These are bootstrap values so we can reference them in the
# __load_service_filter_and_transform in general. We'll set
# these real values later after we load the 'default'.
self.__default_metric_filter = None
self.__default_metric_transformer = None
default_metric_filter, default_metric_transformer = (
self.__load_service_filter_and_transform('default'))
if default_metric_filter is None:
logging.info('default_metric_filter will be IDENTITY')
default_metric_filter = lambda all: all
if default_metric_transformer is None:
logging.info('default_metric_transformer will be IDENTITY')
copy_opts = dict(self.__spectator_options)
copy_opts['default_is_identity'] = True
spec = {}
default_metric_transformer = SpectatorMetricTransformer(
copy_opts, spec)
self.__default_metric_filter = default_metric_filter
self.__default_metric_transformer = default_metric_transformer
def _make_simple_rule_builder(self, options=None):
# transformer isnt used for these tests, but is required to construct
options = options or {}
transformer = SpectatorMetricTransformer(options, {})
rule = TransformationRule(transformer, {
'rename': 'NewName',
'kind': 'Timer',
'tags': ['status'],
})
# The builder only uses the kind part of the rule.
# The other parts of the rule are used when it is applied
# to preprocess the response before adding to the builder.
return AggregatedMetricsBuilder(rule)
def test_stackdriver_timers(self):
spec = {}
transformer = SpectatorMetricTransformer(
{'enforce_stackdriver_names': True}, spec)
do_name = transformer.normalize_meter_name
self.assertEquals('timers', do_name('timers', 'Gauge'))
self.assertEquals('timers', do_name('timers', 'Counter'))
self.assertEquals('timer_latencies', do_name('timer', 'Timer'))
self.assertEquals('timer_latencies', do_name('timers', 'Timer'))
self.assertEquals('timer_latencies', do_name('timer_latencies', 'Timer'))
self.assertEquals('timer_latencies', do_name('timerLatencies', 'Timer'))
do_label = transformer.normalize_label_name
self.assertEquals('status_code', do_label('status_code'))
self.assertEquals('status_code', do_label('statusCode'))
self.assertEquals('status_code_class', do_label('status'))
def test_summary(self):
transformer = SpectatorMetricTransformer({}, {})
rule = TransformationRule(
transformer,
{
'rename': 'NewName',
'kind': 'Summary',
'tags': ['status'],
})
builder = AggregatedMetricsBuilder(rule)
for measurement in self._make_timer_measurements():
builder.add(measurement['values'][0], measurement['tags'])
self.assertEquals(
[{
'values': [{'v': {'count': 123, 'totalTime': 321},
't': self.TIMESTAMP}],
'tags': [{'key': 'status', 'value': '2xx'}]
}],
sorted(builder.build())
)
def test_summary_summarize(self):
options = {'summarize_compound_kinds': True}
transformer = SpectatorMetricTransformer(options, {})
rule = TransformationRule(
transformer,
{
'rename': 'NewName',
'kind': 'DistributionSummary',
'tags': ['status'],
})
builder = AggregatedMetricsBuilder(rule)
for measurement in self._make_timer_measurements():
builder.add(measurement['values'][0], measurement['tags'])
self.assertEquals(
[{
'values': [{'v': {'count': 123, 'totalTime': 321},
't': self.TIMESTAMP}],
'tags': [{'key': 'status', 'value': '2xx'}]
}],
sort_dictionaries(builder.build())
)
def __load_service_filter_and_transform(self, service):
metric_filter = None
metric_transformer = None
if self.__filter_dir:
logging.debug('Loading transform for service=%s', service)
path = os.path.join(self.__filter_dir, service + '.yml')
if os.path.exists(path):
default_path = os.path.join(self.__filter_dir, 'default.yml')
default_spec = {}
if default_path != path and os.path.exists(default_path):
with open(default_path) as fd:
default_spec = yaml.safe_load(fd)
# pylint: disable=invalid-name
with open(path) as fd:
whole_spec = yaml.safe_load(fd)
if default_spec:
logging.info(
'Adding baseline filters from "default.yml"')
whole_spec = merge_specifications(
default_spec, whole_spec)
filter_spec = whole_spec.get('monitoring',
{}).get('filters')
if self.__disable_filter:
logging.info(
'Filtering is disabled -- no filter on %s.',
service)
elif filter_spec is not None:
logging.info('Loading metric filter from "%s"', path)
metric_filter = MetricFilter(service, filter_spec)
else:
logging.info(
'"%s" has no monitoring.filters entry -- ignoring',
path)
transform_spec = whole_spec.get('monitoring',
{}).get('transforms')
if self.__disable_transform:
logging.info(
'Transforming is disabled -- no transform on %s.',
service)
elif transform_spec is not None:
logging.info('Loading metric transformer from "%s"',
path)
metric_transformer = SpectatorMetricTransformer(
self.__spectator_options, transform_spec)
else:
logging.info(
'"%s" has no monitoring.transforms entry -- ignoring',
path)
if metric_filter is None:
logging.info('Using default metric filter for "%s"', service)
metric_filter = self.__default_metric_filter
if metric_transformer is None:
logging.info('Using default metric transformer for "%s"', service)
metric_transformer = self.__default_metric_transformer
self.__service_metric_filter[service] = metric_filter
self.__service_metric_transformer[service] = metric_transformer
return metric_filter, metric_transformer
def test_identity_explicit(self):
spectator_response = EXAMPLE_MEMORY_USED_RESPONSE
spec = {'jvm.memory.used': None}
transformer = SpectatorMetricTransformer({}, spec)
got_response = transformer.process_response(spectator_response)
self.assertResponseEquals(spectator_response, got_response)
def test_discard_default(self):
spectator_response = EXAMPLE_MEMORY_USED_RESPONSE
spec = {}
transformer = SpectatorMetricTransformer({}, spec)
got_response = transformer.process_response(spectator_response)
self.assertResponseEquals({}, got_response)
