def main(): parser = ArgumentParser(description="Update Graph Explorer metrics") parser.add_argument("configfile", metavar="CONFIG_FILENAME", type=str) args = parser.parse_args() config.init(args.configfile) logger = make_logger('update_metrics', config) os.chdir(os.path.dirname(os.path.abspath(__file__))) try: backend = Backend(config, logger) s_metrics = structured_metrics.StructuredMetrics(config, logger) errors = s_metrics.load_plugins() if len(errors) > 0: logger.warn('errors encountered while loading plugins:') for e in errors: print '\t%s' % e logger.info("fetching/saving metrics from graphite...") backend.download_metrics_json() logger.info("generating structured metrics data...") backend.update_data(s_metrics) logger.info("success!") except Exception, e: # pylint: disable=W0703 logger.error("sorry, something went wrong: %s", e) from traceback import print_exc print_exc() sys.exit(2)
def test_parse_count_and_rate(): s_metrics = structured_metrics.StructuredMetrics() s_metrics.load_plugins() tags_base = { 'n1': 'foo', 'n2': 'req', 'plugin': 'catchall_statsd', 'source': 'statsd', } def get_proto2(key, target_type, unit, updates={}): return testhelpers.get_proto2(key, tags_base, target_type, unit, updates) key = "stats.foo.req" expected = get_proto2(key, 'rate', 'unknown/s') real = s_metrics.list_metrics([key]) assert len(real) == 1 assert expected == real.values()[0] key = "stats_counts.foo.req" expected = get_proto2(key, 'count', 'unknown') real = s_metrics.list_metrics([key]) assert len(real) == 1 assert expected == real.values()[0]
def test_native_proto2_disabled(): # by default, the plugin ignores them s_metrics = structured_metrics.StructuredMetrics() s_metrics.load_plugins() key = "foo.bar=blah.baz" real = s_metrics.list_metrics([key]) assert len(real) == 0
def test_simple(): s_metrics = structured_metrics.StructuredMetrics() s_metrics.load_plugins() tags_base = { 'plugin': 'catchall', 'source': 'unknown', } def get_proto2(key, target_type, unit, updates={}): return testhelpers.get_proto2(key, tags_base, target_type, unit, updates) key = "foo.bar" expected = get_proto2(key, 'unknown', 'unknown', {'n1': 'foo', 'n2': 'bar'}) real = s_metrics.list_metrics([key]) assert len(real) == 1 assert expected == real.values()[0]
def test_native_proto2_enabled(): DummyCfg = type('DummyCfg', (object,), {}) DummyCfg.process_native_proto2 = True s_metrics = structured_metrics.StructuredMetrics(DummyCfg) s_metrics.load_plugins() key = "foo.bar=blah.baz.target_type=rate.unit=MiB/d" real = s_metrics.list_metrics([key]) assert len(real) == 1 assert real.values()[0] == { 'id': key, 'tags': { 'n1': 'foo', 'bar': 'blah', 'n3': 'baz', 'target_type': 'rate', 'unit': 'MiB/d' } }
parser = ArgumentParser(description="Process alerting rules") parser.add_argument("configfile", metavar="CONFIG_FILENAME", type=str) args = parser.parse_args() config.init(args.configfile) config.valid_or_die() if not config.alerting: print "alerting disabled in config" os.exit(0) start_timestamp = int(time.time()) success = True s_metrics = structured_metrics.StructuredMetrics(config) db = Db(config.alerting_db) rules = db.get_rules() output = EmailOutput(config) def submit_maybe(result): if result.to_report(): output.submit(result) db.save_notification(result) print "sent notification!" else: print "no notification"
def test_parse_timers(): s_metrics = structured_metrics.StructuredMetrics() s_metrics.load_plugins() tags_base = { 'n1': 'memcached_default_get', 'plugin': 'catchall_statsd', 'source': 'statsd', } def get_proto2(key, target_type, unit, updates={}): return testhelpers.get_proto2(key, tags_base, target_type, unit, updates) key = "stats.timers.memcached_default_get.count" expected = get_proto2(key, 'count', 'Pckt') real = s_metrics.list_metrics([key]) assert len(real) == 1 assert expected == real.values()[0] key = "stats.timers.memcached_default_get.count_ps" expected = get_proto2(key, 'rate', 'Pckt/s') real = s_metrics.list_metrics([key]) assert len(real) == 1 assert expected == real.values()[0] key = "stats.timers.memcached_default_get.lower" expected = get_proto2(key, 'gauge', 'ms', {'stat': 'lower'}) real = s_metrics.list_metrics([key]) assert len(real) == 1 assert expected == real.values()[0] key = "stats.timers.memcached_default_get.mean" expected = get_proto2(key, 'gauge', 'ms', {'stat': 'mean'}) real = s_metrics.list_metrics([key]) assert len(real) == 1 assert expected == real.values()[0] key = "stats.timers.memcached_default_get.mean_90" expected = get_proto2(key, 'gauge', 'ms', {'stat': 'mean_90'}) real = s_metrics.list_metrics([key]) assert len(real) == 1 assert expected == real.values()[0] key = "stats.timers.memcached_default_get.median" expected = get_proto2(key, 'gauge', 'ms', {'stat': 'median'}) real = s_metrics.list_metrics([key]) assert len(real) == 1 assert expected == real.values()[0] key = "stats.timers.memcached_default_get.std" expected = get_proto2(key, 'gauge', 'ms', {'stat': 'std'}) real = s_metrics.list_metrics([key]) assert len(real) == 1 assert expected == real.values()[0] key = "stats.timers.memcached_default_get.sum" expected = get_proto2(key, 'gauge', 'ms', {'stat': 'sum'}) real = s_metrics.list_metrics([key]) assert len(real) == 1 assert expected == real.values()[0] key = "stats.timers.memcached_default_get.sum_90" expected = get_proto2(key, 'gauge', 'ms', {'stat': 'sum_90'}) real = s_metrics.list_metrics([key]) assert len(real) == 1 assert expected == real.values()[0] key = "stats.timers.memcached_default_get.upper" expected = get_proto2(key, 'gauge', 'ms', {'stat': 'upper'}) real = s_metrics.list_metrics([key]) assert len(real) == 1 assert expected == real.values()[0] key = "stats.timers.memcached_default_get.upper_90" expected = get_proto2(key, 'gauge', 'ms', {'stat': 'upper_90'}) real = s_metrics.list_metrics([key]) assert len(real) == 1 assert expected == real.values()[0] key = "stats.timers.memcached_default_get.histogram.bin_0_01" expected = get_proto2(key, 'gauge', 'freq_abs', {'bin_upper': '0.01', 'orig_unit': 'ms'}) real = s_metrics.list_metrics([key]) assert len(real) == 1 assert expected == real.values()[0] key = "stats.timers.memcached_default_get.histogram.bin_5" expected = get_proto2(key, 'gauge', 'freq_abs', {'bin_upper': '5', 'orig_unit': 'ms'}) real = s_metrics.list_metrics([key]) assert len(real) == 1 assert expected == real.values()[0] key = "stats.timers.memcached_default_get.histogram.bin_inf" expected = get_proto2(key, 'gauge', 'freq_abs', {'bin_upper': 'inf', 'orig_unit': 'ms'}) real = s_metrics.list_metrics([key]) assert len(real) == 1 assert expected == real.values()[0]
def test_parse_timers(): s_metrics = structured_metrics.StructuredMetrics() s_metrics.load_plugins() tags_base = { 'server': 'lvimdfsproxy2', 'plugin': 'swift_proxy_server', 'swift_type': 'object', 'http_method': 'GET', 'http_code': '200' } def get_proto2(key, target_type, unit, updates={}): return testhelpers.get_proto2(key, tags_base, target_type, unit, updates) key = "stats.timers.lvimdfsproxy2.proxy-server.object.GET.200.timing.count" expected = get_proto2(key, 'count', 'Pckt') real = s_metrics.list_metrics([key]) assert len(real) == 1 assert expected == real.values()[0] key = "stats.timers.lvimdfsproxy2.proxy-server.object.GET.200.timing.count_ps" expected = get_proto2(key, 'rate', 'Pckt/s') real = s_metrics.list_metrics([key]) assert len(real) == 1 assert expected == real.values()[0] key = "stats.timers.lvimdfsproxy2.proxy-server.object.GET.200.timing.lower" expected = get_proto2(key, 'gauge', 'ms', {'type': 'lower'}) real = s_metrics.list_metrics([key]) assert len(real) == 1 assert expected == real.values()[0] key = "stats.timers.lvimdfsproxy2.proxy-server.object.GET.200.timing.mean" expected = get_proto2(key, 'gauge', 'ms', {'type': 'mean'}) real = s_metrics.list_metrics([key]) assert len(real) == 1 assert expected == real.values()[0] key = "stats.timers.lvimdfsproxy2.proxy-server.object.GET.200.timing.mean_90" expected = get_proto2(key, 'gauge', 'ms', {'type': 'mean_90'}) real = s_metrics.list_metrics([key]) assert len(real) == 1 assert expected == real.values()[0] key = "stats.timers.lvimdfsproxy2.proxy-server.object.GET.200.timing.median" expected = get_proto2(key, 'gauge', 'ms', {'type': 'median'}) real = s_metrics.list_metrics([key]) assert len(real) == 1 assert expected == real.values()[0] key = "stats.timers.lvimdfsproxy2.proxy-server.object.GET.200.timing.std" expected = get_proto2(key, 'gauge', 'ms', {'type': 'std'}) real = s_metrics.list_metrics([key]) assert len(real) == 1 assert expected == real.values()[0] key = "stats.timers.lvimdfsproxy2.proxy-server.object.GET.200.timing.sum" expected = get_proto2(key, 'gauge', 'ms', {'type': 'sum'}) real = s_metrics.list_metrics([key]) assert len(real) == 1 assert expected == real.values()[0] key = "stats.timers.lvimdfsproxy2.proxy-server.object.GET.200.timing.sum_90" expected = get_proto2(key, 'gauge', 'ms', {'type': 'sum_90'}) real = s_metrics.list_metrics([key]) assert len(real) == 1 assert expected == real.values()[0] key = "stats.timers.lvimdfsproxy2.proxy-server.object.GET.200.timing.upper" expected = get_proto2(key, 'gauge', 'ms', {'type': 'upper'}) real = s_metrics.list_metrics([key]) assert len(real) == 1 assert expected == real.values()[0] key = "stats.timers.lvimdfsproxy2.proxy-server.object.GET.200.timing.upper_90" expected = get_proto2(key, 'gauge', 'ms', {'type': 'upper_90'}) real = s_metrics.list_metrics([key]) assert len(real) == 1 assert expected == real.values()[0]
def test_load(): s_metrics = structured_metrics.StructuredMetrics() errors = s_metrics.load_plugins() assert len(errors) == 0
very convenient to verify the working of plugins etc. """ import sys from pprint import pprint import logging from graph_explorer import structured_metrics from graph_explorer import config from graph_explorer.log import make_logger if len(sys.argv) < 3: print "check_update_metric.py <config file> <metric1> [<metric2> [<metric3...]]" sys.exit(1) config.init(sys.argv[1]) config.valid_or_die() logger = make_logger('check_update_metric', config) logger.setLevel(logging.WARN) s_metrics = structured_metrics.StructuredMetrics(config, logger) errors = s_metrics.load_plugins() if len(errors) > 0: print 'errors encountered while loading plugins:' for e in errors: print '\t%s' % e for v in s_metrics.list_metrics(sys.argv[2:]).values(): pprint(v)