def main():
parser = ArgumentParser(description="Update Graph Explorer metrics")
parser.add_argument("configfile", metavar="CONFIG_FILENAME", type=str)
args = parser.parse_args()
config.init(args.configfile)
logger = make_logger('update_metrics', config)
os.chdir(os.path.dirname(os.path.abspath(__file__)))
try:
backend = Backend(config, logger)
s_metrics = structured_metrics.StructuredMetrics(config, logger)
errors = s_metrics.load_plugins()
if len(errors) > 0:
logger.warn('errors encountered while loading plugins:')
for e in errors:
print '\t%s' % e
logger.info("fetching/saving metrics from graphite...")
backend.download_metrics_json()
logger.info("generating structured metrics data...")
backend.update_data(s_metrics)
logger.info("success!")
except Exception, e: # pylint: disable=W0703
logger.error("sorry, something went wrong: %s", e)
from traceback import print_exc
print_exc()
sys.exit(2)
def test_parse_count_and_rate():
s_metrics = structured_metrics.StructuredMetrics()
s_metrics.load_plugins()
tags_base = {
'n1': 'foo',
'n2': 'req',
'plugin': 'catchall_statsd',
'source': 'statsd',
}
def get_proto2(key, target_type, unit, updates={}):
return testhelpers.get_proto2(key, tags_base, target_type, unit,
updates)
key = "stats.foo.req"
expected = get_proto2(key, 'rate', 'unknown/s')
real = s_metrics.list_metrics([key])
assert len(real) == 1
assert expected == real.values()[0]
key = "stats_counts.foo.req"
expected = get_proto2(key, 'count', 'unknown')
real = s_metrics.list_metrics([key])
assert len(real) == 1
assert expected == real.values()[0]
def test_native_proto2_disabled():
# by default, the plugin ignores them
s_metrics = structured_metrics.StructuredMetrics()
s_metrics.load_plugins()
key = "foo.bar=blah.baz"
real = s_metrics.list_metrics([key])
assert len(real) == 0
def test_simple():
s_metrics = structured_metrics.StructuredMetrics()
s_metrics.load_plugins()
tags_base = {
'plugin': 'catchall',
'source': 'unknown',
}
def get_proto2(key, target_type, unit, updates={}):
return testhelpers.get_proto2(key, tags_base, target_type, unit, updates)
key = "foo.bar"
expected = get_proto2(key, 'unknown', 'unknown', {'n1': 'foo', 'n2': 'bar'})
real = s_metrics.list_metrics([key])
assert len(real) == 1
assert expected == real.values()[0]
def test_native_proto2_enabled():
DummyCfg = type('DummyCfg', (object,), {})
DummyCfg.process_native_proto2 = True
s_metrics = structured_metrics.StructuredMetrics(DummyCfg)
s_metrics.load_plugins()
key = "foo.bar=blah.baz.target_type=rate.unit=MiB/d"
real = s_metrics.list_metrics([key])
assert len(real) == 1
assert real.values()[0] == {
'id': key,
'tags': {
'n1': 'foo',
'bar': 'blah',
'n3': 'baz',
'target_type': 'rate',
'unit': 'MiB/d'
}
}
parser = ArgumentParser(description="Process alerting rules")
parser.add_argument("configfile", metavar="CONFIG_FILENAME", type=str)
args = parser.parse_args()
config.init(args.configfile)
config.valid_or_die()
if not config.alerting:
print "alerting disabled in config"
os.exit(0)
start_timestamp = int(time.time())
success = True
s_metrics = structured_metrics.StructuredMetrics(config)
db = Db(config.alerting_db)
rules = db.get_rules()
output = EmailOutput(config)
def submit_maybe(result):
if result.to_report():
output.submit(result)
db.save_notification(result)
print "sent notification!"
else:
print "no notification"
def test_parse_timers():
s_metrics = structured_metrics.StructuredMetrics()
s_metrics.load_plugins()
tags_base = {
'n1': 'memcached_default_get',
'plugin': 'catchall_statsd',
'source': 'statsd',
}
def get_proto2(key, target_type, unit, updates={}):
return testhelpers.get_proto2(key, tags_base, target_type, unit, updates)
key = "stats.timers.memcached_default_get.count"
expected = get_proto2(key, 'count', 'Pckt')
real = s_metrics.list_metrics([key])
assert len(real) == 1
assert expected == real.values()[0]
key = "stats.timers.memcached_default_get.count_ps"
expected = get_proto2(key, 'rate', 'Pckt/s')
real = s_metrics.list_metrics([key])
assert len(real) == 1
assert expected == real.values()[0]
key = "stats.timers.memcached_default_get.lower"
expected = get_proto2(key, 'gauge', 'ms', {'stat': 'lower'})
real = s_metrics.list_metrics([key])
assert len(real) == 1
assert expected == real.values()[0]
key = "stats.timers.memcached_default_get.mean"
expected = get_proto2(key, 'gauge', 'ms', {'stat': 'mean'})
real = s_metrics.list_metrics([key])
assert len(real) == 1
assert expected == real.values()[0]
key = "stats.timers.memcached_default_get.mean_90"
expected = get_proto2(key, 'gauge', 'ms', {'stat': 'mean_90'})
real = s_metrics.list_metrics([key])
assert len(real) == 1
assert expected == real.values()[0]
key = "stats.timers.memcached_default_get.median"
expected = get_proto2(key, 'gauge', 'ms', {'stat': 'median'})
real = s_metrics.list_metrics([key])
assert len(real) == 1
assert expected == real.values()[0]
key = "stats.timers.memcached_default_get.std"
expected = get_proto2(key, 'gauge', 'ms', {'stat': 'std'})
real = s_metrics.list_metrics([key])
assert len(real) == 1
assert expected == real.values()[0]
key = "stats.timers.memcached_default_get.sum"
expected = get_proto2(key, 'gauge', 'ms', {'stat': 'sum'})
real = s_metrics.list_metrics([key])
assert len(real) == 1
assert expected == real.values()[0]
key = "stats.timers.memcached_default_get.sum_90"
expected = get_proto2(key, 'gauge', 'ms', {'stat': 'sum_90'})
real = s_metrics.list_metrics([key])
assert len(real) == 1
assert expected == real.values()[0]
key = "stats.timers.memcached_default_get.upper"
expected = get_proto2(key, 'gauge', 'ms', {'stat': 'upper'})
real = s_metrics.list_metrics([key])
assert len(real) == 1
assert expected == real.values()[0]
key = "stats.timers.memcached_default_get.upper_90"
expected = get_proto2(key, 'gauge', 'ms', {'stat': 'upper_90'})
real = s_metrics.list_metrics([key])
assert len(real) == 1
assert expected == real.values()[0]
key = "stats.timers.memcached_default_get.histogram.bin_0_01"
expected = get_proto2(key, 'gauge', 'freq_abs', {'bin_upper': '0.01', 'orig_unit': 'ms'})
real = s_metrics.list_metrics([key])
assert len(real) == 1
assert expected == real.values()[0]
key = "stats.timers.memcached_default_get.histogram.bin_5"
expected = get_proto2(key, 'gauge', 'freq_abs', {'bin_upper': '5', 'orig_unit': 'ms'})
real = s_metrics.list_metrics([key])
assert len(real) == 1
assert expected == real.values()[0]
key = "stats.timers.memcached_default_get.histogram.bin_inf"
expected = get_proto2(key, 'gauge', 'freq_abs', {'bin_upper': 'inf', 'orig_unit': 'ms'})
real = s_metrics.list_metrics([key])
assert len(real) == 1
assert expected == real.values()[0]
def test_parse_timers():
s_metrics = structured_metrics.StructuredMetrics()
s_metrics.load_plugins()
tags_base = {
'server': 'lvimdfsproxy2',
'plugin': 'swift_proxy_server',
'swift_type': 'object',
'http_method': 'GET',
'http_code': '200'
}
def get_proto2(key, target_type, unit, updates={}):
return testhelpers.get_proto2(key, tags_base, target_type, unit,
updates)
key = "stats.timers.lvimdfsproxy2.proxy-server.object.GET.200.timing.count"
expected = get_proto2(key, 'count', 'Pckt')
real = s_metrics.list_metrics([key])
assert len(real) == 1
assert expected == real.values()[0]
key = "stats.timers.lvimdfsproxy2.proxy-server.object.GET.200.timing.count_ps"
expected = get_proto2(key, 'rate', 'Pckt/s')
real = s_metrics.list_metrics([key])
assert len(real) == 1
assert expected == real.values()[0]
key = "stats.timers.lvimdfsproxy2.proxy-server.object.GET.200.timing.lower"
expected = get_proto2(key, 'gauge', 'ms', {'type': 'lower'})
real = s_metrics.list_metrics([key])
assert len(real) == 1
assert expected == real.values()[0]
key = "stats.timers.lvimdfsproxy2.proxy-server.object.GET.200.timing.mean"
expected = get_proto2(key, 'gauge', 'ms', {'type': 'mean'})
real = s_metrics.list_metrics([key])
assert len(real) == 1
assert expected == real.values()[0]
key = "stats.timers.lvimdfsproxy2.proxy-server.object.GET.200.timing.mean_90"
expected = get_proto2(key, 'gauge', 'ms', {'type': 'mean_90'})
real = s_metrics.list_metrics([key])
assert len(real) == 1
assert expected == real.values()[0]
key = "stats.timers.lvimdfsproxy2.proxy-server.object.GET.200.timing.median"
expected = get_proto2(key, 'gauge', 'ms', {'type': 'median'})
real = s_metrics.list_metrics([key])
assert len(real) == 1
assert expected == real.values()[0]
key = "stats.timers.lvimdfsproxy2.proxy-server.object.GET.200.timing.std"
expected = get_proto2(key, 'gauge', 'ms', {'type': 'std'})
real = s_metrics.list_metrics([key])
assert len(real) == 1
assert expected == real.values()[0]
key = "stats.timers.lvimdfsproxy2.proxy-server.object.GET.200.timing.sum"
expected = get_proto2(key, 'gauge', 'ms', {'type': 'sum'})
real = s_metrics.list_metrics([key])
assert len(real) == 1
assert expected == real.values()[0]
key = "stats.timers.lvimdfsproxy2.proxy-server.object.GET.200.timing.sum_90"
expected = get_proto2(key, 'gauge', 'ms', {'type': 'sum_90'})
real = s_metrics.list_metrics([key])
assert len(real) == 1
assert expected == real.values()[0]
key = "stats.timers.lvimdfsproxy2.proxy-server.object.GET.200.timing.upper"
expected = get_proto2(key, 'gauge', 'ms', {'type': 'upper'})
real = s_metrics.list_metrics([key])
assert len(real) == 1
assert expected == real.values()[0]
key = "stats.timers.lvimdfsproxy2.proxy-server.object.GET.200.timing.upper_90"
expected = get_proto2(key, 'gauge', 'ms', {'type': 'upper_90'})
real = s_metrics.list_metrics([key])
assert len(real) == 1
assert expected == real.values()[0]
def test_load():
s_metrics = structured_metrics.StructuredMetrics()
errors = s_metrics.load_plugins()
assert len(errors) == 0
very convenient to verify the working of plugins etc.
"""
import sys
from pprint import pprint
import logging
from graph_explorer import structured_metrics
from graph_explorer import config
from graph_explorer.log import make_logger
if len(sys.argv) < 3:
print "check_update_metric.py <config file> <metric1> [<metric2> [<metric3...]]"
sys.exit(1)
config.init(sys.argv[1])
config.valid_or_die()
logger = make_logger('check_update_metric', config)
logger.setLevel(logging.WARN)
s_metrics = structured_metrics.StructuredMetrics(config, logger)
errors = s_metrics.load_plugins()
if len(errors) > 0:
print 'errors encountered while loading plugins:'
for e in errors:
print '\t%s' % e
for v in s_metrics.list_metrics(sys.argv[2:]).values():
pprint(v)
