def threshold_suppress():
"""Suppress errors as long as they stay below a threshold."""
try:
yield
except exceptions, ex:
ts = int(time.time())
timebucket = ts - (ts % interval)
if __BUCKET_ERRORS:
(lastbucket, errors) = __BUCKET_ERRORS[0]
else:
lastbucket = None
if lastbucket != timebucket:
errors = defaultdict(int)
__BUCKET_ERRORS.insert(0, (timebucket, errors))
while len(__BUCKET_ERRORS) > 1:
__BUCKET_ERRORS.pop()
errors[type(ex)] += 1
if errors[type(ex)] < threshold:
stats.incr('%s_suppressed' % type(ex).__name__)
logging.exception("Suppressing error: %s", ex)
return
logging.debug("Too many %s errors, raising", type(ex))
stats.incr('%s_suppress_failures' % type(ex).__name__)
raise
def threshold_suppress():
"""Suppress errors as long as they stay below a threshold."""
try:
yield
except exceptions, ex:
ts = int(time.time())
timebucket = ts - (ts % interval)
if __BUCKET_ERRORS:
(lastbucket, errors) = __BUCKET_ERRORS[0]
else:
lastbucket = None
if lastbucket != timebucket:
errors = defaultdict(int)
__BUCKET_ERRORS.insert(0, (timebucket, errors))
while len(__BUCKET_ERRORS) > 1:
__BUCKET_ERRORS.pop()
errors[type(ex)] += 1
if errors[type(ex)] < threshold:
stats.incr('%s_suppressed' % type(ex).__name__)
logging.exception("Suppressing error: %s", ex)
return
logging.debug("Too many %s errors, raising", type(ex))
stats.incr('%s_suppress_failures' % type(ex).__name__)
raise
def test_fork(self):
collection = stats.fork()
stats.incr("widgets", 5)
self.assertEquals({"widgets": 5}, collection.get_counter_stats())
self.assertEquals({"widgets": 5}, stats.get_counter_stats(reset=True))
stats.incr("widgets", 5)
self.assertEquals({"widgets": 10}, collection.get_counter_stats())
self.assertEquals({"widgets": 5}, stats.get_counter_stats(reset=True))
def add_stats(self, data):
# assume they are using the standard ostrich histogram buckets if
# not specified
bucket_offsets = data.get('bucket_offsets', ORIG_BUCKET_OFFSETS)
for k, v in data['timings'].iteritems():
if v['count'] > 0:
stats.add_timing(k, TimingStat.from_raw_dict(v, bucket_offsets))
for k, v in data['counters'].iteritems():
if v > 0:
stats.incr(k, v)
def add_stat(self, data):
if data.has_key('stats'):
for k, v in data['stats'].items():
stats.add_timing(k, v)
if data.has_key('elapsed'):
stats.add_timing(data['name'], data['elapsed'])
elif data.has_key('name'):
stats.incr(data['name'])
else:
pass
def __wrapper__(func, *args, **kwargs):
attempts = 1
while True:
try:
return func(*args, **kwargs)
except exceptions, ex:
stats.incr('%s_retry' % str(func))
logging.warn("Caught %s on %s attempt %d/%d",
repr(ex), str(func), attempts, max_)
if max_ != -1 and attempts < max_:
attempts += 1
continue
logging.exception("Retries of %s exceeded, giving up.",
str(func))
stats.incr('%s_retry_failure' % str(func))
raise
def __wrapper__(func, *args, **kwargs):
attempts = 1
while True:
try:
return func(*args, **kwargs)
except exceptions, ex:
stats.incr('%s_retry' % str(func))
logging.warn("Caught %s on %s attempt %d/%d", repr(ex),
str(func), attempts, max_)
if max_ != -1 and attempts < max_:
attempts += 1
continue
logging.exception("Retries of %s exceeded, giving up.",
str(func))
stats.incr('%s_retry_failure' % str(func))
raise
def test_report_basic_stats(self, mock_time):
my_time = real_time()
mock_time.return_value = my_time
stats.incr("cats")
stats.incr("dogs", 3)
self.collector.collect()
my_time += 60
mock_time.return_value = my_time
stats.incr("dogs")
self.collector.collect()
data = self.collector.get("counter:dogs")
self.assertEquals((int(my_time - (2 * 60)), 0), data[57])
self.assertEquals((int(my_time - 60), 3), data[58])
self.assertEquals((int(my_time), 1), data[59])
def add_request_stats(self, data):
# print "received %r from %s:%d" % (data, host, port)
endpoint = data['endpoint']
method = data['method']
code = data['code']
simple_code = str(code / 100)
method_endpoint = "%s:%s" % (method, endpoint)
method_code_endpoint = "%s:%s:%s" % (method, simple_code, endpoint)
stats.incr(simple_code)
stats.incr(method_endpoint)
stats.incr(method_code_endpoint)
stats.incr("%s:%s" % (method, simple_code))
stats.incr(endpoint)
stats.incr("%s:%s" % (simple_code, endpoint))
stats.add_timing(simple_code, data['elapsed'])
stats.add_timing(endpoint, data['elapsed'])
stats.add_timing(method_endpoint, data['elapsed'])
for k, v in data.get('stats', {}).items():
stats.add_timing("stats:%s" % k, v)
stats.incr("stats:%s" % k)
for group, v in data.get('groups', {}).items():
stats.incr('%s:%s' % (group, v))
stats.incr('%s:%s:%s' % (group, simple_code, v))
stats.add_timing("%s:%s" % (group, v), data['elapsed'])
self.add_response(method_code_endpoint, data)
self.add_response(simple_code, data)
def test_counters(self):
stats.incr("widgets", 1)
stats.incr("wodgets", 12)
stats.incr("wodgets")
self.assertEquals({"widgets": 1, "wodgets": 13}, stats.get_counter_stats())
