def __init__(self, alpha, interval):
self.alpha = alpha
self.interval = interval
self.initialized = False
self._rate = 0.0
self._uncounted = AtomicLong(0)
def __init__(self, sample):
self.sample = sample
self.counter = AtomicLong(0)
self.minimum = AtomicLong(sys.maxsize)
self.maximum = AtomicLong(-sys.maxsize - 1)
self.sum = AtomicLong(0)
self.var = AtomicLongArray([-1, 0])
def test_compare_and_swap(self):
atomic = AtomicLong(1000)
swapped = atomic.compare_and_swap(1000, 1001)
self.assertEqual(1001, atomic.value)
self.assertEqual(True, swapped)
swapped = atomic.compare_and_swap(1000, 1024)
self.assertEqual(1001, atomic.value)
self.assertEqual(False, swapped)
def __init__(self, average_class=EWMA):
self.counter = AtomicLong(0)
self.start_time = now()
self.last_tick = AtomicLong(self.start_time)
self.interval = EWMA.INTERVAL
self.m1_rate = EWMA.m1()
self.m5_rate = EWMA.m5()
self.m15_rate = EWMA.m15()
class Stat(object):
gen_tasks = AtomicLong(0)
finished_tasks = AtomicLong(0)
task_queue = queue.Queue()
# Each result contains several fields
# 1. TaskID
# 2. TimePoints: create time, send time, response time
# 3. Succ
result_queue = queue.Queue()
def test_compare_and_swap(self):
atomic = AtomicLong(1000)
swapped = atomic.compare_and_swap(1000, 1001)
self.assertEqual(1001, atomic.value)
self.assertEqual(True, swapped)
swapped = atomic.compare_and_swap(1000, 1024)
self.assertEqual(1001, atomic.value)
self.assertEqual(False, swapped)
def __init__(self, reservoir_size, alpha):
self.values = []
self.next_scale_time = AtomicLong(0)
self.alpha = alpha
self.reservoir_size = reservoir_size
self.lock = RLock()
self.rescale_threshold = \
ExponentiallyDecayingSample.calculate_rescale_threshold(alpha)
self.clear()
def __init__(self, reservoir_size, alpha):
self.values = []
self.next_scale_time = AtomicLong(0)
self.alpha = alpha
self.reservoir_size = reservoir_size
self.lock = RLock()
self.rescale_threshold = ExponentiallyDecayingSample.calculate_rescale_threshold(alpha)
self.clear()
def __init__(self, average_class=EWMA):
self.counter = AtomicLong(0)
self.start_time = now()
self.last_tick = AtomicLong(self.start_time)
self.interval = EWMA.INTERVAL
self.m1_rate = EWMA.m1()
self.m5_rate = EWMA.m5()
self.m15_rate = EWMA.m15()
class ToggleGauge(Gauge):
_value = AtomicLong(1)
@property
def value(self):
try:
return self._value.value
finally:
self._value.value = 0
class EWMA(object):
INTERVAL = 5.0
SECONDS_PER_MINUTE = 60.0
ONE_MINUTE = 1
FIVE_MINUTES = 5
FIFTEEN_MINUTES = 15
M1_ALPHA = 1 - math.exp(-INTERVAL / SECONDS_PER_MINUTE / ONE_MINUTE)
M5_ALPHA = 1 - math.exp(-INTERVAL / SECONDS_PER_MINUTE / FIVE_MINUTES)
M15_ALPHA = 1 - math.exp(-INTERVAL / SECONDS_PER_MINUTE / FIFTEEN_MINUTES)
@classmethod
def m1(cls):
return EWMA(cls.M1_ALPHA, cls.INTERVAL)
@classmethod
def m5(cls):
return EWMA(cls.M5_ALPHA, cls.INTERVAL)
@classmethod
def m15(cls):
return EWMA(cls.M15_ALPHA, cls.INTERVAL)
def __init__(self, alpha, interval):
self.alpha = alpha
self.interval = interval
self.initialized = False
self._rate = 0.0
self._uncounted = AtomicLong(0)
def clear(self):
self.initialized = False
self._rate = 0.0
self._uncounted.value = 0
def update(self, value):
self._uncounted += value
def tick(self):
count = self._uncounted.swap(0)
instant_rate = count / self.interval
if self.initialized:
self._rate += self.alpha * (instant_rate - self._rate)
else:
self._rate = instant_rate
self.initialized = True
@property
def rate(self):
return self._rate
def __init__(self, sample):
self.sample = sample
self.counter = AtomicLong(0)
self.minimum = AtomicLong(sys.maxsize)
self.maximum = AtomicLong(-sys.maxsize - 1)
self.sum = AtomicLong(0)
self.var = AtomicLongArray([-1, 0])
def __init__(self, service_meta, client_id=None):
self.service_meta = service_meta
self.client_id = client_id
if not self.client_id:
self.client_id = gen_uuid()
if service_meta.serverimpl != "rabbitmq":
raise RuntimeError("serverimpl {} not implemented yet".format(
service_meta.serverimpl))
self._sender = RabbitmqClientSender(service_meta)
if service_meta.callmode != "notify":
# need a queue to receive response
self._receiver = RabbitmqClientReceiver(self.client_id)
else:
self._receiver = None
if service_meta.coder != "mail":
raise RuntimeError("coder {} is not supported yet".format(
service_meta.coder))
self.requestid_base = AtomicLong(0)
self.running = True
class Derive(Meter):
"""
A derive is like a meter but accepts an absolute counter as input.
derive = Metrology.derive('network.io')
derive.mark()
derive.count
"""
def __init__(self, average_class=EWMA):
self.last = AtomicLong(0)
super(Derive, self).__init__(average_class)
def mark(self, value=1):
"""Record an event with the derive.
:param value: counter value to record
"""
last = self.last.get_and_set(value)
if last <= value:
value = value - last
super(Derive, self).mark(value)
def __init__(self, service_meta, **kwargs):
self.raise_if_connect_error = kwargs.get('raise_if_connect_error',True) #add by wxt 2015-12-16 如果初始化失败是否raise异常
self.running = False
try:
self.service_meta = service_meta
# atexit
if self.service_meta.callmode == "notify":
atexit.register(self.batch_notify_flush)
self.client_id = kwargs.get("client_id")
if not self.client_id:
self.client_id = gen_uuid()
self.client_id = '_client.{}.{}'.format(self.service_meta.name,self.client_id)
if service_meta.serverimpl == "rabbitmq":
from . import babelrabbitmq
self.impl = babelrabbitmq
elif service_meta.serverimpl == "redis":
from . import babelredis
self.impl = babelredis
else:
raise RuntimeError("serverimpl {} not implemented yet".format(service_meta.serverimpl))
if "sdc" not in self.service_meta.options:
raise RuntimeError("sdc not in service meta")
self.sdc = self.service_meta.options.get("sdc", "")
if "cdc" not in self.service_meta.options:
raise RuntimeError("cdc not in service meta")
self.cdc = self.service_meta.options.get("cdc", "")
if "," in self.service_meta.options["sdc"] and self.service_meta.callmode != "notify":
raise RuntimeError("only notify supports multiple data center")
self._sender = self.impl.get_client_sender(service_meta, **kwargs)
if service_meta.callmode != "notify":
# need a queue to receive response
self._receiver = self.impl.get_client_receiver(service_meta, self.client_id, **kwargs)
else:
self._receiver = None
if service_meta.coder != "mail":
raise RuntimeError("coder {} is not supported yet".format(service_meta.coder))
self.requestid_base = AtomicLong(0)
self.request_cache = RequestCache()
self.running = True
self.response_task = None
self.metrics_tags = {
"service": service_meta.name,
"delivery": service_meta.delivermode,
"call": service_meta.callmode,
"impl": service_meta.serverimpl,
"clientid": self.client_id,
"sdc": self.sdc,
"cdc": self.cdc
}
# caching requests for batch_notify
self.batch_cache = list()
self.running = True
except Exception,e:
if self.raise_if_connect_error:
print e
raise RuntimeError('babel connect error')
class ExponentiallyDecayingSample(object):
def __init__(self, reservoir_size, alpha):
self.values = []
self.next_scale_time = AtomicLong(0)
self.alpha = alpha
self.reservoir_size = reservoir_size
self.lock = RLock()
self.rescale_threshold = \
ExponentiallyDecayingSample.calculate_rescale_threshold(alpha)
self.clear()
@staticmethod
def calculate_rescale_threshold(alpha):
# determine rescale-threshold such that we will not overflow exp() in
# weight function, and subsequently not overflow into inf on dividing
# by random.random()
min_rand = 1.0 / (2**32) # minimum non-zero value from random()
safety = 2.0 # safety pad for numerical inaccuracy
max_value = sys.float_info.max * min_rand / safety
return int(math.log(max_value) / alpha)
def clear(self):
with self.lock:
self.values = []
self.start_time = now()
self.next_scale_time.value = \
self.start_time + self.rescale_threshold
def size(self):
with self.lock:
return len(self.values)
def __len__(self):
return self.size()
def snapshot(self):
with self.lock:
return Snapshot(val for _, val in self.values)
def weight(self, timestamp):
return math.exp(self.alpha * (timestamp - self.start_time))
def rescale(self, now, next_time):
if self.next_scale_time.compare_and_swap(next_time,
now + self.rescale_threshold):
with self.lock:
rescaleFactor = math.exp(-self.alpha * (now - self.start_time))
self.values = [(k * rescaleFactor, v) for k, v in self.values]
self.start_time = now
def rescale_if_necessary(self):
time = now()
next_time = self.next_scale_time.value
if time > next_time:
self.rescale(time, next_time)
def update(self, value, timestamp=None):
if timestamp is None:
timestamp = now()
self.rescale_if_necessary()
with self.lock:
try:
priority = self.weight(timestamp) / random.random()
except (OverflowError, ZeroDivisionError):
priority = sys.float_info.max
if len(self.values) < self.reservoir_size:
heapq.heappush(self.values, (priority, value))
else:
heapq.heappushpop(self.values, (priority, value))
def __init__(self, service_meta):
self.requestid_gen = AtomicLong(0)
pass
class ExponentiallyDecayingSample(object):
def __init__(self, reservoir_size, alpha):
self.values = []
self.next_scale_time = AtomicLong(0)
self.alpha = alpha
self.reservoir_size = reservoir_size
self.lock = RLock()
self.rescale_threshold = ExponentiallyDecayingSample.calculate_rescale_threshold(alpha)
self.clear()
@staticmethod
def calculate_rescale_threshold(alpha):
# determine rescale-threshold such that we will not overflow exp() in
# weight function, and subsequently not overflow into inf on dividing
# by random.random()
min_rand = 1.0 / (2 ** 32) # minimum non-zero value from random()
safety = 2.0 # safety pad for numerical inaccuracy
max_value = sys.float_info.max * min_rand / safety
return int(math.log(max_value) / alpha)
def clear(self):
with self.lock:
self.values = []
self.start_time = now()
self.next_scale_time.value = self.start_time + self.rescale_threshold
def size(self):
with self.lock:
return len(self.values)
def __len__(self):
return self.size()
def snapshot(self):
with self.lock:
return Snapshot(val for _, val in self.values)
def weight(self, timestamp):
return math.exp(self.alpha * (timestamp - self.start_time))
def rescale(self, now, next_time):
if self.next_scale_time.compare_and_swap(next_time, now + self.rescale_threshold):
with self.lock:
rescaleFactor = math.exp(-self.alpha * (now - self.start_time))
self.values = [(k * rescaleFactor, v) for k, v in self.values]
self.start_time = now
def rescale_if_necessary(self):
time = now()
next_time = self.next_scale_time.value
if time > next_time:
self.rescale(time, next_time)
def update(self, value, timestamp=None):
if timestamp is None:
timestamp = now()
self.rescale_if_necessary()
with self.lock:
try:
priority = self.weight(timestamp) / random.random()
except (OverflowError, ZeroDivisionError):
priority = sys.float_info.max
if len(self.values) < self.reservoir_size:
heapq.heappush(self.values, (priority, value))
else:
heapq.heappushpop(self.values, (priority, value))
class Meter(object):
"""A meter measures the rate of events over time
(e.g., "requests per second").
In addition to the mean rate, you can also track 1, 5 and 15 minutes moving
averages ::
meter = Metrology.meter('requests')
meter.mark()
meter.count
"""
def __init__(self, average_class=EWMA):
self.counter = AtomicLong(0)
self.start_time = now()
self.last_tick = AtomicLong(self.start_time)
self.interval = EWMA.INTERVAL
self.m1_rate = EWMA.m1()
self.m5_rate = EWMA.m5()
self.m15_rate = EWMA.m15()
def _tick(self):
old_tick, new_tick = self.last_tick.value, time()
age = new_tick - old_tick
ticks = int(age / self.interval)
new_tick = old_tick + int(ticks * self.interval)
if ticks and self.last_tick.compare_and_swap(old_tick, new_tick):
for _ in range(ticks):
self.tick()
def __call__(self, *args, **kwargs):
if args and hasattr(args[0], '__call__'):
_orig_func = args[0]
def _decorator(*args, **kwargs):
with self:
return _orig_func(*args, **kwargs)
return _decorator
def __enter__(self):
pass
def __exit__(self, exc, exv, trace):
self.mark()
@property
def count(self):
"""Returns the total number of events that have been recorded."""
return self.counter.value
def clear(self):
self.counter.value = 0
self.start_time = time()
self.m1_rate.clear()
self.m5_rate.clear()
self.m15_rate.clear()
@ticker
def mark(self, value=1):
"""Record an event with the meter. By default it will record one event.
:param value: number of event to record
"""
self.counter += value
self.m1_rate.update(value)
self.m5_rate.update(value)
self.m15_rate.update(value)
def tick(self):
self.m1_rate.tick()
self.m5_rate.tick()
self.m15_rate.tick()
@property
@ticker
def one_minute_rate(self):
"""Returns the one-minute average rate."""
return self.m1_rate.rate
@property
@ticker
def five_minute_rate(self):
"""Returns the five-minute average rate."""
return self.m5_rate.rate
@property
@ticker
def fifteen_minute_rate(self):
"""Returns the fifteen-minute average rate."""
return self.m15_rate.rate
@property
def mean_rate(self):
"""
Returns the mean rate of the events since the start of the process.
"""
if self.counter.value == 0:
return 0.0
else:
elapsed = time() - self.start_time
return self.counter.value / elapsed
def stop(self):
pass
def test_swap(self):
atomic = AtomicLong(1000)
swapped = atomic.swap(1001)
self.assertEqual(1001, atomic.value)
self.assertEqual(1000, swapped)
def test_value(self):
atomic = AtomicLong(0)
atomic.value = 1
self.assertEqual(1, atomic.value)
class UnsafeServiceClient(object):
def __init__(self, service_meta, client_id=None):
self.service_meta = service_meta
self.client_id = client_id
if not self.client_id:
self.client_id = gen_uuid()
if service_meta.serverimpl != "rabbitmq":
raise RuntimeError("serverimpl {} not implemented yet".format(
service_meta.serverimpl))
self._sender = RabbitmqClientSender(service_meta)
if service_meta.callmode != "notify":
# need a queue to receive response
self._receiver = RabbitmqClientReceiver(self.client_id)
else:
self._receiver = None
if service_meta.coder != "mail":
raise RuntimeError("coder {} is not supported yet".format(
service_meta.coder))
self.requestid_base = AtomicLong(0)
self.running = True
def gen_next_requestid(self):
while True:
result = self.requestid_base.value
if not self.requestid_base.compare_and_set(result, result + 1):
continue
return "{}_{}".format(self.client_id, result)
def start(self):
if self._receiver:
self._receiver.start_consuming()
def close(self):
self.running = False
if self._sender:
self._sender.close()
self._sender = None
if self._receiver:
self._receiver.close()
self._receiver = None
def send(self, request, key, block=True, timeout=10):
if not self.running:
raise RuntimeError("the service client is closed")
requestid = self.gen_next_requestid()
request_mail = Mail.new_mail(self.client_id, self.service_meta.name,
requestid)
populate_data_into_mail(request_mail, request)
expire = millis_now() + int(timeout * 1000)
request_mail.add_header("expire", str(expire + 5000))
countOfReplies = 0
if self._receiver:
countOfReplies = self.service_meta.options.get(
"servercardinality", 1)
try:
self._sender.send(request_mail.get_json(), key, block, timeout)
except Queue.Full:
# it's too busy in the send queue
raise
if not countOfReplies:
return True, None
receivedEvents = list()
while self.running and countOfReplies > len(
receivedEvents) and millis_now() < expire:
try:
content = self._receiver.get(True, 1)
e = self.process_mail(content)
receivedEvents.append(e)
except Queue.Empty:
time.sleep(0.1)
continue
if self.service_meta.callmode == "rpc":
# special treatment for one response
# no response
if len(receivedEvents) == 0:
return False, None
# remote exception
response = receivedEvents[0]
if isinstance(response, Exception):
raise response
# success
return True, response
else:
return len(receivedEvents) == countOfReplies, receivedEvents
def process_mails(self):
if not self._receiver:
return
while self.running:
try:
content = self._receiver.get(True, 1)
self.process_mail(content)
except Queue.Empty:
pass
self._receiver.close()
for content in self._receiver.dump_cache():
self.process_mail(content)
def process_mail(self, mail_injson):
if not mail_injson:
return
response_mail = Mail.from_json(mail_injson)
try:
response = extract_data_from_mail(response_mail)
except Exception as err:
response = err
return response
# useful handlers
def notify(self, request, key="", block=True, timeout=10):
assert self.service_meta.callmode == "notify"
self.send(request, key, block, timeout)
def rpc(self, request, key="", block=True, timeout=10):
assert self.service_meta.callmode == "rpc"
return self.send(request, key, block, timeout)
def mrpc(self, request, key="", block=True, timeout=10):
assert self.service_meta.callmode == "mrpc"
return self.send(request, key, block, timeout)
def __init__(self):
self._count = AtomicLong(0)
class Meter(object):
"""A meter measures the rate of events over time
(e.g., "requests per second").
In addition to the mean rate, you can also track 1, 5 and 15 minutes moving
averages ::
meter = Metrology.meter('requests')
meter.mark()
meter.count
"""
def __init__(self, average_class=EWMA):
self.counter = AtomicLong(0)
self.start_time = now()
self.last_tick = AtomicLong(self.start_time)
self.interval = EWMA.INTERVAL
self.m1_rate = EWMA.m1()
self.m5_rate = EWMA.m5()
self.m15_rate = EWMA.m15()
def _tick(self):
old_tick, new_tick = self.last_tick.value, time()
age = new_tick - old_tick
ticks = int(age / self.interval)
new_tick = old_tick + int(ticks * self.interval)
if ticks and self.last_tick.compare_and_swap(old_tick, new_tick):
for _ in range(ticks):
self.tick()
def __call__(self, *args, **kwargs):
if args and hasattr(args[0], '__call__'):
_orig_func = args[0]
def _decorator(*args, **kwargs):
with self:
return _orig_func(*args, **kwargs)
return _decorator
def __enter__(self):
pass
def __exit__(self, exc, exv, trace):
self.mark()
@property
def count(self):
"""Returns the total number of events that have been recorded."""
return self.counter.value
def clear(self):
self.counter.value = 0
self.start_time = time()
self.m1_rate.clear()
self.m5_rate.clear()
self.m15_rate.clear()
@ticker
def mark(self, value=1):
"""Record an event with the meter. By default it will record one event.
:param value: number of event to record
"""
self.counter += value
self.m1_rate.update(value)
self.m5_rate.update(value)
self.m15_rate.update(value)
def tick(self):
self.m1_rate.tick()
self.m5_rate.tick()
self.m15_rate.tick()
@property
@ticker
def one_minute_rate(self):
"""Returns the one-minute average rate."""
return self.m1_rate.rate
@property
@ticker
def five_minute_rate(self):
"""Returns the five-minute average rate."""
return self.m5_rate.rate
@property
@ticker
def fifteen_minute_rate(self):
"""Returns the fifteen-minute average rate."""
return self.m15_rate.rate
@property
def mean_rate(self):
"""
Returns the mean rate of the events since the start of the process.
"""
if self.counter.value == 0:
return 0.0
else:
elapsed = time() - self.start_time
return self.counter.value / elapsed
def stop(self):
pass
class Histogram(object):
"""
A histogram measures the statistical distribution of values in a stream of
data. In addition to minimum, maximum, mean, it also measures median,
75th, 90th, 95th, 98th, 99th, and 99.9th percentiles ::
histogram = Metrology.histogram('response-sizes')
histogram.update(len(response.content))
Metrology provides two types of histograms: uniform and exponentially
decaying.
"""
DEFAULT_SAMPLE_SIZE = 1028
DEFAULT_ALPHA = 0.015
def __init__(self, sample):
self.sample = sample
self.counter = AtomicLong(0)
self.minimum = AtomicLong(sys.maxsize)
self.maximum = AtomicLong(-sys.maxsize - 1)
self.sum = AtomicLong(0)
self.var = AtomicLongArray([-1, 0])
def clear(self):
self.sample.clear()
self.counter.value = 0
self.minimum.value = sys.maxsize
self.maximum.value = (-sys.maxsize - 1)
self.sum.value = 0
self.var.value = [-1, 0]
def update(self, value):
self.counter += 1
self.sample.update(value)
self.max = value
self.min = value
self.sum += value
self.update_variance(value)
@property
def snapshot(self):
return self.sample.snapshot()
@property
def count(self):
"""Return number of values."""
return self.counter.value
def get_max(self):
if self.counter.value > 0:
return self.maximum.value
return 0.0
def set_max(self, potential_max):
done = False
while not done:
current_max = self.maximum.value
done = (current_max is not None and current_max >= potential_max) \
or self.maximum.compare_and_swap(current_max, potential_max)
max = property(get_max, set_max, doc="""Returns the maximun value.""")
def get_min(self):
if self.counter.value > 0:
return self.minimum.value
return 0.0
def set_min(self, potential_min):
done = False
while not done:
current_min = self.minimum.value
done = (current_min is not None and current_min <= potential_min) \
or self.minimum.compare_and_swap(current_min, potential_min)
min = property(get_min, set_min, doc="""Returns the minimum value.""")
@property
def total(self):
"""Returns the total value."""
return self.sum.value
@property
def mean(self):
"""Returns the mean value."""
if self.counter.value > 0:
return self.sum.value / self.counter.value
return 0.0
@property
def stddev(self):
"""Returns the standard deviation."""
if self.counter.value > 0:
return self.variance ** .5
return 0.0
@property
def variance(self):
"""Returns variance"""
if self.counter.value <= 1:
return 0.0
return self.var.value[1] / (self.counter.value - 1)
def update_variance(self, value):
def variance(old_values):
if old_values[0] == -1:
new_values = (value, 0)
else:
old_m = old_values[0]
old_s = old_values[1]
new_m = old_m + ((value - old_m) / self.counter.value)
new_s = old_s + ((value - old_m) * (value - new_m))
new_values = (new_m, new_s)
return new_values
self.var.value = variance(self.var.value)
def test_swap(self):
atomic = AtomicLong(1000)
swapped = atomic.swap(1001)
self.assertEqual(1001, atomic.value)
self.assertEqual(1000, swapped)
class StopSignal(object):
stop_workers = AtomicLong(0)
def test_set_obj(self):
atomic = AtomicLongArray([-1, 0])
atomic[1] = AtomicLong(-2)
self.assertEqual(atomic[1], -2)
"""Hive signal handling."""
import logging
from signal import SIGINT
from atomic import AtomicLong
log = logging.getLogger(__name__)
EXCEPTION_THROWN = AtomicLong(0)
FINISH_SIGNAL_DURING_SYNC = AtomicLong(0)
def finish_signals_handler(signal, frame):
global FINISH_SIGNAL_DURING_SYNC
FINISH_SIGNAL_DURING_SYNC += 1
log.info("""
**********************************************************
CAUGHT {}. PLEASE WAIT... PROCESSING DATA IN QUEUES...
**********************************************************
""".format("SIGINT" if signal == SIGINT else "SIGTERM"))
def set_exception_thrown():
global EXCEPTION_THROWN
EXCEPTION_THROWN += 1
def can_continue_thread():
return EXCEPTION_THROWN.value == 0 and FINISH_SIGNAL_DURING_SYNC.value == 0
def test_init(self):
atomic = AtomicLong()
self.assertEqual(0, atomic.value)
atomic = AtomicLong(0)
self.assertEqual(0, atomic.value)
def __init__(self, reservoir_size):
self.counter = AtomicLong(0)
self.values = [0] * reservoir_size
def __init__(self, average_class=EWMA):
self.last = AtomicLong(0)
super(Derive, self).__init__(average_class)
def test_add(self):
atomic = AtomicLong(1000)
atomic += 1
self.assertEqual(1001, atomic.value)
class Histogram(object):
"""
A histogram measures the statistical distribution of values in a stream of data. In addition to minimum, maximum, mean, it also measures median, 75th, 90th, 95th, 98th, 99th, and 99.9th percentiles ::
histogram = Metrology.histogram('response-sizes')
histogram.update(len(response.content))
Metrology provides two types of histograms: uniform and exponentially decaying.
"""
DEFAULT_SAMPLE_SIZE = 1028
DEFAULT_ALPHA = 0.015
def __init__(self, sample):
self.sample = sample
self.counter = AtomicLong(0)
self.minimum = AtomicLong(sys.maxsize)
self.maximum = AtomicLong(-sys.maxsize - 1)
self.sum = AtomicLong(0)
self.var = AtomicLongArray([-1, 0])
def clear(self):
self.sample.clear()
self.counter.value = 0
self.minimum.value = sys.maxsize
self.maximum.value = (-sys.maxsize - 1)
self.sum.value = 0
self.var.value = [-1, 0]
def update(self, value):
self.counter += 1
self.sample.update(value)
self.max = value
self.min = value
self.sum += value
self.update_variance(value)
@property
def snapshot(self):
return self.sample.snapshot()
@property
def count(self):
"""Return number of values."""
return self.counter.value
def get_max(self):
if self.counter.value > 0:
return self.maximum.value
return 0.0
def set_max(self, potential_max):
done = False
while not done:
current_max = self.maximum.value
done = (current_max is not None and current_max >= potential_max) \
or self.maximum.compare_and_swap(current_max, potential_max)
max = property(get_max, set_max, doc="""Returns the maximun value.""")
def get_min(self):
if self.counter.value > 0:
return self.minimum.value
return 0.0
def set_min(self, potential_min):
done = False
while not done:
current_min = self.minimum.value
done = (current_min is not None and current_min <= potential_min) \
or self.minimum.compare_and_swap(current_min, potential_min)
min = property(get_min, set_min, doc="""Returns the minimum value.""")
@property
def mean(self):
"""Returns the mean value."""
if self.counter.value > 0:
return self.sum.value / self.counter.value
return 0.0
@property
def stddev(self):
"""Returns the standard deviation."""
if self.counter.value > 0:
return self.variance**.5
return 0.0
@property
def variance(self):
"""Returns variance"""
if self.counter.value <= 1:
return 0.0
return self.var.value[1] / (self.counter.value - 1)
def update_variance(self, value):
def variance(old_values):
if old_values[0] == -1:
new_values = (value, 0)
else:
old_m = old_values[0]
old_s = old_values[1]
new_m = old_m + ((value - old_m) / self.counter.value)
new_s = old_s + ((value - old_m) * (value - new_m))
new_values = (new_m, new_s)
return new_values
self.var.value = variance(self.var.value)
def test_sub(self):
atomic = AtomicLong(1000)
atomic -= 1
self.assertEqual(999, atomic.value)
def test_value(self):
atomic = AtomicLong(0)
atomic.value = 1
self.assertEqual(1, atomic.value)