class Histogram(object):
"""
A histogram measures the statistical distribution of values in a stream of data. In addition to minimum, maximum, mean, it also measures median, 75th, 90th, 95th, 98th, 99th, and 99.9th percentiles ::
histogram = Metrology.histogram('response-sizes')
histogram.update(len(response.content))
Metrology provides two types of histograms: uniform and exponentially decaying.
"""
DEFAULT_SAMPLE_SIZE = 1028
DEFAULT_ALPHA = 0.015
def __init__(self, sample):
self.sample = sample
self.counter = Atomic(0)
self.minimum = Atomic()
self.maximum = Atomic()
self.sum = Atomic(0)
self.var = Atomic([-1, 0])
def clear(self):
self.sample.clear()
self.counter.value = 0
self.minimum.value = None
self.maximum.value = None
self.sum.value = 0
self.var.value = [-1, 0]
def update(self, value):
self.counter.update(lambda v: v + 1)
self.sample.update(value)
self.max = value
self.min = value
self.sum.update(lambda v: v + value)
self.update_variance(value)
@property
def snapshot(self):
return self.sample.snapshot()
@property
def count(self):
"""Return number of values."""
return self.counter.value
def get_max(self):
if self.counter.value > 0:
return self.maximum.value
return 0.0
def set_max(self, potential_max):
done = False
while not done:
current_max = self.maximum.value
done = (current_max is not None and current_max >= potential_max) \
or self.maximum.compare_and_swap(current_max, potential_max)
max = property(get_max, set_max, doc="""Returns the maximun value.""")
def get_min(self):
if self.counter.value > 0:
return self.minimum.value
return 0.0
def set_min(self, potential_min):
done = False
while not done:
current_min = self.minimum.value
done = (current_min is not None and current_min <= potential_min) \
or self.minimum.compare_and_swap(current_min, potential_min)
min = property(get_min, set_min, doc="""Returns the minimum value.""")
@property
def mean(self):
"""Returns the mean value."""
if self.counter.value > 0:
return self.sum.value / self.counter.value
return 0.0
@property
def stddev(self):
"""Returns the standard deviation."""
if self.counter.value > 0:
return self.variance
return 0.0
@property
def variance(self):
"""Returns variance"""
if self.counter.value <= 1:
return 0.0
return self.var.value[1] / (self.counter.value - 1)
def update_variance(self, value):
def variance(old_values):
if old_values[0] == -1:
new_values = (value, 0)
else:
old_m = old_values[0]
old_s = old_values[1]
new_m = old_m + ((value - old_m) / self.counter.value)
new_s = old_s + ((value - old_m) * (value - new_m))
new_values = (new_m, new_s)
return new_values
self.var.update(variance)
class ExponentiallyDecayingSample(object):
RESCALE_THRESHOLD = 60 * 60
def __init__(self, reservoir_size, alpha):
self.values = RBTree()
self.counter = Atomic(0)
self.next_scale_time = Atomic(0)
self.alpha = alpha
self.reservoir_size = reservoir_size
self.lock = RLock()
self.clear()
def clear(self):
with self.lock:
self.values.clear()
self.counter.value = 0
self.next_scale_time.value = time() + self.RESCALE_THRESHOLD
self.start_time = time()
def size(self):
count = self.counter.value
if count < self.reservoir_size:
return count
return self.reservoir_size
def __len__(self):
return self.size()
def snapshot(self):
with self.lock:
return Snapshot(list(self.values.values()))
def weight(self, timestamp):
return math.exp(self.alpha * timestamp)
def rescale(self, now, next_time):
if self.next_scale_time.compare_and_swap(next_time, now + self.RESCALE_THRESHOLD):
with self.lock:
old_start_time = self.start_time
self.start_time = time()
for key in list(self.values.keys()):
value = self.values.remove(key)
self.values[key * math.exp(-self.alpha * (self.start_time - old_start_time))] = value
def update(self, value, timestamp=None):
if not timestamp:
timestamp = time()
with self.lock:
try:
priority = self.weight(timestamp - self.start_time) / random.random()
except OverflowError:
priority = sys.float_info.max
new_count = self.counter.update(lambda v: v + 1)
if math.isnan(priority):
return
if new_count <= self.reservoir_size:
self.values[priority] = value
else:
first_priority = self.values.root.key
if first_priority < priority:
if priority in self.values:
self.values[priority] = value
if not self.values.remove(first_priority):
first_priority = self.values.root()
class Meter(object):
"""A meter measures the rate of events over time (e.g., "requests per second").
In addition to the mean rate, you can also track 1, 5 and 15 minutes moving averages ::
meter = Metrology.meter('requests')
meter.mark()
meter.count
"""
def __init__(self, average_class=EWMA):
self.counter = Atomic(0)
self.start_time = time()
self.last_tick = Atomic(self.start_time)
self.interval = EWMA.INTERVAL
self.m1_rate = EWMA.m1()
self.m5_rate = EWMA.m5()
self.m15_rate = EWMA.m15()
def _tick(self):
old_tick, new_tick = self.last_tick.value, time()
age = new_tick - old_tick
ticks = int(age / self.interval)
new_tick = old_tick + (ticks * self.interval)
if ticks and self.last_tick.compare_and_swap(old_tick, new_tick):
for _ in range(ticks):
self.tick()
@property
def count(self):
"""Returns the total number of events that have been recorded."""
return self.counter.value
def clear(self):
self.counter.value = 0
self.start_time = time()
self.m1_rate.clear()
self.m5_rate.clear()
self.m15_rate.clear()
@ticker
def mark(self, value=1):
"""Record an event with the meter. By default it will record one event.
:param value: number of event to record
"""
self.counter.update(lambda v: v + value)
self.m1_rate.update(value)
self.m5_rate.update(value)
self.m15_rate.update(value)
def tick(self):
self.m1_rate.tick()
self.m5_rate.tick()
self.m15_rate.tick()
@property
@ticker
def one_minute_rate(self):
"""Returns the one-minute average rate."""
return self.m1_rate.rate
@property
@ticker
def five_minute_rate(self):
"""Returns the five-minute average rate."""
return self.m5_rate.rate
@property
@ticker
def fifteen_minute_rate(self):
"""Returns the fifteen-minute average rate."""
return self.m15_rate.rate
@property
def mean_rate(self):
"""Returns the mean rate of the events since the start of the process."""
if self.counter.value == 0:
return 0.0
else:
elapsed = time() - self.start_time
return self.counter.value / elapsed
def stop(self):
pass
class ExponentiallyDecayingSample(object):
def __init__(self, reservoir_size, alpha):
self.values = []
self.next_scale_time = Atomic(0)
self.alpha = alpha
self.reservoir_size = reservoir_size
self.lock = RLock()
self.rescale_threshold = ExponentiallyDecayingSample.calculate_rescale_threshold(alpha)
self.clear()
@staticmethod
def calculate_rescale_threshold(alpha):
# determine rescale-threshold such that we will not overflow exp() in
# weight function, and subsequently not overflow into inf on dividing
# by random.random()
min_rand = 1.0 / (2**32) # minimum non-zero value from random()
safety = 2.0 # safety pad for numerical inaccuracy
max_value = sys.float_info.max * min_rand / safety
return math.log(max_value) / alpha
def clear(self):
with self.lock:
self.values = []
self.start_time = time()
self.next_scale_time.value = self.start_time + self.rescale_threshold
def size(self):
with self.lock:
return len(self.values)
def __len__(self):
return self.size()
def snapshot(self):
with self.lock:
return Snapshot(val for _, val in self.values)
def weight(self, timestamp):
return math.exp(self.alpha * (timestamp - self.start_time))
def rescale(self, now, next_time):
if self.next_scale_time.compare_and_swap(next_time, now + self.rescale_threshold):
with self.lock:
rescaleFactor = math.exp(-self.alpha * (now - self.start_time))
self.values = [(k * rescaleFactor, v) for k, v in self.values]
self.start_time = now
def rescale_if_necessary(self):
now = time()
next_time = self.next_scale_time.get_value()
if now > next_time:
self.rescale(now, next_time)
def update(self, value, timestamp=None):
if timestamp is None:
timestamp = time()
self.rescale_if_necessary()
with self.lock:
try:
priority = self.weight(timestamp) / random.random()
except (OverflowError, ZeroDivisionError):
priority = sys.float_info.max
if len(self.values) < self.reservoir_size:
heapq.heappush(self.values, (priority, value))
else:
heapq.heappushpop(self.values, (priority, value))
class Meter(object):
"""A meter measures the rate of events over time (e.g., "requests per second").
In addition to the mean rate, you can also track 1, 5 and 15 minutes moving averages ::
meter = Metrology.meter('requests')
meter.mark()
meter.count
"""
def __init__(self, average_class=EWMA):
self.counter = Atomic(0)
self.start_time = time()
self.last_tick = Atomic(self.start_time)
self.interval = EWMA.INTERVAL
self.m1_rate = EWMA.m1()
self.m5_rate = EWMA.m5()
self.m15_rate = EWMA.m15()
def _tick(self):
old_tick, new_tick = self.last_tick.value, time()
age = new_tick - old_tick
ticks = int(age / self.interval)
new_tick = old_tick + (ticks * self.interval)
if ticks and self.last_tick.compare_and_swap(old_tick, new_tick):
for _ in range(ticks):
self.tick()
@property
def count(self):
"""Returns the total number of events that have been recorded."""
return self.counter.value
def clear(self):
self.counter.value = 0
self.start_time = time()
self.m1_rate.clear()
self.m5_rate.clear()
self.m15_rate.clear()
@ticker
def mark(self, value=1):
"""Record an event with the meter. By default it will record one event.
:param value: number of event to record
"""
self.counter.update(lambda v: v + value)
self.m1_rate.update(value)
self.m5_rate.update(value)
self.m15_rate.update(value)
def tick(self):
self.m1_rate.tick()
self.m5_rate.tick()
self.m15_rate.tick()
@property
@ticker
def one_minute_rate(self):
"""Returns the one-minute average rate."""
return self.m1_rate.rate
@property
@ticker
def five_minute_rate(self):
"""Returns the five-minute average rate."""
return self.m5_rate.rate
@property
@ticker
def fifteen_minute_rate(self):
"""Returns the fifteen-minute average rate."""
return self.m15_rate.rate
@property
def mean_rate(self):
"""Returns the mean rate of the events since the start of the process."""
if self.counter.value == 0:
return 0.0
else:
elapsed = time() - self.start_time
return self.counter.value / elapsed
def stop(self):
pass
class Histogram(object):
"""
A histogram measures the statistical distribution of values in a stream of data. In addition to minimum, maximum, mean, it also measures median, 75th, 90th, 95th, 98th, 99th, and 99.9th percentiles ::
histogram = Metrology.histogram('response-sizes')
histogram.update(len(response.content))
Metrology provides two types of histograms: uniform and exponentially decaying.
"""
DEFAULT_SAMPLE_SIZE = 1028
DEFAULT_ALPHA = 0.015
def __init__(self, sample):
self.sample = sample
self.counter = Atomic(0)
self.minimum = Atomic()
self.maximum = Atomic()
self.sum = Atomic(0)
self.var = Atomic([-1, 0])
def clear(self):
self.sample.clear()
self.counter.value = 0
self.minimum.value = None
self.maximum.value = None
self.sum.value = 0
self.var.value = [-1, 0]
def update(self, value):
self.counter.update(lambda v: v + 1)
self.sample.update(value)
self.max = value
self.min = value
self.sum.update(lambda v: v + value)
self.update_variance(value)
@property
def snapshot(self):
return self.sample.snapshot()
@property
def count(self):
"""Return number of values."""
return self.counter.value
def get_max(self):
if self.counter.value > 0:
return self.maximum.value
return 0.0
def set_max(self, potential_max):
done = False
while not done:
current_max = self.maximum.value
done = (current_max is not None and current_max >= potential_max) \
or self.maximum.compare_and_swap(current_max, potential_max)
max = property(get_max, set_max, doc="""Returns the maximun value.""")
def get_min(self):
if self.counter.value > 0:
return self.minimum.value
return 0.0
def set_min(self, potential_min):
done = False
while not done:
current_min = self.minimum.value
done = (current_min is not None and current_min <= potential_min) \
or self.minimum.compare_and_swap(current_min, potential_min)
min = property(get_min, set_min, doc="""Returns the minimum value.""")
@property
def mean(self):
"""Returns the mean value."""
if self.counter.value > 0:
return self.sum.value / self.counter.value
return 0.0
@property
def stddev(self):
"""Returns the standard deviation."""
if self.counter.value > 0:
return self.variance**.5
return 0.0
@property
def variance(self):
"""Returns variance"""
if self.counter.value <= 1:
return 0.0
return self.var.value[1] / (self.counter.value - 1)
def update_variance(self, value):
def variance(old_values):
if old_values[0] == -1:
new_values = (value, 0)
else:
old_m = old_values[0]
old_s = old_values[1]
new_m = old_m + ((value - old_m) / self.counter.value)
new_s = old_s + ((value - old_m) * (value - new_m))
new_values = (new_m, new_s)
return new_values
self.var.update(variance)
class Histogram(object):
DEFAULT_SAMPLE_SIZE = 1028
DEFAULT_ALPHA = 0.015
def __init__(self, sample):
self.sample = sample
self.counter = Atomic(0)
self.minimum = Atomic()
self.maximum = Atomic()
self.sum = Atomic(0)
self.var = Atomic([-1, 0])
def clear(self):
self.sample.clear()
self.counter.value = 0
self.minimum.value = None
self.maximum.value = None
self.sum.value = 0
self.var.value = [-1, 0]
def update(self, value):
with self.counter:
self.counter.value += 1
self.sample.update(value)
self.max = value
self.min = value
with self.sum:
self.sum.value += value
self.update_variance(value)
@property
def snapshot(self):
return self.sample.snapshot()
@property
def count(self):
return self.counter.value
def get_max(self):
if self.counter.value > 0:
return self.maximum.value
return 0.0
def set_max(self, potential_max):
done = False
while not done:
current_max = self.maximum.value
done = (current_max is not None and current_max >= potential_max) or self.maximum.compare_and_swap(
current_max, potential_max
)
max = property(get_max, set_max)
def get_min(self):
if self.counter.value > 0:
return self.minimum.value
return 0.0
def set_min(self, potential_min):
done = False
while not done:
current_min = self.minimum.value
done = (current_min is not None and current_min <= potential_min) or self.minimum.compare_and_swap(
current_min, potential_min
)
min = property(get_min, set_min)
@property
def mean(self):
if self.counter.value > 0:
return self.sum.value / self.counter.value
return 0.0
@property
def stddev(self):
if self.counter.value > 0:
return self.var.value
return 0.0
@property
def variance(self):
if self.counter.value <= 1:
return 0.0
return self.var.value[1] / (self.counter.value - 1)
def update_variance(self, value):
with self.var:
old_values = self.var.value
if old_values[0] == -1:
new_values = (value, 0)
else:
old_m = old_values[0]
old_s = old_values[1]
new_m = old_m + ((value - old_m) / self.counter.value)
new_s = old_s + ((value - old_m) * (value - new_m))
new_values = (new_m, new_s)
self.var.value = new_values
return new_values
class ExponentiallyDecayingSample(object):
RESCALE_THRESHOLD = 60 * 60
def __init__(self, reservoir_size, alpha):
self.values = RBTree()
self.counter = Atomic(0)
self.next_scale_time = Atomic(0)
self.alpha = alpha
self.reservoir_size = reservoir_size
self.lock = RLock()
self.clear()
def clear(self):
with self.lock:
self.values.clear()
self.counter.value = 0
self.next_scale_time.value = time() + self.RESCALE_THRESHOLD
self.start_time = time()
def size(self):
count = self.counter.value
if count < self.reservoir_size:
return count
return self.reservoir_size
def __len__(self):
return self.size()
def snapshot(self):
with self.lock:
return Snapshot(list(self.values.values()))
def weight(self, timestamp):
return math.exp(self.alpha * timestamp)
def rescale(self, now, next_time):
if self.next_scale_time.compare_and_swap(next_time,
now + self.RESCALE_THRESHOLD):
with self.lock:
old_start_time = self.start_time
self.start_time = time()
for key in list(self.values.keys()):
value = self.values.remove(key)
self.values[key * math.exp(
-self.alpha * (self.start_time - old_start_time))] = value
def update(self, value, timestamp=None):
if not timestamp:
timestamp = time()
with self.lock:
try:
priority = self.weight(timestamp -
self.start_time) / random.random()
except OverflowError:
priority = sys.float_info.max
new_count = self.counter.update(lambda v: v + 1)
if math.isnan(priority):
return
if new_count <= self.reservoir_size:
self.values[priority] = value
else:
first_priority = self.values.root.key
if first_priority < priority:
if priority in self.values:
self.values[priority] = value
if not self.values.remove(first_priority):
first_priority = self.values.root()