def test_compare_and_swap(self):
atomic = AtomicLong(1000)
swapped = atomic.compare_and_swap(1000, 1001)
self.assertEqual(1001, atomic.value)
self.assertEqual(True, swapped)
swapped = atomic.compare_and_swap(1000, 1024)
self.assertEqual(1001, atomic.value)
self.assertEqual(False, swapped)
def test_compare_and_swap(self):
atomic = AtomicLong(1000)
swapped = atomic.compare_and_swap(1000, 1001)
self.assertEqual(1001, atomic.value)
self.assertEqual(True, swapped)
swapped = atomic.compare_and_swap(1000, 1024)
self.assertEqual(1001, atomic.value)
self.assertEqual(False, swapped)
class Meter(object):
"""A meter measures the rate of events over time
(e.g., "requests per second").
In addition to the mean rate, you can also track 1, 5 and 15 minutes moving
averages ::
meter = Metrology.meter('requests')
meter.mark()
meter.count
"""
def __init__(self, average_class=EWMA):
self.counter = AtomicLong(0)
self.start_time = now()
self.last_tick = AtomicLong(self.start_time)
self.interval = EWMA.INTERVAL
self.m1_rate = EWMA.m1()
self.m5_rate = EWMA.m5()
self.m15_rate = EWMA.m15()
def _tick(self):
old_tick, new_tick = self.last_tick.value, time()
age = new_tick - old_tick
ticks = int(age / self.interval)
new_tick = old_tick + int(ticks * self.interval)
if ticks and self.last_tick.compare_and_swap(old_tick, new_tick):
for _ in range(ticks):
self.tick()
def __call__(self, *args, **kwargs):
if args and hasattr(args[0], '__call__'):
_orig_func = args[0]
def _decorator(*args, **kwargs):
with self:
return _orig_func(*args, **kwargs)
return _decorator
def __enter__(self):
pass
def __exit__(self, exc, exv, trace):
self.mark()
@property
def count(self):
"""Returns the total number of events that have been recorded."""
return self.counter.value
def clear(self):
self.counter.value = 0
self.start_time = time()
self.m1_rate.clear()
self.m5_rate.clear()
self.m15_rate.clear()
@ticker
def mark(self, value=1):
"""Record an event with the meter. By default it will record one event.
:param value: number of event to record
"""
self.counter += value
self.m1_rate.update(value)
self.m5_rate.update(value)
self.m15_rate.update(value)
def tick(self):
self.m1_rate.tick()
self.m5_rate.tick()
self.m15_rate.tick()
@property
@ticker
def one_minute_rate(self):
"""Returns the one-minute average rate."""
return self.m1_rate.rate
@property
@ticker
def five_minute_rate(self):
"""Returns the five-minute average rate."""
return self.m5_rate.rate
@property
@ticker
def fifteen_minute_rate(self):
"""Returns the fifteen-minute average rate."""
return self.m15_rate.rate
@property
def mean_rate(self):
"""
Returns the mean rate of the events since the start of the process.
"""
if self.counter.value == 0:
return 0.0
else:
elapsed = time() - self.start_time
return self.counter.value / elapsed
def stop(self):
pass
class ExponentiallyDecayingSample(object):
def __init__(self, reservoir_size, alpha):
self.values = []
self.next_scale_time = AtomicLong(0)
self.alpha = alpha
self.reservoir_size = reservoir_size
self.lock = RLock()
self.rescale_threshold = \
ExponentiallyDecayingSample.calculate_rescale_threshold(alpha)
self.clear()
@staticmethod
def calculate_rescale_threshold(alpha):
# determine rescale-threshold such that we will not overflow exp() in
# weight function, and subsequently not overflow into inf on dividing
# by random.random()
min_rand = 1.0 / (2**32) # minimum non-zero value from random()
safety = 2.0 # safety pad for numerical inaccuracy
max_value = sys.float_info.max * min_rand / safety
return int(math.log(max_value) / alpha)
def clear(self):
with self.lock:
self.values = []
self.start_time = now()
self.next_scale_time.value = \
self.start_time + self.rescale_threshold
def size(self):
with self.lock:
return len(self.values)
def __len__(self):
return self.size()
def snapshot(self):
with self.lock:
return Snapshot(val for _, val in self.values)
def weight(self, timestamp):
return math.exp(self.alpha * (timestamp - self.start_time))
def rescale(self, now, next_time):
if self.next_scale_time.compare_and_swap(next_time,
now + self.rescale_threshold):
with self.lock:
rescaleFactor = math.exp(-self.alpha * (now - self.start_time))
self.values = [(k * rescaleFactor, v) for k, v in self.values]
self.start_time = now
def rescale_if_necessary(self):
time = now()
next_time = self.next_scale_time.value
if time > next_time:
self.rescale(time, next_time)
def update(self, value, timestamp=None):
if timestamp is None:
timestamp = now()
self.rescale_if_necessary()
with self.lock:
try:
priority = self.weight(timestamp) / random.random()
except (OverflowError, ZeroDivisionError):
priority = sys.float_info.max
if len(self.values) < self.reservoir_size:
heapq.heappush(self.values, (priority, value))
else:
heapq.heappushpop(self.values, (priority, value))
class ExponentiallyDecayingSample(object):
def __init__(self, reservoir_size, alpha):
self.values = []
self.next_scale_time = AtomicLong(0)
self.alpha = alpha
self.reservoir_size = reservoir_size
self.lock = RLock()
self.rescale_threshold = ExponentiallyDecayingSample.calculate_rescale_threshold(alpha)
self.clear()
@staticmethod
def calculate_rescale_threshold(alpha):
# determine rescale-threshold such that we will not overflow exp() in
# weight function, and subsequently not overflow into inf on dividing
# by random.random()
min_rand = 1.0 / (2 ** 32) # minimum non-zero value from random()
safety = 2.0 # safety pad for numerical inaccuracy
max_value = sys.float_info.max * min_rand / safety
return int(math.log(max_value) / alpha)
def clear(self):
with self.lock:
self.values = []
self.start_time = now()
self.next_scale_time.value = self.start_time + self.rescale_threshold
def size(self):
with self.lock:
return len(self.values)
def __len__(self):
return self.size()
def snapshot(self):
with self.lock:
return Snapshot(val for _, val in self.values)
def weight(self, timestamp):
return math.exp(self.alpha * (timestamp - self.start_time))
def rescale(self, now, next_time):
if self.next_scale_time.compare_and_swap(next_time, now + self.rescale_threshold):
with self.lock:
rescaleFactor = math.exp(-self.alpha * (now - self.start_time))
self.values = [(k * rescaleFactor, v) for k, v in self.values]
self.start_time = now
def rescale_if_necessary(self):
time = now()
next_time = self.next_scale_time.value
if time > next_time:
self.rescale(time, next_time)
def update(self, value, timestamp=None):
if timestamp is None:
timestamp = now()
self.rescale_if_necessary()
with self.lock:
try:
priority = self.weight(timestamp) / random.random()
except (OverflowError, ZeroDivisionError):
priority = sys.float_info.max
if len(self.values) < self.reservoir_size:
heapq.heappush(self.values, (priority, value))
else:
heapq.heappushpop(self.values, (priority, value))
class Meter(object):
"""A meter measures the rate of events over time
(e.g., "requests per second").
In addition to the mean rate, you can also track 1, 5 and 15 minutes moving
averages ::
meter = Metrology.meter('requests')
meter.mark()
meter.count
"""
def __init__(self, average_class=EWMA):
self.counter = AtomicLong(0)
self.start_time = now()
self.last_tick = AtomicLong(self.start_time)
self.interval = EWMA.INTERVAL
self.m1_rate = EWMA.m1()
self.m5_rate = EWMA.m5()
self.m15_rate = EWMA.m15()
def _tick(self):
old_tick, new_tick = self.last_tick.value, time()
age = new_tick - old_tick
ticks = int(age / self.interval)
new_tick = old_tick + int(ticks * self.interval)
if ticks and self.last_tick.compare_and_swap(old_tick, new_tick):
for _ in range(ticks):
self.tick()
def __call__(self, *args, **kwargs):
if args and hasattr(args[0], '__call__'):
_orig_func = args[0]
def _decorator(*args, **kwargs):
with self:
return _orig_func(*args, **kwargs)
return _decorator
def __enter__(self):
pass
def __exit__(self, exc, exv, trace):
self.mark()
@property
def count(self):
"""Returns the total number of events that have been recorded."""
return self.counter.value
def clear(self):
self.counter.value = 0
self.start_time = time()
self.m1_rate.clear()
self.m5_rate.clear()
self.m15_rate.clear()
@ticker
def mark(self, value=1):
"""Record an event with the meter. By default it will record one event.
:param value: number of event to record
"""
self.counter += value
self.m1_rate.update(value)
self.m5_rate.update(value)
self.m15_rate.update(value)
def tick(self):
self.m1_rate.tick()
self.m5_rate.tick()
self.m15_rate.tick()
@property
@ticker
def one_minute_rate(self):
"""Returns the one-minute average rate."""
return self.m1_rate.rate
@property
@ticker
def five_minute_rate(self):
"""Returns the five-minute average rate."""
return self.m5_rate.rate
@property
@ticker
def fifteen_minute_rate(self):
"""Returns the fifteen-minute average rate."""
return self.m15_rate.rate
@property
def mean_rate(self):
"""
Returns the mean rate of the events since the start of the process.
"""
if self.counter.value == 0:
return 0.0
else:
elapsed = time() - self.start_time
return self.counter.value / elapsed
def stop(self):
pass
class Histogram(object):
"""
A histogram measures the statistical distribution of values in a stream of
data. In addition to minimum, maximum, mean, it also measures median,
75th, 90th, 95th, 98th, 99th, and 99.9th percentiles ::
histogram = Metrology.histogram('response-sizes')
histogram.update(len(response.content))
Metrology provides two types of histograms: uniform and exponentially
decaying.
"""
DEFAULT_SAMPLE_SIZE = 1028
DEFAULT_ALPHA = 0.015
def __init__(self, sample):
self.sample = sample
self.counter = AtomicLong(0)
self.minimum = AtomicLong(sys.maxsize)
self.maximum = AtomicLong(-sys.maxsize - 1)
self.sum = AtomicLong(0)
self.var = AtomicLongArray([-1, 0])
def clear(self):
self.sample.clear()
self.counter.value = 0
self.minimum.value = sys.maxsize
self.maximum.value = (-sys.maxsize - 1)
self.sum.value = 0
self.var.value = [-1, 0]
def update(self, value):
self.counter += 1
self.sample.update(value)
self.max = value
self.min = value
self.sum += value
self.update_variance(value)
@property
def snapshot(self):
return self.sample.snapshot()
@property
def count(self):
"""Return number of values."""
return self.counter.value
def get_max(self):
if self.counter.value > 0:
return self.maximum.value
return 0.0
def set_max(self, potential_max):
done = False
while not done:
current_max = self.maximum.value
done = (current_max is not None and current_max >= potential_max) \
or self.maximum.compare_and_swap(current_max, potential_max)
max = property(get_max, set_max, doc="""Returns the maximun value.""")
def get_min(self):
if self.counter.value > 0:
return self.minimum.value
return 0.0
def set_min(self, potential_min):
done = False
while not done:
current_min = self.minimum.value
done = (current_min is not None and current_min <= potential_min) \
or self.minimum.compare_and_swap(current_min, potential_min)
min = property(get_min, set_min, doc="""Returns the minimum value.""")
@property
def total(self):
"""Returns the total value."""
return self.sum.value
@property
def mean(self):
"""Returns the mean value."""
if self.counter.value > 0:
return self.sum.value / self.counter.value
return 0.0
@property
def stddev(self):
"""Returns the standard deviation."""
if self.counter.value > 0:
return self.variance ** .5
return 0.0
@property
def variance(self):
"""Returns variance"""
if self.counter.value <= 1:
return 0.0
return self.var.value[1] / (self.counter.value - 1)
def update_variance(self, value):
def variance(old_values):
if old_values[0] == -1:
new_values = (value, 0)
else:
old_m = old_values[0]
old_s = old_values[1]
new_m = old_m + ((value - old_m) / self.counter.value)
new_s = old_s + ((value - old_m) * (value - new_m))
new_values = (new_m, new_s)
return new_values
self.var.value = variance(self.var.value)
class Histogram(object):
"""
A histogram measures the statistical distribution of values in a stream of data. In addition to minimum, maximum, mean, it also measures median, 75th, 90th, 95th, 98th, 99th, and 99.9th percentiles ::
histogram = Metrology.histogram('response-sizes')
histogram.update(len(response.content))
Metrology provides two types of histograms: uniform and exponentially decaying.
"""
DEFAULT_SAMPLE_SIZE = 1028
DEFAULT_ALPHA = 0.015
def __init__(self, sample):
self.sample = sample
self.counter = AtomicLong(0)
self.minimum = AtomicLong(sys.maxsize)
self.maximum = AtomicLong(-sys.maxsize - 1)
self.sum = AtomicLong(0)
self.var = AtomicLongArray([-1, 0])
def clear(self):
self.sample.clear()
self.counter.value = 0
self.minimum.value = sys.maxsize
self.maximum.value = (-sys.maxsize - 1)
self.sum.value = 0
self.var.value = [-1, 0]
def update(self, value):
self.counter += 1
self.sample.update(value)
self.max = value
self.min = value
self.sum += value
self.update_variance(value)
@property
def snapshot(self):
return self.sample.snapshot()
@property
def count(self):
"""Return number of values."""
return self.counter.value
def get_max(self):
if self.counter.value > 0:
return self.maximum.value
return 0.0
def set_max(self, potential_max):
done = False
while not done:
current_max = self.maximum.value
done = (current_max is not None and current_max >= potential_max) \
or self.maximum.compare_and_swap(current_max, potential_max)
max = property(get_max, set_max, doc="""Returns the maximun value.""")
def get_min(self):
if self.counter.value > 0:
return self.minimum.value
return 0.0
def set_min(self, potential_min):
done = False
while not done:
current_min = self.minimum.value
done = (current_min is not None and current_min <= potential_min) \
or self.minimum.compare_and_swap(current_min, potential_min)
min = property(get_min, set_min, doc="""Returns the minimum value.""")
@property
def mean(self):
"""Returns the mean value."""
if self.counter.value > 0:
return self.sum.value / self.counter.value
return 0.0
@property
def stddev(self):
"""Returns the standard deviation."""
if self.counter.value > 0:
return self.variance**.5
return 0.0
@property
def variance(self):
"""Returns variance"""
if self.counter.value <= 1:
return 0.0
return self.var.value[1] / (self.counter.value - 1)
def update_variance(self, value):
def variance(old_values):
if old_values[0] == -1:
new_values = (value, 0)
else:
old_m = old_values[0]
old_s = old_values[1]
new_m = old_m + ((value - old_m) / self.counter.value)
new_s = old_s + ((value - old_m) * (value - new_m))
new_values = (new_m, new_s)
return new_values
self.var.value = variance(self.var.value)