def __init__(self, sample):
self.sample = sample
self.counter = Atomic(0)
self.minimum = Atomic()
self.maximum = Atomic()
self.sum = Atomic(0)
self.var = Atomic([-1, 0])
def __init__(self, sample):
self.sample = sample
self.counter = Atomic(0)
self.minimum = Atomic()
self.maximum = Atomic()
self.sum = Atomic(0)
self.var = Atomic([-1, 0])
def test_complex_update(self):
def complex_update(v):
return [v + 1 for v in v]
atomic = Atomic([-1, 0])
value = atomic.update(complex_update)
self.assertEqual([0, 1], atomic.value)
self.assertEqual([0, 1], value)
class Counter(object):
"""
A counter is like a gauge, but you can increment or decrement its value ::
counter = Metrology.counter('pending-jobs')
counter.increment()
counter.decrement()
counter.count
"""
def __init__(self):
self._count = Atomic(0)
def increment(self, value=1):
"""Increment the counter. By default it will increment by 1.
:param value: value to increment the counter.
"""
self._count.update(lambda v: v + value)
def decrement(self, value=1):
"""Decrement the counter. By default it will decrement by 1.
:param value: value to decrement the counter.
"""
self._count.update(lambda v: v - value)
def clear(self):
self._count.value = 0
@property
def count(self):
"""Return the current value of the counter."""
return self._count.value
def __init__(self, alpha, interval):
self.alpha = alpha
self.interval = interval
self.initialized = False
self._rate = 0.0
self._uncounted = Atomic(0)
def __init__(self, reservoir_size, alpha):
self.values = RBTree()
self.counter = Atomic(0)
self.next_scale_time = Atomic(0)
self.alpha = alpha
self.reservoir_size = reservoir_size
self.lock = RLock()
self.clear()
def test_complex_update(self):
def complex_update(v):
return [v + 1 for v in v]
atomic = Atomic([-1, 0])
value = atomic.update(complex_update)
self.assertEqual([0, 1], atomic.value)
self.assertEqual([0, 1], value)
def __init__(self, reservoir_size, alpha):
self.values = RBTree()
self.counter = Atomic(0)
self.next_scale_time = Atomic(0)
self.alpha = alpha
self.reservoir_size = reservoir_size
self.lock = RLock()
self.clear()
def __init__(self, reservoir_size, alpha):
self.values = []
self.next_scale_time = Atomic(0)
self.alpha = alpha
self.reservoir_size = reservoir_size
self.lock = RLock()
self.rescale_threshold = ExponentiallyDecayingSample.calculate_rescale_threshold(alpha)
self.clear()
def __init__(self, average_class=EWMA):
self.counter = Atomic(0)
self.start_time = time()
self.last_tick = Atomic(self.start_time)
self.interval = EWMA.INTERVAL
self.m1_rate = EWMA.m1()
self.m5_rate = EWMA.m5()
self.m15_rate = EWMA.m15()
def __init__(self, average_class=EWMA):
self.counter = Atomic(0)
self.start_time = time()
self.last_tick = Atomic(self.start_time)
self.interval = EWMA.INTERVAL
self.m1_rate = EWMA.m1()
self.m5_rate = EWMA.m5()
self.m15_rate = EWMA.m15()
class EWMA(object):
INTERVAL = 5.0
SECONDS_PER_MINUTE = 60.0
ONE_MINUTE = 1
FIVE_MINUTES = 5
FIFTEEN_MINUTES = 15
M1_ALPHA = 1 - math.exp(-INTERVAL / SECONDS_PER_MINUTE / ONE_MINUTE)
M5_ALPHA = 1 - math.exp(-INTERVAL / SECONDS_PER_MINUTE / FIVE_MINUTES)
M15_ALPHA = 1 - math.exp(-INTERVAL / SECONDS_PER_MINUTE / FIFTEEN_MINUTES)
@classmethod
def m1(cls):
return EWMA(cls.M1_ALPHA, cls.INTERVAL)
@classmethod
def m5(cls):
return EWMA(cls.M5_ALPHA, cls.INTERVAL)
@classmethod
def m15(cls):
return EWMA(cls.M15_ALPHA, cls.INTERVAL)
def __init__(self, alpha, interval):
self.alpha = alpha
self.interval = interval
self.initialized = False
self._rate = 0.0
self._uncounted = Atomic(0)
def clear(self):
self.initialized = False
self._rate = 0.0
self._uncounted.value = 0
def update(self, value):
self._uncounted.update(lambda v: v + value)
def tick(self):
count = self._uncounted.swap(0)
instant_rate = count / self.interval
if self.initialized:
self._rate += self.alpha * (instant_rate - self._rate)
else:
self._rate = instant_rate
self.initialized = True
@property
def rate(self):
return self._rate
def set_args(self, args):
Atomic.set_args(self, args)
if 'graph' in self.args and self.args.graph:
self.graphdir = self.args.graph
else:
if os.path.exists("/var/lib/docker") and os.path.exists("/var/lib/docker-latest"):
raise ValueError("You must specify the --graph storage path to reset /var/lib/docker or /var/lib/docker-latest")
if os.path.exists("/var/lib/docker"):
self.graphdir = "/var/lib/docker"
else:
if os.path.exists("/var/lib/docker-latest"):
self.graphdir = "/var/lib/docker-latest"
else:
raise ValueError("Could not find any default graph storage path. Specify one using --graph option")
class UniformSample(object):
def __init__(self, reservoir_size):
self.counter = Atomic(0)
self.values = [0] * reservoir_size
def clear(self):
self.values = [0] * len(self.values)
self.counter.value = 0
def size(self):
count = self.counter.value
if count > len(self.values):
return len(self.values)
return count
def __len__(self):
return self.size
def snapshot(self):
return Snapshot(self.values[0:self.size()])
def update(self, value):
new_count = self.counter.update(lambda v: v + 1)
if new_count <= len(self.values):
self.values[new_count - 1] = value
else:
index = random.uniform(0, new_count)
if index < len(self.values):
self.values[int(index)] = value
class UniformSample(object):
def __init__(self, reservoir_size):
self.counter = Atomic(0)
self.values = [0] * reservoir_size
def clear(self):
self.values = [0] * len(self.values)
self.counter.value = 0
def size(self):
count = self.counter.value
if count > len(self.values):
return len(self.values)
return count
def __len__(self):
return self.size
def snapshot(self):
return Snapshot(self.values[0:self.size()])
def update(self, value):
new_count = self.counter.update(lambda v: v + 1)
if new_count <= len(self.values):
self.values[new_count - 1] = value
else:
index = random.uniform(0, new_count)
if index < len(self.values):
self.values[int(index)] = value
class UniqueIdSupplier(Supplier):
def __init__(self,
scheme):
super(UniqueIdSupplier, self).__init__()
self._scheme = scheme
self._id_count = Atomic(0)
def get(self):
new_id = self._id_count.update(lambda v: v + 1)
return UniqueId.of(self._scheme, str(new_id))
def get_with_value_prefix(self, value_prefix):
new_id = self._id_count.update(lambda v: v + 1)
return UniqueId.of(self._scheme, value_prefix + str(new_id))
def __str__(self):
return 'UniqueIdSupplier[' + self._scheme + ']'
class ToggleGauge(Gauge):
_value = Atomic(1)
@property
def value(self):
try:
return self._value.value
finally:
self._value.value = 0
def set_args(self, args):
Atomic.set_args(self, args)
if 'graph' in self.args and self.args.graph:
self.graphdir = self.args.graph
else:
if os.path.exists("/var/lib/docker") and os.path.exists(
"/var/lib/docker-latest"):
raise ValueError(
"You must specify the --graph storage path to reset /var/lib/docker or /var/lib/docker-latest"
)
if os.path.exists("/var/lib/docker"):
self.graphdir = "/var/lib/docker"
else:
if os.path.exists("/var/lib/docker-latest"):
self.graphdir = "/var/lib/docker-latest"
else:
raise ValueError(
"Could not find any default graph storage path. Specify one using --graph option"
)
def __init__(self, average_class=EWMA):
self.counter = Atomic(0)
self.start_time = time()
self.m1_rate = EWMA.m1()
self.m5_rate = EWMA.m5()
self.m15_rate = EWMA.m15()
self.task = PeriodicTask(interval=average_class.INTERVAL,
target=self.tick)
self.task.start()
class ObjectIdSupplier(Supplier):
"""
A supplier of object identifiers.
An object identifier consists of a scheme and value.
This class creates object identifiers for a fixed scheme name, where each
value is an incrementing number. The values are created in a thread-safe way.
This class is thread-safe and not externally mutable.
"""
def __init__(self,
scheme):
self._scheme = scheme
self._id_count = Atomic(0)
def get(self):
new_id = self._id_count.update(lambda v: v + 1)
return ObjectId.of(self._scheme, str(new_id))
def get_with_value_prefix(self, value_prefix):
new_id = self._id_count.update(lambda v: v + 1)
return ObjectId.of(self._scheme, value_prefix + str(new_id))
def __str__(self):
return 'ObjectIdSupplier[' + self._scheme + ']'
class Derive(Meter):
"""
A derive is like a meter but accepts an absolute counter as input.
derive = Metrology.derive('network.io')
derive.mark()
derive.count
"""
def __init__(self, average_class=EWMA):
self.last = Atomic(0)
super(Derive, self).__init__(average_class)
def mark(self, value=1):
"""Record an event with the derive.
:param value: counter value to record
"""
last = self.last.get_and_set(value)
if last <= value:
value = value - last
super(Derive, self).mark(value)
def test_swap(self):
atomic = Atomic(1000)
swapped = atomic.swap(1001)
self.assertEqual(1001, atomic.value)
self.assertEqual(1000, swapped)
class Histogram(object):
DEFAULT_SAMPLE_SIZE = 1028
DEFAULT_ALPHA = 0.015
def __init__(self, sample):
self.sample = sample
self.counter = Atomic(0)
self.minimum = Atomic()
self.maximum = Atomic()
self.sum = Atomic(0)
self.var = Atomic([-1, 0])
def clear(self):
self.sample.clear()
self.counter.value = 0
self.minimum.value = None
self.maximum.value = None
self.sum.value = 0
self.var.value = [-1, 0]
def update(self, value):
with self.counter:
self.counter.value += 1
self.sample.update(value)
self.max = value
self.min = value
with self.sum:
self.sum.value += value
self.update_variance(value)
@property
def snapshot(self):
return self.sample.snapshot()
@property
def count(self):
return self.counter.value
def get_max(self):
if self.counter.value > 0:
return self.maximum.value
return 0.0
def set_max(self, potential_max):
done = False
while not done:
current_max = self.maximum.value
done = (current_max is not None and current_max >= potential_max) or self.maximum.compare_and_swap(
current_max, potential_max
)
max = property(get_max, set_max)
def get_min(self):
if self.counter.value > 0:
return self.minimum.value
return 0.0
def set_min(self, potential_min):
done = False
while not done:
current_min = self.minimum.value
done = (current_min is not None and current_min <= potential_min) or self.minimum.compare_and_swap(
current_min, potential_min
)
min = property(get_min, set_min)
@property
def mean(self):
if self.counter.value > 0:
return self.sum.value / self.counter.value
return 0.0
@property
def stddev(self):
if self.counter.value > 0:
return self.var.value
return 0.0
@property
def variance(self):
if self.counter.value <= 1:
return 0.0
return self.var.value[1] / (self.counter.value - 1)
def update_variance(self, value):
with self.var:
old_values = self.var.value
if old_values[0] == -1:
new_values = (value, 0)
else:
old_m = old_values[0]
old_s = old_values[1]
new_m = old_m + ((value - old_m) / self.counter.value)
new_s = old_s + ((value - old_m) * (value - new_m))
new_values = (new_m, new_s)
self.var.value = new_values
return new_values
def test_init(self):
atomic = Atomic()
self.assertEqual(None, atomic.value)
atomic = Atomic(0)
self.assertEqual(0, atomic.value)
def __init__(self):
self._count = Atomic(0)
def test_complex_value(self):
atomic = Atomic([-1, 0])
self.assertEqual([-1, 0], atomic.value)
def __init__(self,
scheme):
self._scheme = scheme
self._id_count = Atomic(0)
def test_swap(self):
atomic = Atomic(1000)
swapped = atomic.swap(1001)
self.assertEqual(1001, atomic.value)
self.assertEqual(1000, swapped)
def test_try_update(self):
atomic = Atomic(1000)
value = atomic.try_update(lambda v: v + 1)
self.assertEqual(1001, atomic.value)
self.assertEqual(1001, value)
class Meter(object):
"""A meter measures the rate of events over time (e.g., "requests per second").
In addition to the mean rate, you can also track 1, 5 and 15 minutes moving averages ::
meter = Metrology.meter('requests')
meter.mark()
meter.count
"""
def __init__(self, average_class=EWMA):
self.counter = Atomic(0)
self.start_time = time()
self.last_tick = Atomic(self.start_time)
self.interval = EWMA.INTERVAL
self.m1_rate = EWMA.m1()
self.m5_rate = EWMA.m5()
self.m15_rate = EWMA.m15()
def _tick(self):
old_tick, new_tick = self.last_tick.value, time()
age = new_tick - old_tick
ticks = int(age / self.interval)
new_tick = old_tick + (ticks * self.interval)
if ticks and self.last_tick.compare_and_swap(old_tick, new_tick):
for _ in range(ticks):
self.tick()
@property
def count(self):
"""Returns the total number of events that have been recorded."""
return self.counter.value
def clear(self):
self.counter.value = 0
self.start_time = time()
self.m1_rate.clear()
self.m5_rate.clear()
self.m15_rate.clear()
@ticker
def mark(self, value=1):
"""Record an event with the meter. By default it will record one event.
:param value: number of event to record
"""
self.counter.update(lambda v: v + value)
self.m1_rate.update(value)
self.m5_rate.update(value)
self.m15_rate.update(value)
def tick(self):
self.m1_rate.tick()
self.m5_rate.tick()
self.m15_rate.tick()
@property
@ticker
def one_minute_rate(self):
"""Returns the one-minute average rate."""
return self.m1_rate.rate
@property
@ticker
def five_minute_rate(self):
"""Returns the five-minute average rate."""
return self.m5_rate.rate
@property
@ticker
def fifteen_minute_rate(self):
"""Returns the fifteen-minute average rate."""
return self.m15_rate.rate
@property
def mean_rate(self):
"""Returns the mean rate of the events since the start of the process."""
if self.counter.value == 0:
return 0.0
else:
elapsed = time() - self.start_time
return self.counter.value / elapsed
def stop(self):
pass
class Histogram(object):
"""
A histogram measures the statistical distribution of values in a stream of data. In addition to minimum, maximum, mean, it also measures median, 75th, 90th, 95th, 98th, 99th, and 99.9th percentiles ::
histogram = Metrology.histogram('response-sizes')
histogram.update(len(response.content))
Metrology provides two types of histograms: uniform and exponentially decaying.
"""
DEFAULT_SAMPLE_SIZE = 1028
DEFAULT_ALPHA = 0.015
def __init__(self, sample):
self.sample = sample
self.counter = Atomic(0)
self.minimum = Atomic()
self.maximum = Atomic()
self.sum = Atomic(0)
self.var = Atomic([-1, 0])
def clear(self):
self.sample.clear()
self.counter.value = 0
self.minimum.value = None
self.maximum.value = None
self.sum.value = 0
self.var.value = [-1, 0]
def update(self, value):
self.counter.update(lambda v: v + 1)
self.sample.update(value)
self.max = value
self.min = value
self.sum.update(lambda v: v + value)
self.update_variance(value)
@property
def snapshot(self):
return self.sample.snapshot()
@property
def count(self):
"""Return number of values."""
return self.counter.value
def get_max(self):
if self.counter.value > 0:
return self.maximum.value
return 0.0
def set_max(self, potential_max):
done = False
while not done:
current_max = self.maximum.value
done = (current_max is not None and current_max >= potential_max) \
or self.maximum.compare_and_swap(current_max, potential_max)
max = property(get_max, set_max, doc="""Returns the maximun value.""")
def get_min(self):
if self.counter.value > 0:
return self.minimum.value
return 0.0
def set_min(self, potential_min):
done = False
while not done:
current_min = self.minimum.value
done = (current_min is not None and current_min <= potential_min) \
or self.minimum.compare_and_swap(current_min, potential_min)
min = property(get_min, set_min, doc="""Returns the minimum value.""")
@property
def mean(self):
"""Returns the mean value."""
if self.counter.value > 0:
return self.sum.value / self.counter.value
return 0.0
@property
def stddev(self):
"""Returns the standard deviation."""
if self.counter.value > 0:
return self.variance**.5
return 0.0
@property
def variance(self):
"""Returns variance"""
if self.counter.value <= 1:
return 0.0
return self.var.value[1] / (self.counter.value - 1)
def update_variance(self, value):
def variance(old_values):
if old_values[0] == -1:
new_values = (value, 0)
else:
old_m = old_values[0]
old_s = old_values[1]
new_m = old_m + ((value - old_m) / self.counter.value)
new_s = old_s + ((value - old_m) * (value - new_m))
new_values = (new_m, new_s)
return new_values
self.var.update(variance)
class ExponentiallyDecayingSample(object):
RESCALE_THRESHOLD = 60 * 60
def __init__(self, reservoir_size, alpha):
self.values = RBTree()
self.counter = Atomic(0)
self.next_scale_time = Atomic(0)
self.alpha = alpha
self.reservoir_size = reservoir_size
self.lock = RLock()
self.clear()
def clear(self):
with self.lock:
self.values.clear()
self.counter.value = 0
self.next_scale_time.value = time() + self.RESCALE_THRESHOLD
self.start_time = time()
def size(self):
count = self.counter.value
if count < self.reservoir_size:
return count
return self.reservoir_size
def __len__(self):
return self.size()
def snapshot(self):
with self.lock:
return Snapshot(list(self.values.values()))
def weight(self, timestamp):
return math.exp(self.alpha * timestamp)
def rescale(self, now, next_time):
if self.next_scale_time.compare_and_swap(next_time, now + self.RESCALE_THRESHOLD):
with self.lock:
old_start_time = self.start_time
self.start_time = time()
for key in list(self.values.keys()):
value = self.values.remove(key)
self.values[key * math.exp(-self.alpha * (self.start_time - old_start_time))] = value
def update(self, value, timestamp=None):
if not timestamp:
timestamp = time()
with self.lock:
try:
priority = self.weight(timestamp - self.start_time) / random.random()
except OverflowError:
priority = sys.float_info.max
new_count = self.counter.update(lambda v: v + 1)
if math.isnan(priority):
return
if new_count <= self.reservoir_size:
self.values[priority] = value
else:
first_priority = self.values.root.key
if first_priority < priority:
if priority in self.values:
self.values[priority] = value
if not self.values.remove(first_priority):
first_priority = self.values.root()
class Meter(object):
"""A meter measures the rate of events over time (e.g., "requests per second").
In addition to the mean rate, you can also track 1, 5 and 15 minutes moving averages ::
meter = Metrology.meter('requests')
meter.mark()
meter.count
"""
def __init__(self, average_class=EWMA):
self.counter = Atomic(0)
self.start_time = time()
self.last_tick = Atomic(self.start_time)
self.interval = EWMA.INTERVAL
self.m1_rate = EWMA.m1()
self.m5_rate = EWMA.m5()
self.m15_rate = EWMA.m15()
def _tick(self):
old_tick, new_tick = self.last_tick.value, time()
age = new_tick - old_tick
ticks = int(age / self.interval)
new_tick = old_tick + (ticks * self.interval)
if ticks and self.last_tick.compare_and_swap(old_tick, new_tick):
for _ in range(ticks):
self.tick()
@property
def count(self):
"""Returns the total number of events that have been recorded."""
return self.counter.value
def clear(self):
self.counter.value = 0
self.start_time = time()
self.m1_rate.clear()
self.m5_rate.clear()
self.m15_rate.clear()
@ticker
def mark(self, value=1):
"""Record an event with the meter. By default it will record one event.
:param value: number of event to record
"""
self.counter.update(lambda v: v + value)
self.m1_rate.update(value)
self.m5_rate.update(value)
self.m15_rate.update(value)
def tick(self):
self.m1_rate.tick()
self.m5_rate.tick()
self.m15_rate.tick()
@property
@ticker
def one_minute_rate(self):
"""Returns the one-minute average rate."""
return self.m1_rate.rate
@property
@ticker
def five_minute_rate(self):
"""Returns the five-minute average rate."""
return self.m5_rate.rate
@property
@ticker
def fifteen_minute_rate(self):
"""Returns the fifteen-minute average rate."""
return self.m15_rate.rate
@property
def mean_rate(self):
"""Returns the mean rate of the events since the start of the process."""
if self.counter.value == 0:
return 0.0
else:
elapsed = time() - self.start_time
return self.counter.value / elapsed
def stop(self):
pass
def __init__(self, reservoir_size):
self.counter = Atomic(0)
self.values = [0] * reservoir_size
class Meter(object):
"""A meter measures the rate of events over time (e.g., "requests per second").
In addition to the mean rate, you can also track 1, 5 and 15 minutes moving averages ::
meter = Metrology.meter('requests')
meter.mark()
meter.count
"""
def __init__(self, average_class=EWMA):
self.counter = Atomic(0)
self.start_time = time()
self.m1_rate = EWMA.m1()
self.m5_rate = EWMA.m5()
self.m15_rate = EWMA.m15()
self.task = PeriodicTask(interval=average_class.INTERVAL,
target=self.tick)
self.task.start()
@property
def count(self):
"""Returns the total number of events that have been recorded."""
return self.counter.value
def clear(self):
self.counter.value = 0
self.start_time = time()
self.m1_rate.clear()
self.m5_rate.clear()
self.m15_rate.clear()
def mark(self, value=1):
"""Record an event with the meter. By default it will record one event.
:param value: number of event to record
"""
self.counter.update(lambda v: v + value)
self.m1_rate.update(value)
self.m5_rate.update(value)
self.m15_rate.update(value)
def tick(self):
self.m1_rate.tick()
self.m5_rate.tick()
self.m15_rate.tick()
@property
def one_minute_rate(self):
"""Returns the one-minute average rate."""
return self.m1_rate.rate
@property
def five_minute_rate(self):
"""Returns the five-minute average rate."""
return self.m5_rate.rate
@property
def fifteen_minute_rate(self):
"""Returns the fifteen-minute average rate."""
return self.m15_rate.rate
@property
def mean_rate(self):
"""Returns the mean rate of the events since the start of the process."""
if self.counter.value == 0:
return 0.0
else:
elapsed = time() - self.start_time
return self.counter.value / elapsed
def stop(self):
self.task.stop()
class ExponentiallyDecayingSample(object):
def __init__(self, reservoir_size, alpha):
self.values = []
self.next_scale_time = Atomic(0)
self.alpha = alpha
self.reservoir_size = reservoir_size
self.lock = RLock()
self.rescale_threshold = ExponentiallyDecayingSample.calculate_rescale_threshold(alpha)
self.clear()
@staticmethod
def calculate_rescale_threshold(alpha):
# determine rescale-threshold such that we will not overflow exp() in
# weight function, and subsequently not overflow into inf on dividing
# by random.random()
min_rand = 1.0 / (2**32) # minimum non-zero value from random()
safety = 2.0 # safety pad for numerical inaccuracy
max_value = sys.float_info.max * min_rand / safety
return math.log(max_value) / alpha
def clear(self):
with self.lock:
self.values = []
self.start_time = time()
self.next_scale_time.value = self.start_time + self.rescale_threshold
def size(self):
with self.lock:
return len(self.values)
def __len__(self):
return self.size()
def snapshot(self):
with self.lock:
return Snapshot(val for _, val in self.values)
def weight(self, timestamp):
return math.exp(self.alpha * (timestamp - self.start_time))
def rescale(self, now, next_time):
if self.next_scale_time.compare_and_swap(next_time, now + self.rescale_threshold):
with self.lock:
rescaleFactor = math.exp(-self.alpha * (now - self.start_time))
self.values = [(k * rescaleFactor, v) for k, v in self.values]
self.start_time = now
def rescale_if_necessary(self):
now = time()
next_time = self.next_scale_time.get_value()
if now > next_time:
self.rescale(now, next_time)
def update(self, value, timestamp=None):
if timestamp is None:
timestamp = time()
self.rescale_if_necessary()
with self.lock:
try:
priority = self.weight(timestamp) / random.random()
except (OverflowError, ZeroDivisionError):
priority = sys.float_info.max
if len(self.values) < self.reservoir_size:
heapq.heappush(self.values, (priority, value))
else:
heapq.heappushpop(self.values, (priority, value))
def __init__(self, reservoir_size):
self.counter = Atomic(0)
self.values = [0] * reservoir_size
def test_try_update(self):
atomic = Atomic(1000)
value = atomic.try_update(lambda v: v + 1)
self.assertEqual(1001, atomic.value)
self.assertEqual(1001, value)
def __init__(self, average_class=EWMA):
self.last = Atomic(0)
super(Derive, self).__init__(average_class)
def test_value(self):
atomic = Atomic(0)
atomic.value = 1
self.assertEqual(1, atomic.value)
def test_value(self):
atomic = Atomic(0)
atomic.value = 1
self.assertEqual(1, atomic.value)
def __init__(self,
scheme):
super(UniqueIdSupplier, self).__init__()
self._scheme = scheme
self._id_count = Atomic(0)
class Histogram(object):
"""
A histogram measures the statistical distribution of values in a stream of data. In addition to minimum, maximum, mean, it also measures median, 75th, 90th, 95th, 98th, 99th, and 99.9th percentiles ::
histogram = Metrology.histogram('response-sizes')
histogram.update(len(response.content))
Metrology provides two types of histograms: uniform and exponentially decaying.
"""
DEFAULT_SAMPLE_SIZE = 1028
DEFAULT_ALPHA = 0.015
def __init__(self, sample):
self.sample = sample
self.counter = Atomic(0)
self.minimum = Atomic()
self.maximum = Atomic()
self.sum = Atomic(0)
self.var = Atomic([-1, 0])
def clear(self):
self.sample.clear()
self.counter.value = 0
self.minimum.value = None
self.maximum.value = None
self.sum.value = 0
self.var.value = [-1, 0]
def update(self, value):
self.counter.update(lambda v: v + 1)
self.sample.update(value)
self.max = value
self.min = value
self.sum.update(lambda v: v + value)
self.update_variance(value)
@property
def snapshot(self):
return self.sample.snapshot()
@property
def count(self):
"""Return number of values."""
return self.counter.value
def get_max(self):
if self.counter.value > 0:
return self.maximum.value
return 0.0
def set_max(self, potential_max):
done = False
while not done:
current_max = self.maximum.value
done = (current_max is not None and current_max >= potential_max) \
or self.maximum.compare_and_swap(current_max, potential_max)
max = property(get_max, set_max, doc="""Returns the maximun value.""")
def get_min(self):
if self.counter.value > 0:
return self.minimum.value
return 0.0
def set_min(self, potential_min):
done = False
while not done:
current_min = self.minimum.value
done = (current_min is not None and current_min <= potential_min) \
or self.minimum.compare_and_swap(current_min, potential_min)
min = property(get_min, set_min, doc="""Returns the minimum value.""")
@property
def mean(self):
"""Returns the mean value."""
if self.counter.value > 0:
return self.sum.value / self.counter.value
return 0.0
@property
def stddev(self):
"""Returns the standard deviation."""
if self.counter.value > 0:
return self.variance
return 0.0
@property
def variance(self):
"""Returns variance"""
if self.counter.value <= 1:
return 0.0
return self.var.value[1] / (self.counter.value - 1)
def update_variance(self, value):
def variance(old_values):
if old_values[0] == -1:
new_values = (value, 0)
else:
old_m = old_values[0]
old_s = old_values[1]
new_m = old_m + ((value - old_m) / self.counter.value)
new_s = old_s + ((value - old_m) * (value - new_m))
new_values = (new_m, new_s)
return new_values
self.var.update(variance)
class ExponentiallyDecayingSample(object):
RESCALE_THRESHOLD = 60 * 60
def __init__(self, reservoir_size, alpha):
self.values = RBTree()
self.counter = Atomic(0)
self.next_scale_time = Atomic(0)
self.alpha = alpha
self.reservoir_size = reservoir_size
self.lock = RLock()
self.clear()
def clear(self):
with self.lock:
self.values.clear()
self.counter.value = 0
self.next_scale_time.value = time() + self.RESCALE_THRESHOLD
self.start_time = time()
def size(self):
count = self.counter.value
if count < self.reservoir_size:
return count
return self.reservoir_size
def __len__(self):
return self.size()
def snapshot(self):
with self.lock:
return Snapshot(list(self.values.values()))
def weight(self, timestamp):
return math.exp(self.alpha * timestamp)
def rescale(self, now, next_time):
if self.next_scale_time.compare_and_swap(next_time,
now + self.RESCALE_THRESHOLD):
with self.lock:
old_start_time = self.start_time
self.start_time = time()
for key in list(self.values.keys()):
value = self.values.remove(key)
self.values[key * math.exp(
-self.alpha * (self.start_time - old_start_time))] = value
def update(self, value, timestamp=None):
if not timestamp:
timestamp = time()
with self.lock:
try:
priority = self.weight(timestamp -
self.start_time) / random.random()
except OverflowError:
priority = sys.float_info.max
new_count = self.counter.update(lambda v: v + 1)
if math.isnan(priority):
return
if new_count <= self.reservoir_size:
self.values[priority] = value
else:
first_priority = self.values.root.key
if first_priority < priority:
if priority in self.values:
self.values[priority] = value
if not self.values.remove(first_priority):
first_priority = self.values.root()