class PmfStat(Stat):
"""A stat that stores min, max, mean, standard deviation, and some
percentiles for arbitrary floating-point data. This is potentially a
bit expensive, so its child values are only updated once every
twenty seconds."""
def __init__(self, name, _=None):
Stat.__init__(self, name, None)
self.__sample = UniformSample()
self.__timestamp = time.time()
self.percentile99 = None
def _getDefault(self, instance):
return PmfStatDict(self, instance)
def __set__(self, instance, value):
self.__sample.update(value)
if time.time() > self.__timestamp + 20 and len(self.__sample) > 1:
self.__timestamp = time.time()
histogram = self.__get__(instance, None)
histogram['min'] = self.__sample.min
histogram['max'] = self.__sample.max
histogram['mean'] = self.__sample.mean
histogram['stddev'] = self.__sample.stddev
percentiles = self.__sample.percentiles([0.5, 0.75, 0.95, 0.98, 0.99, 0.999])
histogram['median'] = percentiles[0]
histogram['75percentile'] = percentiles[1]
histogram['95percentile'] = percentiles[2]
histogram['98percentile'] = percentiles[3]
histogram['99percentile'] = percentiles[4]
self.percentile99 = percentiles[4]
histogram['999percentile'] = percentiles[5]
def testGaussian(self):
"""Test with gaussian random numbers."""
random.seed(42)
us = UniformSample()
for _ in range(300):
us.update(random.gauss(42.0, 13.0))
self.assertAlmostEqual(us.mean, 43.143067271195235, places=5)
self.assertAlmostEqual(us.stddev, 13.008553229943168, places=5)
us.clear()
for _ in range(30000):
us.update(random.gauss(0.0012, 0.00005))
self.assertAlmostEqual(us.mean, 0.0012015284549517493, places=5)
self.assertAlmostEqual(us.stddev, 4.9776450250869146e-05, places=5)
def testGaussian(self):
"""Test with gaussian random numbers."""
random.seed(42)
us = UniformSample()
for _ in range(300):
us.update(random.gauss(42.0, 13.0))
self.assertAlmostEqual(us.mean, 43.143067271195235, places=5)
self.assertAlmostEqual(us.stddev, 13.008553229943168, places=5)
us.clear()
for _ in range(30000):
us.update(random.gauss(0.0012, 0.00005))
self.assertAlmostEqual(us.mean, 0.0012015284549517493, places=5)
self.assertAlmostEqual(us.stddev, 4.9776450250869146e-05, places=5)
def __init__(self, name, _=None): Stat.__init__(self, name, None) self.__sample = UniformSample() self.__timestamp = time.time() self.percentile99 = None
def __init__(self): UserDict.__init__(self) self.__sample = UniformSample() self.__timestamp = 0 self.percentile99 = None self['count'] = 0
class PmfStatDict(UserDict):
"""Ugly hack defaultdict-like thing."""
class TimeManager(object):
"""Context manager for timing."""
def __init__(self, container):
self.container = container
self.msg99 = None
self.start = None
self.__discard = False
def __enter__(self):
self.start = time.time()
return self
def __exit__(self, *_):
if not self.__discard:
latency = time.time() - self.start
self.container.addValue(latency)
if self.container.percentile99 is not None and latency >= self.container.percentile99:
if self.msg99 is not None:
logger, msg, args = self.msg99
logger.warn(msg, *args)
def warn99(self, logger, msg, *args):
"""If this time through the timed section of code takes longer
than the 99th percentile time, then this will call
logger.warn(msg, *args) at the end of the section."""
self.msg99 = (logger, msg, args)
def discard(self):
"""Discard this sample."""
self.__discard = True
def __init__(self):
UserDict.__init__(self)
self.__sample = UniformSample()
self.__timestamp = 0
self.percentile99 = None
self['count'] = 0
def __getitem__(self, item):
if item in self:
return UserDict.__getitem__(self, item)
else:
return 0.0
def addValue(self, value):
"""Updates the dictionary."""
self['count'] += 1
self.__sample.update(value)
if time.time() > self.__timestamp + 20 and len(self.__sample) > 1:
self.__timestamp = time.time()
self['min'] = self.__sample.min
self['max'] = self.__sample.max
self['mean'] = self.__sample.mean
self['stddev'] = self.__sample.stddev
percentiles = self.__sample.percentiles([0.5, 0.75, 0.95, 0.98, 0.99, 0.999])
self['median'] = percentiles[0]
self['75percentile'] = percentiles[1]
self['95percentile'] = percentiles[2]
self['98percentile'] = percentiles[3]
self['99percentile'] = percentiles[4]
self.percentile99 = percentiles[4]
self['999percentile'] = percentiles[5]
def time(self):
"""Measure the time this section of code takes. For use in with statements."""
return self.TimeManager(self)
def __init__(self):
UserDict.__init__(self)
self.__sample = UniformSample()
self.__timestamp = 0
self.percentile99 = None
self['count'] = 0
class PmfStatDict(UserDict):
"""Ugly hack defaultdict-like thing."""
class TimeManager(object):
"""Context manager for timing."""
def __init__(self, container):
self.container = container
self.msg99 = None
self.start = None
self.__discard = False
def __enter__(self):
self.start = time.time()
return self
def __exit__(self, *_):
if not self.__discard:
latency = time.time() - self.start
self.container.addValue(latency)
if self.container.percentile99 is not None and latency >= self.container.percentile99:
if self.msg99 is not None:
logger, msg, args = self.msg99
logger.warn(msg, *args)
def warn99(self, logger, msg, *args):
"""If this time through the timed section of code takes longer
than the 99th percentile time, then this will call
logger.warn(msg, *args) at the end of the section."""
self.msg99 = (logger, msg, args)
def discard(self):
"""Discard this sample."""
self.__discard = True
def __init__(self):
UserDict.__init__(self)
self.__sample = UniformSample()
self.__timestamp = 0
self.percentile99 = None
self['count'] = 0
def __getitem__(self, item):
if item in self:
return UserDict.__getitem__(self, item)
else:
return 0.0
def addValue(self, value):
"""Updates the dictionary."""
self['count'] += 1
self.__sample.update(value)
if time.time() > self.__timestamp + 20 and len(self.__sample) > 1:
self.__timestamp = time.time()
self['min'] = self.__sample.min
self['max'] = self.__sample.max
self['mean'] = self.__sample.mean
self['stddev'] = self.__sample.stddev
percentiles = self.__sample.percentiles(
[0.5, 0.75, 0.95, 0.98, 0.99, 0.999])
self['median'] = percentiles[0]
self['75percentile'] = percentiles[1]
self['95percentile'] = percentiles[2]
self['98percentile'] = percentiles[3]
self['99percentile'] = percentiles[4]
self.percentile99 = percentiles[4]
self['999percentile'] = percentiles[5]
def time(self):
"""Measure the time this section of code takes. For use in with statements."""
return self.TimeManager(self)