Example #1
0
 def __init__(self, sample):
     self.sample = sample
     self.counter = Atomic(0)
     self.minimum = Atomic()
     self.maximum = Atomic()
     self.sum = Atomic(0)
     self.var = Atomic([-1, 0])
Example #2
0
 def __init__(self, sample):
     self.sample = sample
     self.counter = Atomic(0)
     self.minimum = Atomic()
     self.maximum = Atomic()
     self.sum = Atomic(0)
     self.var = Atomic([-1, 0])
Example #3
0
 def test_complex_update(self):
     def complex_update(v):
         return [v + 1 for v in v]
     atomic = Atomic([-1, 0])
     value = atomic.update(complex_update)
     self.assertEqual([0, 1], atomic.value)
     self.assertEqual([0, 1], value)
Example #4
0
class Counter(object):
    """
    A counter is like a gauge, but you can increment or decrement its value ::

      counter = Metrology.counter('pending-jobs')
      counter.increment()
      counter.decrement()
      counter.count

    """
    def __init__(self):
        self._count = Atomic(0)

    def increment(self, value=1):
        """Increment the counter. By default it will increment by 1.

        :param value: value to increment the counter.
        """
        self._count.update(lambda v: v + value)

    def decrement(self, value=1):
        """Decrement the counter. By default it will decrement by 1.

        :param value: value to decrement the counter.
        """
        self._count.update(lambda v: v - value)

    def clear(self):
        self._count.value = 0

    @property
    def count(self):
        """Return the current value of the counter."""
        return self._count.value
Example #5
0
    def __init__(self, alpha, interval):
        self.alpha = alpha
        self.interval = interval

        self.initialized = False
        self._rate = 0.0
        self._uncounted = Atomic(0)
Example #6
0
 def __init__(self, reservoir_size, alpha):
     self.values = RBTree()
     self.counter = Atomic(0)
     self.next_scale_time = Atomic(0)
     self.alpha = alpha
     self.reservoir_size = reservoir_size
     self.lock = RLock()
     self.clear()
Example #7
0
    def test_complex_update(self):
        def complex_update(v):
            return [v + 1 for v in v]

        atomic = Atomic([-1, 0])
        value = atomic.update(complex_update)
        self.assertEqual([0, 1], atomic.value)
        self.assertEqual([0, 1], value)
Example #8
0
 def __init__(self, reservoir_size, alpha):
     self.values = RBTree()
     self.counter = Atomic(0)
     self.next_scale_time = Atomic(0)
     self.alpha = alpha
     self.reservoir_size = reservoir_size
     self.lock = RLock()
     self.clear()
Example #9
0
 def __init__(self, reservoir_size, alpha):
     self.values = []
     self.next_scale_time = Atomic(0)
     self.alpha = alpha
     self.reservoir_size = reservoir_size
     self.lock = RLock()
     self.rescale_threshold = ExponentiallyDecayingSample.calculate_rescale_threshold(alpha)
     self.clear()
Example #10
0
    def __init__(self, average_class=EWMA):
        self.counter = Atomic(0)
        self.start_time = time()
        self.last_tick = Atomic(self.start_time)

        self.interval = EWMA.INTERVAL
        self.m1_rate = EWMA.m1()
        self.m5_rate = EWMA.m5()
        self.m15_rate = EWMA.m15()
Example #11
0
    def __init__(self, average_class=EWMA):
        self.counter = Atomic(0)
        self.start_time = time()
        self.last_tick = Atomic(self.start_time)

        self.interval = EWMA.INTERVAL
        self.m1_rate = EWMA.m1()
        self.m5_rate = EWMA.m5()
        self.m15_rate = EWMA.m15()
Example #12
0
class EWMA(object):
    INTERVAL = 5.0
    SECONDS_PER_MINUTE = 60.0

    ONE_MINUTE = 1
    FIVE_MINUTES = 5
    FIFTEEN_MINUTES = 15

    M1_ALPHA = 1 - math.exp(-INTERVAL / SECONDS_PER_MINUTE / ONE_MINUTE)
    M5_ALPHA = 1 - math.exp(-INTERVAL / SECONDS_PER_MINUTE / FIVE_MINUTES)
    M15_ALPHA = 1 - math.exp(-INTERVAL / SECONDS_PER_MINUTE / FIFTEEN_MINUTES)

    @classmethod
    def m1(cls):
        return EWMA(cls.M1_ALPHA, cls.INTERVAL)

    @classmethod
    def m5(cls):
        return EWMA(cls.M5_ALPHA, cls.INTERVAL)

    @classmethod
    def m15(cls):
        return EWMA(cls.M15_ALPHA, cls.INTERVAL)

    def __init__(self, alpha, interval):
        self.alpha = alpha
        self.interval = interval

        self.initialized = False
        self._rate = 0.0
        self._uncounted = Atomic(0)

    def clear(self):
        self.initialized = False
        self._rate = 0.0
        self._uncounted.value = 0

    def update(self, value):
        self._uncounted.update(lambda v: v + value)

    def tick(self):
        count = self._uncounted.swap(0)
        instant_rate = count / self.interval

        if self.initialized:
            self._rate += self.alpha * (instant_rate - self._rate)
        else:
            self._rate = instant_rate
            self.initialized = True

    @property
    def rate(self):
        return self._rate
Example #13
0
 def set_args(self, args):
     Atomic.set_args(self, args)
     if 'graph' in self.args and self.args.graph:
         self.graphdir = self.args.graph
     else:
         if os.path.exists("/var/lib/docker") and os.path.exists("/var/lib/docker-latest"):
             raise ValueError("You must specify the --graph storage path to reset /var/lib/docker or /var/lib/docker-latest")
         if os.path.exists("/var/lib/docker"):
             self.graphdir = "/var/lib/docker"
         else:
             if os.path.exists("/var/lib/docker-latest"):
                 self.graphdir = "/var/lib/docker-latest"
             else:
                 raise ValueError("Could not find any default graph storage path. Specify one using --graph option")
Example #14
0
class UniformSample(object):
    def __init__(self, reservoir_size):
        self.counter = Atomic(0)
        self.values = [0] * reservoir_size

    def clear(self):
        self.values = [0] * len(self.values)
        self.counter.value = 0

    def size(self):
        count = self.counter.value
        if count > len(self.values):
            return len(self.values)
        return count

    def __len__(self):
        return self.size

    def snapshot(self):
        return Snapshot(self.values[0:self.size()])

    def update(self, value):
        new_count = self.counter.update(lambda v: v + 1)

        if new_count <= len(self.values):
            self.values[new_count - 1] = value
        else:
            index = random.uniform(0, new_count)
            if index < len(self.values):
                self.values[int(index)] = value
Example #15
0
class UniformSample(object):
    def __init__(self, reservoir_size):
        self.counter = Atomic(0)
        self.values = [0] * reservoir_size

    def clear(self):
        self.values = [0] * len(self.values)
        self.counter.value = 0

    def size(self):
        count = self.counter.value
        if count > len(self.values):
            return len(self.values)
        return count

    def __len__(self):
        return self.size

    def snapshot(self):
        return Snapshot(self.values[0:self.size()])

    def update(self, value):
        new_count = self.counter.update(lambda v: v + 1)

        if new_count <= len(self.values):
            self.values[new_count - 1] = value
        else:
            index = random.uniform(0, new_count)
            if index < len(self.values):
                self.values[int(index)] = value
Example #16
0
class UniqueIdSupplier(Supplier):
    def __init__(self,
                 scheme):
        super(UniqueIdSupplier, self).__init__()
        self._scheme = scheme
        self._id_count = Atomic(0)

    def get(self):
        new_id = self._id_count.update(lambda v: v + 1)
        return UniqueId.of(self._scheme, str(new_id))

    def get_with_value_prefix(self, value_prefix):
        new_id = self._id_count.update(lambda v: v + 1)
        return UniqueId.of(self._scheme, value_prefix + str(new_id))

    def __str__(self):
        return 'UniqueIdSupplier[' + self._scheme + ']'
Example #17
0
class ToggleGauge(Gauge):
    _value = Atomic(1)

    @property
    def value(self):
        try:
            return self._value.value
        finally:
            self._value.value = 0
Example #18
0
 def set_args(self, args):
     Atomic.set_args(self, args)
     if 'graph' in self.args and self.args.graph:
         self.graphdir = self.args.graph
     else:
         if os.path.exists("/var/lib/docker") and os.path.exists(
                 "/var/lib/docker-latest"):
             raise ValueError(
                 "You must specify the --graph storage path to reset /var/lib/docker or /var/lib/docker-latest"
             )
         if os.path.exists("/var/lib/docker"):
             self.graphdir = "/var/lib/docker"
         else:
             if os.path.exists("/var/lib/docker-latest"):
                 self.graphdir = "/var/lib/docker-latest"
             else:
                 raise ValueError(
                     "Could not find any default graph storage path. Specify one using --graph option"
                 )
Example #19
0
    def __init__(self, average_class=EWMA):
        self.counter = Atomic(0)
        self.start_time = time()

        self.m1_rate = EWMA.m1()
        self.m5_rate = EWMA.m5()
        self.m15_rate = EWMA.m15()

        self.task = PeriodicTask(interval=average_class.INTERVAL,
            target=self.tick)
        self.task.start()
Example #20
0
class ObjectIdSupplier(Supplier):
    """
    A supplier of object identifiers.
    An object identifier consists of a scheme and value.
    This class creates object identifiers for a fixed scheme name, where each
    value is an incrementing number. The values are created in a thread-safe way.
    This class is thread-safe and not externally mutable.
    """
    def __init__(self,
                 scheme):
        self._scheme = scheme
        self._id_count = Atomic(0)

    def get(self):
        new_id = self._id_count.update(lambda v: v + 1)
        return ObjectId.of(self._scheme, str(new_id))

    def get_with_value_prefix(self, value_prefix):
        new_id = self._id_count.update(lambda v: v + 1)
        return ObjectId.of(self._scheme, value_prefix + str(new_id))

    def __str__(self):
        return 'ObjectIdSupplier[' + self._scheme + ']'
Example #21
0
class Derive(Meter):
    """
    A derive is like a meter but accepts an absolute counter as input.

      derive = Metrology.derive('network.io')
      derive.mark()
      derive.count

    """
    def __init__(self, average_class=EWMA):
        self.last = Atomic(0)
        super(Derive, self).__init__(average_class)

    def mark(self, value=1):
        """Record an event with the derive.

        :param value: counter value to record
        """
        last = self.last.get_and_set(value)
        if last <= value:
            value = value - last
        super(Derive, self).mark(value)
Example #22
0
 def test_swap(self):
     atomic = Atomic(1000)
     swapped = atomic.swap(1001)
     self.assertEqual(1001, atomic.value)
     self.assertEqual(1000, swapped)
Example #23
0
class Histogram(object):
    DEFAULT_SAMPLE_SIZE = 1028
    DEFAULT_ALPHA = 0.015

    def __init__(self, sample):
        self.sample = sample
        self.counter = Atomic(0)
        self.minimum = Atomic()
        self.maximum = Atomic()
        self.sum = Atomic(0)
        self.var = Atomic([-1, 0])

    def clear(self):
        self.sample.clear()
        self.counter.value = 0
        self.minimum.value = None
        self.maximum.value = None
        self.sum.value = 0
        self.var.value = [-1, 0]

    def update(self, value):
        with self.counter:
            self.counter.value += 1
        self.sample.update(value)
        self.max = value
        self.min = value
        with self.sum:
            self.sum.value += value
        self.update_variance(value)

    @property
    def snapshot(self):
        return self.sample.snapshot()

    @property
    def count(self):
        return self.counter.value

    def get_max(self):
        if self.counter.value > 0:
            return self.maximum.value
        return 0.0

    def set_max(self, potential_max):
        done = False
        while not done:
            current_max = self.maximum.value
            done = (current_max is not None and current_max >= potential_max) or self.maximum.compare_and_swap(
                current_max, potential_max
            )

    max = property(get_max, set_max)

    def get_min(self):
        if self.counter.value > 0:
            return self.minimum.value
        return 0.0

    def set_min(self, potential_min):
        done = False
        while not done:
            current_min = self.minimum.value
            done = (current_min is not None and current_min <= potential_min) or self.minimum.compare_and_swap(
                current_min, potential_min
            )

    min = property(get_min, set_min)

    @property
    def mean(self):
        if self.counter.value > 0:
            return self.sum.value / self.counter.value
        return 0.0

    @property
    def stddev(self):
        if self.counter.value > 0:
            return self.var.value
        return 0.0

    @property
    def variance(self):
        if self.counter.value <= 1:
            return 0.0
        return self.var.value[1] / (self.counter.value - 1)

    def update_variance(self, value):
        with self.var:
            old_values = self.var.value
            if old_values[0] == -1:
                new_values = (value, 0)
            else:
                old_m = old_values[0]
                old_s = old_values[1]

                new_m = old_m + ((value - old_m) / self.counter.value)
                new_s = old_s + ((value - old_m) * (value - new_m))

                new_values = (new_m, new_s)

            self.var.value = new_values
            return new_values
Example #24
0
    def test_init(self):
        atomic = Atomic()
        self.assertEqual(None, atomic.value)

        atomic = Atomic(0)
        self.assertEqual(0, atomic.value)
Example #25
0
 def __init__(self):
     self._count = Atomic(0)
Example #26
0
 def test_complex_value(self):
     atomic = Atomic([-1, 0])
     self.assertEqual([-1, 0], atomic.value)
Example #27
0
 def __init__(self,
              scheme):
     self._scheme = scheme
     self._id_count = Atomic(0)
Example #28
0
 def test_swap(self):
     atomic = Atomic(1000)
     swapped = atomic.swap(1001)
     self.assertEqual(1001, atomic.value)
     self.assertEqual(1000, swapped)
Example #29
0
 def test_try_update(self):
     atomic = Atomic(1000)
     value = atomic.try_update(lambda v: v + 1)
     self.assertEqual(1001, atomic.value)
     self.assertEqual(1001, value)
Example #30
0
class Meter(object):
    """A meter measures the rate of events over time (e.g., "requests per second").
    In addition to the mean rate, you can also track 1, 5 and 15 minutes moving averages ::

      meter = Metrology.meter('requests')
      meter.mark()
      meter.count

    """
    def __init__(self, average_class=EWMA):
        self.counter = Atomic(0)
        self.start_time = time()
        self.last_tick = Atomic(self.start_time)

        self.interval = EWMA.INTERVAL
        self.m1_rate = EWMA.m1()
        self.m5_rate = EWMA.m5()
        self.m15_rate = EWMA.m15()

    def _tick(self):
        old_tick, new_tick = self.last_tick.value, time()
        age = new_tick - old_tick
        ticks = int(age / self.interval)
        new_tick = old_tick + (ticks * self.interval)
        if ticks and self.last_tick.compare_and_swap(old_tick, new_tick):
            for _ in range(ticks):
                self.tick()

    @property
    def count(self):
        """Returns the total number of events that have been recorded."""
        return self.counter.value

    def clear(self):
        self.counter.value = 0
        self.start_time = time()

        self.m1_rate.clear()
        self.m5_rate.clear()
        self.m15_rate.clear()

    @ticker
    def mark(self, value=1):
        """Record an event with the meter. By default it will record one event.

        :param value: number of event to record
        """
        self.counter.update(lambda v: v + value)
        self.m1_rate.update(value)
        self.m5_rate.update(value)
        self.m15_rate.update(value)

    def tick(self):
        self.m1_rate.tick()
        self.m5_rate.tick()
        self.m15_rate.tick()

    @property
    @ticker
    def one_minute_rate(self):
        """Returns the one-minute average rate."""
        return self.m1_rate.rate

    @property
    @ticker
    def five_minute_rate(self):
        """Returns the five-minute average rate."""
        return self.m5_rate.rate

    @property
    @ticker
    def fifteen_minute_rate(self):
        """Returns the fifteen-minute average rate."""
        return self.m15_rate.rate

    @property
    def mean_rate(self):
        """Returns the mean rate of the events since the start of the process."""
        if self.counter.value == 0:
            return 0.0
        else:
            elapsed = time() - self.start_time
            return self.counter.value / elapsed

    def stop(self):
        pass
Example #31
0
class Histogram(object):
    """
    A histogram measures the statistical distribution of values in a stream of data. In addition to minimum, maximum, mean, it also measures median, 75th, 90th, 95th, 98th, 99th, and 99.9th percentiles ::

      histogram = Metrology.histogram('response-sizes')
      histogram.update(len(response.content))

    Metrology provides two types of histograms: uniform and exponentially decaying.
    """
    DEFAULT_SAMPLE_SIZE = 1028
    DEFAULT_ALPHA = 0.015

    def __init__(self, sample):
        self.sample = sample
        self.counter = Atomic(0)
        self.minimum = Atomic()
        self.maximum = Atomic()
        self.sum = Atomic(0)
        self.var = Atomic([-1, 0])

    def clear(self):
        self.sample.clear()
        self.counter.value = 0
        self.minimum.value = None
        self.maximum.value = None
        self.sum.value = 0
        self.var.value = [-1, 0]

    def update(self, value):
        self.counter.update(lambda v: v + 1)
        self.sample.update(value)
        self.max = value
        self.min = value
        self.sum.update(lambda v: v + value)
        self.update_variance(value)

    @property
    def snapshot(self):
        return self.sample.snapshot()

    @property
    def count(self):
        """Return number of values."""
        return self.counter.value

    def get_max(self):
        if self.counter.value > 0:
            return self.maximum.value
        return 0.0

    def set_max(self, potential_max):
        done = False
        while not done:
            current_max = self.maximum.value
            done = (current_max is not None and current_max >= potential_max) \
                or self.maximum.compare_and_swap(current_max, potential_max)

    max = property(get_max, set_max, doc="""Returns the maximun value.""")

    def get_min(self):
        if self.counter.value > 0:
            return self.minimum.value
        return 0.0

    def set_min(self, potential_min):
        done = False
        while not done:
            current_min = self.minimum.value
            done = (current_min is not None and current_min <= potential_min) \
                or self.minimum.compare_and_swap(current_min, potential_min)

    min = property(get_min, set_min, doc="""Returns the minimum value.""")

    @property
    def mean(self):
        """Returns the mean value."""
        if self.counter.value > 0:
            return self.sum.value / self.counter.value
        return 0.0

    @property
    def stddev(self):
        """Returns the standard deviation."""
        if self.counter.value > 0:
            return self.variance**.5
        return 0.0

    @property
    def variance(self):
        """Returns variance"""
        if self.counter.value <= 1:
            return 0.0
        return self.var.value[1] / (self.counter.value - 1)

    def update_variance(self, value):
        def variance(old_values):
            if old_values[0] == -1:
                new_values = (value, 0)
            else:
                old_m = old_values[0]
                old_s = old_values[1]

                new_m = old_m + ((value - old_m) / self.counter.value)
                new_s = old_s + ((value - old_m) * (value - new_m))

                new_values = (new_m, new_s)
            return new_values

        self.var.update(variance)
Example #32
0
class ExponentiallyDecayingSample(object):
    RESCALE_THRESHOLD = 60 * 60

    def __init__(self, reservoir_size, alpha):
        self.values = RBTree()
        self.counter = Atomic(0)
        self.next_scale_time = Atomic(0)
        self.alpha = alpha
        self.reservoir_size = reservoir_size
        self.lock = RLock()
        self.clear()

    def clear(self):
        with self.lock:
            self.values.clear()
            self.counter.value = 0
            self.next_scale_time.value = time() + self.RESCALE_THRESHOLD
            self.start_time = time()

    def size(self):
        count = self.counter.value
        if count < self.reservoir_size:
            return count
        return self.reservoir_size

    def __len__(self):
        return self.size()

    def snapshot(self):
        with self.lock:
            return Snapshot(list(self.values.values()))

    def weight(self, timestamp):
        return math.exp(self.alpha * timestamp)

    def rescale(self, now, next_time):
        if self.next_scale_time.compare_and_swap(next_time, now + self.RESCALE_THRESHOLD):
            with self.lock:
                old_start_time = self.start_time
                self.start_time = time()
                for key in list(self.values.keys()):
                    value = self.values.remove(key)
                self.values[key * math.exp(-self.alpha * (self.start_time - old_start_time))] = value

    def update(self, value, timestamp=None):
        if not timestamp:
            timestamp = time()
        with self.lock:
            try:
                priority = self.weight(timestamp - self.start_time) / random.random()
            except OverflowError:
                priority = sys.float_info.max
            new_count = self.counter.update(lambda v: v + 1)

            if math.isnan(priority):
                return

            if new_count <= self.reservoir_size:
                self.values[priority] = value
            else:
                first_priority = self.values.root.key
                if first_priority < priority:
                    if priority in self.values:
                        self.values[priority] = value
                        if not self.values.remove(first_priority):
                            first_priority = self.values.root()
Example #33
0
class Meter(object):
    """A meter measures the rate of events over time (e.g., "requests per second").
    In addition to the mean rate, you can also track 1, 5 and 15 minutes moving averages ::

      meter = Metrology.meter('requests')
      meter.mark()
      meter.count

    """
    def __init__(self, average_class=EWMA):
        self.counter = Atomic(0)
        self.start_time = time()
        self.last_tick = Atomic(self.start_time)

        self.interval = EWMA.INTERVAL
        self.m1_rate = EWMA.m1()
        self.m5_rate = EWMA.m5()
        self.m15_rate = EWMA.m15()

    def _tick(self):
        old_tick, new_tick = self.last_tick.value, time()
        age = new_tick - old_tick
        ticks = int(age / self.interval)
        new_tick = old_tick + (ticks * self.interval)
        if ticks and self.last_tick.compare_and_swap(old_tick, new_tick):
            for _ in range(ticks):
                self.tick()

    @property
    def count(self):
        """Returns the total number of events that have been recorded."""
        return self.counter.value

    def clear(self):
        self.counter.value = 0
        self.start_time = time()

        self.m1_rate.clear()
        self.m5_rate.clear()
        self.m15_rate.clear()

    @ticker
    def mark(self, value=1):
        """Record an event with the meter. By default it will record one event.

        :param value: number of event to record
        """
        self.counter.update(lambda v: v + value)
        self.m1_rate.update(value)
        self.m5_rate.update(value)
        self.m15_rate.update(value)

    def tick(self):
        self.m1_rate.tick()
        self.m5_rate.tick()
        self.m15_rate.tick()

    @property
    @ticker
    def one_minute_rate(self):
        """Returns the one-minute average rate."""
        return self.m1_rate.rate

    @property
    @ticker
    def five_minute_rate(self):
        """Returns the five-minute average rate."""
        return self.m5_rate.rate

    @property
    @ticker
    def fifteen_minute_rate(self):
        """Returns the fifteen-minute average rate."""
        return self.m15_rate.rate

    @property
    def mean_rate(self):
        """Returns the mean rate of the events since the start of the process."""
        if self.counter.value == 0:
            return 0.0
        else:
            elapsed = time() - self.start_time
            return self.counter.value / elapsed

    def stop(self):
        pass
Example #34
0
 def __init__(self, reservoir_size):
     self.counter = Atomic(0)
     self.values = [0] * reservoir_size
Example #35
0
class Meter(object):
    """A meter measures the rate of events over time (e.g., "requests per second").
    In addition to the mean rate, you can also track 1, 5 and 15 minutes moving averages ::

      meter = Metrology.meter('requests')
      meter.mark()
      meter.count

    """
    def __init__(self, average_class=EWMA):
        self.counter = Atomic(0)
        self.start_time = time()

        self.m1_rate = EWMA.m1()
        self.m5_rate = EWMA.m5()
        self.m15_rate = EWMA.m15()

        self.task = PeriodicTask(interval=average_class.INTERVAL,
            target=self.tick)
        self.task.start()

    @property
    def count(self):
        """Returns the total number of events that have been recorded."""
        return self.counter.value

    def clear(self):
        self.counter.value = 0
        self.start_time = time()

        self.m1_rate.clear()
        self.m5_rate.clear()
        self.m15_rate.clear()

    def mark(self, value=1):
        """Record an event with the meter. By default it will record one event.

        :param value: number of event to record
        """
        self.counter.update(lambda v: v + value)
        self.m1_rate.update(value)
        self.m5_rate.update(value)
        self.m15_rate.update(value)

    def tick(self):
        self.m1_rate.tick()
        self.m5_rate.tick()
        self.m15_rate.tick()

    @property
    def one_minute_rate(self):
        """Returns the one-minute average rate."""
        return self.m1_rate.rate

    @property
    def five_minute_rate(self):
        """Returns the five-minute average rate."""
        return self.m5_rate.rate

    @property
    def fifteen_minute_rate(self):
        """Returns the fifteen-minute average rate."""
        return self.m15_rate.rate

    @property
    def mean_rate(self):
        """Returns the mean rate of the events since the start of the process."""
        if self.counter.value == 0:
            return 0.0
        else:
            elapsed = time() - self.start_time
            return self.counter.value / elapsed

    def stop(self):
        self.task.stop()
Example #36
0
class ExponentiallyDecayingSample(object):

    def __init__(self, reservoir_size, alpha):
        self.values = []
        self.next_scale_time = Atomic(0)
        self.alpha = alpha
        self.reservoir_size = reservoir_size
        self.lock = RLock()
        self.rescale_threshold = ExponentiallyDecayingSample.calculate_rescale_threshold(alpha)
        self.clear()

    @staticmethod
    def calculate_rescale_threshold(alpha):
        # determine rescale-threshold such that we will not overflow exp() in
        # weight function, and subsequently not overflow into inf on dividing
        # by random.random()
        min_rand = 1.0 / (2**32)    # minimum non-zero value from random()
        safety = 2.0                # safety pad for numerical inaccuracy
        max_value = sys.float_info.max * min_rand / safety
        return math.log(max_value) / alpha

    def clear(self):
        with self.lock:
            self.values = []
            self.start_time = time()
            self.next_scale_time.value = self.start_time + self.rescale_threshold

    def size(self):
        with self.lock:
            return len(self.values)

    def __len__(self):
        return self.size()

    def snapshot(self):
        with self.lock:
            return Snapshot(val for _, val in self.values)

    def weight(self, timestamp):
        return math.exp(self.alpha * (timestamp - self.start_time))

    def rescale(self, now, next_time):
        if self.next_scale_time.compare_and_swap(next_time, now + self.rescale_threshold):
            with self.lock:
                rescaleFactor = math.exp(-self.alpha * (now - self.start_time))
                self.values = [(k * rescaleFactor, v) for k, v in self.values]
                self.start_time = now

    def rescale_if_necessary(self):
        now = time()
        next_time = self.next_scale_time.get_value()
        if now > next_time:
            self.rescale(now, next_time)

    def update(self, value, timestamp=None):
        if timestamp is None:
            timestamp = time()

        self.rescale_if_necessary()
        with self.lock:
            try:
                priority = self.weight(timestamp) / random.random()
            except (OverflowError, ZeroDivisionError):
                priority = sys.float_info.max

            if len(self.values) < self.reservoir_size:
                heapq.heappush(self.values, (priority, value))
            else:
                heapq.heappushpop(self.values, (priority, value))
Example #37
0
 def __init__(self, reservoir_size):
     self.counter = Atomic(0)
     self.values = [0] * reservoir_size
Example #38
0
 def test_try_update(self):
     atomic = Atomic(1000)
     value = atomic.try_update(lambda v: v + 1)
     self.assertEqual(1001, atomic.value)
     self.assertEqual(1001, value)
Example #39
0
 def __init__(self, average_class=EWMA):
     self.last = Atomic(0)
     super(Derive, self).__init__(average_class)
Example #40
0
 def test_value(self):
     atomic = Atomic(0)
     atomic.value = 1
     self.assertEqual(1, atomic.value)
Example #41
0
 def test_value(self):
     atomic = Atomic(0)
     atomic.value = 1
     self.assertEqual(1, atomic.value)
Example #42
0
 def __init__(self,
              scheme):
     super(UniqueIdSupplier, self).__init__()
     self._scheme = scheme
     self._id_count = Atomic(0)
Example #43
0
class Histogram(object):
    """
    A histogram measures the statistical distribution of values in a stream of data. In addition to minimum, maximum, mean, it also measures median, 75th, 90th, 95th, 98th, 99th, and 99.9th percentiles ::

      histogram = Metrology.histogram('response-sizes')
      histogram.update(len(response.content))

    Metrology provides two types of histograms: uniform and exponentially decaying.
    """
    DEFAULT_SAMPLE_SIZE = 1028
    DEFAULT_ALPHA = 0.015

    def __init__(self, sample):
        self.sample = sample
        self.counter = Atomic(0)
        self.minimum = Atomic()
        self.maximum = Atomic()
        self.sum = Atomic(0)
        self.var = Atomic([-1, 0])

    def clear(self):
        self.sample.clear()
        self.counter.value = 0
        self.minimum.value = None
        self.maximum.value = None
        self.sum.value = 0
        self.var.value = [-1, 0]

    def update(self, value):
        self.counter.update(lambda v: v + 1)
        self.sample.update(value)
        self.max = value
        self.min = value
        self.sum.update(lambda v: v + value)
        self.update_variance(value)

    @property
    def snapshot(self):
        return self.sample.snapshot()

    @property
    def count(self):
        """Return number of values."""
        return self.counter.value

    def get_max(self):
        if self.counter.value > 0:
            return self.maximum.value
        return 0.0

    def set_max(self, potential_max):
        done = False
        while not done:
            current_max = self.maximum.value
            done = (current_max is not None and current_max >= potential_max) \
                or self.maximum.compare_and_swap(current_max, potential_max)

    max = property(get_max, set_max, doc="""Returns the maximun value.""")

    def get_min(self):
        if self.counter.value > 0:
            return self.minimum.value
        return 0.0

    def set_min(self, potential_min):
        done = False
        while not done:
            current_min = self.minimum.value
            done = (current_min is not None and current_min <= potential_min) \
                or self.minimum.compare_and_swap(current_min, potential_min)

    min = property(get_min, set_min, doc="""Returns the minimum value.""")

    @property
    def mean(self):
        """Returns the mean value."""
        if self.counter.value > 0:
            return self.sum.value / self.counter.value
        return 0.0

    @property
    def stddev(self):
        """Returns the standard deviation."""
        if self.counter.value > 0:
            return self.variance
        return 0.0

    @property
    def variance(self):
        """Returns variance"""
        if self.counter.value <= 1:
            return 0.0
        return self.var.value[1] / (self.counter.value - 1)

    def update_variance(self, value):
        def variance(old_values):
            if old_values[0] == -1:
                new_values = (value, 0)
            else:
                old_m = old_values[0]
                old_s = old_values[1]

                new_m = old_m + ((value - old_m) / self.counter.value)
                new_s = old_s + ((value - old_m) * (value - new_m))

                new_values = (new_m, new_s)
            return new_values
        self.var.update(variance)
Example #44
0
class ExponentiallyDecayingSample(object):
    RESCALE_THRESHOLD = 60 * 60

    def __init__(self, reservoir_size, alpha):
        self.values = RBTree()
        self.counter = Atomic(0)
        self.next_scale_time = Atomic(0)
        self.alpha = alpha
        self.reservoir_size = reservoir_size
        self.lock = RLock()
        self.clear()

    def clear(self):
        with self.lock:
            self.values.clear()
            self.counter.value = 0
            self.next_scale_time.value = time() + self.RESCALE_THRESHOLD
            self.start_time = time()

    def size(self):
        count = self.counter.value
        if count < self.reservoir_size:
            return count
        return self.reservoir_size

    def __len__(self):
        return self.size()

    def snapshot(self):
        with self.lock:
            return Snapshot(list(self.values.values()))

    def weight(self, timestamp):
        return math.exp(self.alpha * timestamp)

    def rescale(self, now, next_time):
        if self.next_scale_time.compare_and_swap(next_time,
                                                 now + self.RESCALE_THRESHOLD):
            with self.lock:
                old_start_time = self.start_time
                self.start_time = time()
                for key in list(self.values.keys()):
                    value = self.values.remove(key)
                self.values[key * math.exp(
                    -self.alpha * (self.start_time - old_start_time))] = value

    def update(self, value, timestamp=None):
        if not timestamp:
            timestamp = time()
        with self.lock:
            try:
                priority = self.weight(timestamp -
                                       self.start_time) / random.random()
            except OverflowError:
                priority = sys.float_info.max
            new_count = self.counter.update(lambda v: v + 1)

            if math.isnan(priority):
                return

            if new_count <= self.reservoir_size:
                self.values[priority] = value
            else:
                first_priority = self.values.root.key
                if first_priority < priority:
                    if priority in self.values:
                        self.values[priority] = value
                        if not self.values.remove(first_priority):
                            first_priority = self.values.root()