Esempio n. 1
0
    def __init__(self, alpha, interval):
        self.alpha = alpha
        self.interval = interval

        self.initialized = False
        self._rate = 0.0
        self._uncounted = AtomicLong(0)
Esempio n. 2
0
 def __init__(self, sample):
     self.sample = sample
     self.counter = AtomicLong(0)
     self.minimum = AtomicLong(sys.maxsize)
     self.maximum = AtomicLong(-sys.maxsize - 1)
     self.sum = AtomicLong(0)
     self.var = AtomicLongArray([-1, 0])
Esempio n. 3
0
    def test_compare_and_swap(self):
        atomic = AtomicLong(1000)
        swapped = atomic.compare_and_swap(1000, 1001)
        self.assertEqual(1001, atomic.value)
        self.assertEqual(True, swapped)

        swapped = atomic.compare_and_swap(1000, 1024)
        self.assertEqual(1001, atomic.value)
        self.assertEqual(False, swapped)
Esempio n. 4
0
    def __init__(self, average_class=EWMA):
        self.counter = AtomicLong(0)
        self.start_time = now()
        self.last_tick = AtomicLong(self.start_time)

        self.interval = EWMA.INTERVAL
        self.m1_rate = EWMA.m1()
        self.m5_rate = EWMA.m5()
        self.m15_rate = EWMA.m15()
Esempio n. 5
0
class Stat(object):
    gen_tasks = AtomicLong(0)
    finished_tasks = AtomicLong(0)
    task_queue = queue.Queue()
    # Each result contains several fields
    # 1. TaskID
    # 2. TimePoints: create time, send time, response time
    # 3. Succ
    result_queue = queue.Queue()
Esempio n. 6
0
    def test_compare_and_swap(self):
        atomic = AtomicLong(1000)
        swapped = atomic.compare_and_swap(1000, 1001)
        self.assertEqual(1001, atomic.value)
        self.assertEqual(True, swapped)

        swapped = atomic.compare_and_swap(1000, 1024)
        self.assertEqual(1001, atomic.value)
        self.assertEqual(False, swapped)
Esempio n. 7
0
 def __init__(self, reservoir_size, alpha):
     self.values = []
     self.next_scale_time = AtomicLong(0)
     self.alpha = alpha
     self.reservoir_size = reservoir_size
     self.lock = RLock()
     self.rescale_threshold = \
         ExponentiallyDecayingSample.calculate_rescale_threshold(alpha)
     self.clear()
Esempio n. 8
0
 def __init__(self, reservoir_size, alpha):
     self.values = []
     self.next_scale_time = AtomicLong(0)
     self.alpha = alpha
     self.reservoir_size = reservoir_size
     self.lock = RLock()
     self.rescale_threshold = ExponentiallyDecayingSample.calculate_rescale_threshold(alpha)
     self.clear()
Esempio n. 9
0
    def __init__(self, average_class=EWMA):
        self.counter = AtomicLong(0)
        self.start_time = now()
        self.last_tick = AtomicLong(self.start_time)

        self.interval = EWMA.INTERVAL
        self.m1_rate = EWMA.m1()
        self.m5_rate = EWMA.m5()
        self.m15_rate = EWMA.m15()
Esempio n. 10
0
class ToggleGauge(Gauge):
    _value = AtomicLong(1)

    @property
    def value(self):
        try:
            return self._value.value
        finally:
            self._value.value = 0
Esempio n. 11
0
class EWMA(object):
    INTERVAL = 5.0
    SECONDS_PER_MINUTE = 60.0

    ONE_MINUTE = 1
    FIVE_MINUTES = 5
    FIFTEEN_MINUTES = 15

    M1_ALPHA = 1 - math.exp(-INTERVAL / SECONDS_PER_MINUTE / ONE_MINUTE)
    M5_ALPHA = 1 - math.exp(-INTERVAL / SECONDS_PER_MINUTE / FIVE_MINUTES)
    M15_ALPHA = 1 - math.exp(-INTERVAL / SECONDS_PER_MINUTE / FIFTEEN_MINUTES)

    @classmethod
    def m1(cls):
        return EWMA(cls.M1_ALPHA, cls.INTERVAL)

    @classmethod
    def m5(cls):
        return EWMA(cls.M5_ALPHA, cls.INTERVAL)

    @classmethod
    def m15(cls):
        return EWMA(cls.M15_ALPHA, cls.INTERVAL)

    def __init__(self, alpha, interval):
        self.alpha = alpha
        self.interval = interval

        self.initialized = False
        self._rate = 0.0
        self._uncounted = AtomicLong(0)

    def clear(self):
        self.initialized = False
        self._rate = 0.0
        self._uncounted.value = 0

    def update(self, value):
        self._uncounted += value

    def tick(self):
        count = self._uncounted.swap(0)
        instant_rate = count / self.interval

        if self.initialized:
            self._rate += self.alpha * (instant_rate - self._rate)
        else:
            self._rate = instant_rate
            self.initialized = True

    @property
    def rate(self):
        return self._rate
Esempio n. 12
0
 def __init__(self, sample):
     self.sample = sample
     self.counter = AtomicLong(0)
     self.minimum = AtomicLong(sys.maxsize)
     self.maximum = AtomicLong(-sys.maxsize - 1)
     self.sum = AtomicLong(0)
     self.var = AtomicLongArray([-1, 0])
Esempio n. 13
0
    def __init__(self, service_meta, client_id=None):
        self.service_meta = service_meta
        self.client_id = client_id
        if not self.client_id:
            self.client_id = gen_uuid()

        if service_meta.serverimpl != "rabbitmq":
            raise RuntimeError("serverimpl {} not implemented yet".format(
                service_meta.serverimpl))

        self._sender = RabbitmqClientSender(service_meta)
        if service_meta.callmode != "notify":
            # need a queue to receive response
            self._receiver = RabbitmqClientReceiver(self.client_id)
        else:
            self._receiver = None

        if service_meta.coder != "mail":
            raise RuntimeError("coder {} is not supported yet".format(
                service_meta.coder))

        self.requestid_base = AtomicLong(0)
        self.running = True
Esempio n. 14
0
class Derive(Meter):
    """
    A derive is like a meter but accepts an absolute counter as input.

      derive = Metrology.derive('network.io')
      derive.mark()
      derive.count

    """
    def __init__(self, average_class=EWMA):
        self.last = AtomicLong(0)
        super(Derive, self).__init__(average_class)

    def mark(self, value=1):
        """Record an event with the derive.

        :param value: counter value to record
        """
        last = self.last.get_and_set(value)
        if last <= value:
            value = value - last
        super(Derive, self).mark(value)
Esempio n. 15
0
    def __init__(self, service_meta, **kwargs):
        self.raise_if_connect_error = kwargs.get('raise_if_connect_error',True) #add by wxt 2015-12-16 如果初始化失败是否raise异常
        self.running = False
        try:
            self.service_meta = service_meta
            # atexit
            if self.service_meta.callmode == "notify":
                atexit.register(self.batch_notify_flush)

            self.client_id = kwargs.get("client_id")
            if not self.client_id:
                self.client_id = gen_uuid()

            self.client_id = '_client.{}.{}'.format(self.service_meta.name,self.client_id)

            if service_meta.serverimpl == "rabbitmq":
                from . import babelrabbitmq
                self.impl = babelrabbitmq
            elif service_meta.serverimpl == "redis":
                from . import babelredis
                self.impl = babelredis
            else:
                raise RuntimeError("serverimpl {} not implemented yet".format(service_meta.serverimpl))

            if "sdc" not in self.service_meta.options:
                raise RuntimeError("sdc not in service meta")
            self.sdc = self.service_meta.options.get("sdc", "")

            if "cdc" not in self.service_meta.options:
                raise RuntimeError("cdc not in service meta")
            self.cdc = self.service_meta.options.get("cdc", "")

            if "," in self.service_meta.options["sdc"] and self.service_meta.callmode != "notify":
                raise RuntimeError("only notify supports multiple data center")

            self._sender = self.impl.get_client_sender(service_meta, **kwargs)
            if service_meta.callmode != "notify":
                # need a queue to receive response
                self._receiver = self.impl.get_client_receiver(service_meta, self.client_id, **kwargs)
            else:
                self._receiver = None

            if service_meta.coder != "mail":
                raise RuntimeError("coder {} is not supported yet".format(service_meta.coder))

            self.requestid_base = AtomicLong(0)
            self.request_cache = RequestCache()
            self.running = True
            self.response_task = None

            self.metrics_tags = {
                "service": service_meta.name,
                "delivery": service_meta.delivermode,
                "call": service_meta.callmode,
                "impl": service_meta.serverimpl,
                "clientid": self.client_id,
                "sdc": self.sdc,
                "cdc": self.cdc
            }

            # caching requests for batch_notify
            self.batch_cache = list()
            self.running = True
        except Exception,e:
            if self.raise_if_connect_error:
                print e
                raise RuntimeError('babel connect error')
Esempio n. 16
0
class ExponentiallyDecayingSample(object):
    def __init__(self, reservoir_size, alpha):
        self.values = []
        self.next_scale_time = AtomicLong(0)
        self.alpha = alpha
        self.reservoir_size = reservoir_size
        self.lock = RLock()
        self.rescale_threshold = \
            ExponentiallyDecayingSample.calculate_rescale_threshold(alpha)
        self.clear()

    @staticmethod
    def calculate_rescale_threshold(alpha):
        # determine rescale-threshold such that we will not overflow exp() in
        # weight function, and subsequently not overflow into inf on dividing
        # by random.random()
        min_rand = 1.0 / (2**32)  # minimum non-zero value from random()
        safety = 2.0  # safety pad for numerical inaccuracy
        max_value = sys.float_info.max * min_rand / safety
        return int(math.log(max_value) / alpha)

    def clear(self):
        with self.lock:
            self.values = []
            self.start_time = now()
            self.next_scale_time.value = \
                self.start_time + self.rescale_threshold

    def size(self):
        with self.lock:
            return len(self.values)

    def __len__(self):
        return self.size()

    def snapshot(self):
        with self.lock:
            return Snapshot(val for _, val in self.values)

    def weight(self, timestamp):
        return math.exp(self.alpha * (timestamp - self.start_time))

    def rescale(self, now, next_time):
        if self.next_scale_time.compare_and_swap(next_time,
                                                 now + self.rescale_threshold):
            with self.lock:
                rescaleFactor = math.exp(-self.alpha * (now - self.start_time))
                self.values = [(k * rescaleFactor, v) for k, v in self.values]
                self.start_time = now

    def rescale_if_necessary(self):
        time = now()
        next_time = self.next_scale_time.value
        if time > next_time:
            self.rescale(time, next_time)

    def update(self, value, timestamp=None):
        if timestamp is None:
            timestamp = now()

        self.rescale_if_necessary()
        with self.lock:
            try:
                priority = self.weight(timestamp) / random.random()
            except (OverflowError, ZeroDivisionError):
                priority = sys.float_info.max

            if len(self.values) < self.reservoir_size:
                heapq.heappush(self.values, (priority, value))
            else:
                heapq.heappushpop(self.values, (priority, value))
Esempio n. 17
0
 def __init__(self, service_meta):
     self.requestid_gen = AtomicLong(0)
     pass
Esempio n. 18
0
class ExponentiallyDecayingSample(object):
    def __init__(self, reservoir_size, alpha):
        self.values = []
        self.next_scale_time = AtomicLong(0)
        self.alpha = alpha
        self.reservoir_size = reservoir_size
        self.lock = RLock()
        self.rescale_threshold = ExponentiallyDecayingSample.calculate_rescale_threshold(alpha)
        self.clear()

    @staticmethod
    def calculate_rescale_threshold(alpha):
        # determine rescale-threshold such that we will not overflow exp() in
        # weight function, and subsequently not overflow into inf on dividing
        # by random.random()
        min_rand = 1.0 / (2 ** 32)  # minimum non-zero value from random()
        safety = 2.0  # safety pad for numerical inaccuracy
        max_value = sys.float_info.max * min_rand / safety
        return int(math.log(max_value) / alpha)

    def clear(self):
        with self.lock:
            self.values = []
            self.start_time = now()
            self.next_scale_time.value = self.start_time + self.rescale_threshold

    def size(self):
        with self.lock:
            return len(self.values)

    def __len__(self):
        return self.size()

    def snapshot(self):
        with self.lock:
            return Snapshot(val for _, val in self.values)

    def weight(self, timestamp):
        return math.exp(self.alpha * (timestamp - self.start_time))

    def rescale(self, now, next_time):
        if self.next_scale_time.compare_and_swap(next_time, now + self.rescale_threshold):
            with self.lock:
                rescaleFactor = math.exp(-self.alpha * (now - self.start_time))
                self.values = [(k * rescaleFactor, v) for k, v in self.values]
                self.start_time = now

    def rescale_if_necessary(self):
        time = now()
        next_time = self.next_scale_time.value
        if time > next_time:
            self.rescale(time, next_time)

    def update(self, value, timestamp=None):
        if timestamp is None:
            timestamp = now()

        self.rescale_if_necessary()
        with self.lock:
            try:
                priority = self.weight(timestamp) / random.random()
            except (OverflowError, ZeroDivisionError):
                priority = sys.float_info.max

            if len(self.values) < self.reservoir_size:
                heapq.heappush(self.values, (priority, value))
            else:
                heapq.heappushpop(self.values, (priority, value))
Esempio n. 19
0
class Meter(object):
    """A meter measures the rate of events over time
    (e.g., "requests per second").

    In addition to the mean rate, you can also track 1, 5 and 15 minutes moving
    averages ::

      meter = Metrology.meter('requests')
      meter.mark()
      meter.count

    """
    def __init__(self, average_class=EWMA):
        self.counter = AtomicLong(0)
        self.start_time = now()
        self.last_tick = AtomicLong(self.start_time)

        self.interval = EWMA.INTERVAL
        self.m1_rate = EWMA.m1()
        self.m5_rate = EWMA.m5()
        self.m15_rate = EWMA.m15()

    def _tick(self):
        old_tick, new_tick = self.last_tick.value, time()
        age = new_tick - old_tick
        ticks = int(age / self.interval)
        new_tick = old_tick + int(ticks * self.interval)
        if ticks and self.last_tick.compare_and_swap(old_tick, new_tick):
            for _ in range(ticks):
                self.tick()

    def __call__(self, *args, **kwargs):
        if args and hasattr(args[0], '__call__'):
            _orig_func = args[0]

            def _decorator(*args, **kwargs):
                with self:
                    return _orig_func(*args, **kwargs)

            return _decorator

    def __enter__(self):
        pass

    def __exit__(self, exc, exv, trace):
        self.mark()

    @property
    def count(self):
        """Returns the total number of events that have been recorded."""
        return self.counter.value

    def clear(self):
        self.counter.value = 0
        self.start_time = time()

        self.m1_rate.clear()
        self.m5_rate.clear()
        self.m15_rate.clear()

    @ticker
    def mark(self, value=1):
        """Record an event with the meter. By default it will record one event.

        :param value: number of event to record
        """
        self.counter += value
        self.m1_rate.update(value)
        self.m5_rate.update(value)
        self.m15_rate.update(value)

    def tick(self):
        self.m1_rate.tick()
        self.m5_rate.tick()
        self.m15_rate.tick()

    @property
    @ticker
    def one_minute_rate(self):
        """Returns the one-minute average rate."""
        return self.m1_rate.rate

    @property
    @ticker
    def five_minute_rate(self):
        """Returns the five-minute average rate."""
        return self.m5_rate.rate

    @property
    @ticker
    def fifteen_minute_rate(self):
        """Returns the fifteen-minute average rate."""
        return self.m15_rate.rate

    @property
    def mean_rate(self):
        """
        Returns the mean rate of the events since the start of the process.
        """
        if self.counter.value == 0:
            return 0.0
        else:
            elapsed = time() - self.start_time
            return self.counter.value / elapsed

    def stop(self):
        pass
Esempio n. 20
0
 def test_swap(self):
     atomic = AtomicLong(1000)
     swapped = atomic.swap(1001)
     self.assertEqual(1001, atomic.value)
     self.assertEqual(1000, swapped)
Esempio n. 21
0
 def test_value(self):
     atomic = AtomicLong(0)
     atomic.value = 1
     self.assertEqual(1, atomic.value)
Esempio n. 22
0
class UnsafeServiceClient(object):
    def __init__(self, service_meta, client_id=None):
        self.service_meta = service_meta
        self.client_id = client_id
        if not self.client_id:
            self.client_id = gen_uuid()

        if service_meta.serverimpl != "rabbitmq":
            raise RuntimeError("serverimpl {} not implemented yet".format(
                service_meta.serverimpl))

        self._sender = RabbitmqClientSender(service_meta)
        if service_meta.callmode != "notify":
            # need a queue to receive response
            self._receiver = RabbitmqClientReceiver(self.client_id)
        else:
            self._receiver = None

        if service_meta.coder != "mail":
            raise RuntimeError("coder {} is not supported yet".format(
                service_meta.coder))

        self.requestid_base = AtomicLong(0)
        self.running = True

    def gen_next_requestid(self):
        while True:
            result = self.requestid_base.value
            if not self.requestid_base.compare_and_set(result, result + 1):
                continue
            return "{}_{}".format(self.client_id, result)

    def start(self):
        if self._receiver:
            self._receiver.start_consuming()

    def close(self):
        self.running = False

        if self._sender:
            self._sender.close()
            self._sender = None

        if self._receiver:
            self._receiver.close()
            self._receiver = None

    def send(self, request, key, block=True, timeout=10):
        if not self.running:
            raise RuntimeError("the service client is closed")
        requestid = self.gen_next_requestid()
        request_mail = Mail.new_mail(self.client_id, self.service_meta.name,
                                     requestid)
        populate_data_into_mail(request_mail, request)

        expire = millis_now() + int(timeout * 1000)
        request_mail.add_header("expire", str(expire + 5000))

        countOfReplies = 0
        if self._receiver:
            countOfReplies = self.service_meta.options.get(
                "servercardinality", 1)

        try:
            self._sender.send(request_mail.get_json(), key, block, timeout)
        except Queue.Full:
            # it's too busy in the send queue
            raise

        if not countOfReplies:
            return True, None

        receivedEvents = list()
        while self.running and countOfReplies > len(
                receivedEvents) and millis_now() < expire:
            try:
                content = self._receiver.get(True, 1)
                e = self.process_mail(content)
                receivedEvents.append(e)
            except Queue.Empty:
                time.sleep(0.1)
                continue

        if self.service_meta.callmode == "rpc":
            # special treatment for one response
            # no response
            if len(receivedEvents) == 0:
                return False, None

            # remote exception
            response = receivedEvents[0]
            if isinstance(response, Exception):
                raise response

            # success
            return True, response
        else:
            return len(receivedEvents) == countOfReplies, receivedEvents

    def process_mails(self):
        if not self._receiver:
            return

        while self.running:
            try:
                content = self._receiver.get(True, 1)
                self.process_mail(content)
            except Queue.Empty:
                pass

        self._receiver.close()
        for content in self._receiver.dump_cache():
            self.process_mail(content)

    def process_mail(self, mail_injson):
        if not mail_injson:
            return
        response_mail = Mail.from_json(mail_injson)
        try:
            response = extract_data_from_mail(response_mail)
        except Exception as err:
            response = err

        return response

    # useful handlers
    def notify(self, request, key="", block=True, timeout=10):
        assert self.service_meta.callmode == "notify"
        self.send(request, key, block, timeout)

    def rpc(self, request, key="", block=True, timeout=10):
        assert self.service_meta.callmode == "rpc"
        return self.send(request, key, block, timeout)

    def mrpc(self, request, key="", block=True, timeout=10):
        assert self.service_meta.callmode == "mrpc"
        return self.send(request, key, block, timeout)
Esempio n. 23
0
 def __init__(self):
     self._count = AtomicLong(0)
Esempio n. 24
0
class Meter(object):
    """A meter measures the rate of events over time
    (e.g., "requests per second").

    In addition to the mean rate, you can also track 1, 5 and 15 minutes moving
    averages ::

      meter = Metrology.meter('requests')
      meter.mark()
      meter.count

    """
    def __init__(self, average_class=EWMA):
        self.counter = AtomicLong(0)
        self.start_time = now()
        self.last_tick = AtomicLong(self.start_time)

        self.interval = EWMA.INTERVAL
        self.m1_rate = EWMA.m1()
        self.m5_rate = EWMA.m5()
        self.m15_rate = EWMA.m15()

    def _tick(self):
        old_tick, new_tick = self.last_tick.value, time()
        age = new_tick - old_tick
        ticks = int(age / self.interval)
        new_tick = old_tick + int(ticks * self.interval)
        if ticks and self.last_tick.compare_and_swap(old_tick, new_tick):
            for _ in range(ticks):
                self.tick()

    def __call__(self, *args, **kwargs):
        if args and hasattr(args[0], '__call__'):
            _orig_func = args[0]

            def _decorator(*args, **kwargs):
                with self:
                    return _orig_func(*args, **kwargs)
            return _decorator

    def __enter__(self):
        pass

    def __exit__(self, exc, exv, trace):
        self.mark()

    @property
    def count(self):
        """Returns the total number of events that have been recorded."""
        return self.counter.value

    def clear(self):
        self.counter.value = 0
        self.start_time = time()

        self.m1_rate.clear()
        self.m5_rate.clear()
        self.m15_rate.clear()

    @ticker
    def mark(self, value=1):
        """Record an event with the meter. By default it will record one event.

        :param value: number of event to record
        """
        self.counter += value
        self.m1_rate.update(value)
        self.m5_rate.update(value)
        self.m15_rate.update(value)

    def tick(self):
        self.m1_rate.tick()
        self.m5_rate.tick()
        self.m15_rate.tick()

    @property
    @ticker
    def one_minute_rate(self):
        """Returns the one-minute average rate."""
        return self.m1_rate.rate

    @property
    @ticker
    def five_minute_rate(self):
        """Returns the five-minute average rate."""
        return self.m5_rate.rate

    @property
    @ticker
    def fifteen_minute_rate(self):
        """Returns the fifteen-minute average rate."""
        return self.m15_rate.rate

    @property
    def mean_rate(self):
        """
        Returns the mean rate of the events since the start of the process.
        """
        if self.counter.value == 0:
            return 0.0
        else:
            elapsed = time() - self.start_time
            return self.counter.value / elapsed

    def stop(self):
        pass
Esempio n. 25
0
class Histogram(object):
    """
    A histogram measures the statistical distribution of values in a stream of
    data. In addition to minimum, maximum, mean, it also measures median,
    75th, 90th, 95th, 98th, 99th, and 99.9th percentiles ::

      histogram = Metrology.histogram('response-sizes')
      histogram.update(len(response.content))

    Metrology provides two types of histograms: uniform and exponentially
    decaying.
    """
    DEFAULT_SAMPLE_SIZE = 1028
    DEFAULT_ALPHA = 0.015

    def __init__(self, sample):
        self.sample = sample
        self.counter = AtomicLong(0)
        self.minimum = AtomicLong(sys.maxsize)
        self.maximum = AtomicLong(-sys.maxsize - 1)
        self.sum = AtomicLong(0)
        self.var = AtomicLongArray([-1, 0])

    def clear(self):
        self.sample.clear()
        self.counter.value = 0
        self.minimum.value = sys.maxsize
        self.maximum.value = (-sys.maxsize - 1)
        self.sum.value = 0
        self.var.value = [-1, 0]

    def update(self, value):
        self.counter += 1
        self.sample.update(value)
        self.max = value
        self.min = value
        self.sum += value
        self.update_variance(value)

    @property
    def snapshot(self):
        return self.sample.snapshot()

    @property
    def count(self):
        """Return number of values."""
        return self.counter.value

    def get_max(self):
        if self.counter.value > 0:
            return self.maximum.value
        return 0.0

    def set_max(self, potential_max):
        done = False
        while not done:
            current_max = self.maximum.value
            done = (current_max is not None and current_max >= potential_max) \
                or self.maximum.compare_and_swap(current_max, potential_max)

    max = property(get_max, set_max, doc="""Returns the maximun value.""")

    def get_min(self):
        if self.counter.value > 0:
            return self.minimum.value
        return 0.0

    def set_min(self, potential_min):
        done = False
        while not done:
            current_min = self.minimum.value
            done = (current_min is not None and current_min <= potential_min) \
                or self.minimum.compare_and_swap(current_min, potential_min)

    min = property(get_min, set_min, doc="""Returns the minimum value.""")

    @property
    def total(self):
        """Returns the total value."""
        return self.sum.value

    @property
    def mean(self):
        """Returns the mean value."""
        if self.counter.value > 0:
            return self.sum.value / self.counter.value
        return 0.0

    @property
    def stddev(self):
        """Returns the standard deviation."""
        if self.counter.value > 0:
            return self.variance ** .5
        return 0.0

    @property
    def variance(self):
        """Returns variance"""
        if self.counter.value <= 1:
            return 0.0
        return self.var.value[1] / (self.counter.value - 1)

    def update_variance(self, value):
        def variance(old_values):
            if old_values[0] == -1:
                new_values = (value, 0)
            else:
                old_m = old_values[0]
                old_s = old_values[1]

                new_m = old_m + ((value - old_m) / self.counter.value)
                new_s = old_s + ((value - old_m) * (value - new_m))

                new_values = (new_m, new_s)
            return new_values
        self.var.value = variance(self.var.value)
Esempio n. 26
0
 def test_swap(self):
     atomic = AtomicLong(1000)
     swapped = atomic.swap(1001)
     self.assertEqual(1001, atomic.value)
     self.assertEqual(1000, swapped)
Esempio n. 27
0
class StopSignal(object):
    stop_workers = AtomicLong(0)
Esempio n. 28
0
 def test_set_obj(self):
     atomic = AtomicLongArray([-1, 0])
     atomic[1] = AtomicLong(-2)
     self.assertEqual(atomic[1], -2)
Esempio n. 29
0
"""Hive signal handling."""

import logging

from signal import SIGINT
from atomic import AtomicLong

log = logging.getLogger(__name__)

EXCEPTION_THROWN = AtomicLong(0)
FINISH_SIGNAL_DURING_SYNC = AtomicLong(0)


def finish_signals_handler(signal, frame):
    global FINISH_SIGNAL_DURING_SYNC
    FINISH_SIGNAL_DURING_SYNC += 1
    log.info("""
                  **********************************************************
                  CAUGHT {}. PLEASE WAIT... PROCESSING DATA IN QUEUES...
                  **********************************************************
    """.format("SIGINT" if signal == SIGINT else "SIGTERM"))


def set_exception_thrown():
    global EXCEPTION_THROWN
    EXCEPTION_THROWN += 1


def can_continue_thread():
    return EXCEPTION_THROWN.value == 0 and FINISH_SIGNAL_DURING_SYNC.value == 0
Esempio n. 30
0
    def test_init(self):
        atomic = AtomicLong()
        self.assertEqual(0, atomic.value)

        atomic = AtomicLong(0)
        self.assertEqual(0, atomic.value)
Esempio n. 31
0
 def __init__(self, reservoir_size):
     self.counter = AtomicLong(0)
     self.values = [0] * reservoir_size
Esempio n. 32
0
 def __init__(self, average_class=EWMA):
     self.last = AtomicLong(0)
     super(Derive, self).__init__(average_class)
Esempio n. 33
0
 def test_add(self):
     atomic = AtomicLong(1000)
     atomic += 1
     self.assertEqual(1001, atomic.value)
Esempio n. 34
0
class Histogram(object):
    """
    A histogram measures the statistical distribution of values in a stream of data. In addition to minimum, maximum, mean, it also measures median, 75th, 90th, 95th, 98th, 99th, and 99.9th percentiles ::

      histogram = Metrology.histogram('response-sizes')
      histogram.update(len(response.content))

    Metrology provides two types of histograms: uniform and exponentially decaying.
    """
    DEFAULT_SAMPLE_SIZE = 1028
    DEFAULT_ALPHA = 0.015

    def __init__(self, sample):
        self.sample = sample
        self.counter = AtomicLong(0)
        self.minimum = AtomicLong(sys.maxsize)
        self.maximum = AtomicLong(-sys.maxsize - 1)
        self.sum = AtomicLong(0)
        self.var = AtomicLongArray([-1, 0])

    def clear(self):
        self.sample.clear()
        self.counter.value = 0
        self.minimum.value = sys.maxsize
        self.maximum.value = (-sys.maxsize - 1)
        self.sum.value = 0
        self.var.value = [-1, 0]

    def update(self, value):
        self.counter += 1
        self.sample.update(value)
        self.max = value
        self.min = value
        self.sum += value
        self.update_variance(value)

    @property
    def snapshot(self):
        return self.sample.snapshot()

    @property
    def count(self):
        """Return number of values."""
        return self.counter.value

    def get_max(self):
        if self.counter.value > 0:
            return self.maximum.value
        return 0.0

    def set_max(self, potential_max):
        done = False
        while not done:
            current_max = self.maximum.value
            done = (current_max is not None and current_max >= potential_max) \
                or self.maximum.compare_and_swap(current_max, potential_max)

    max = property(get_max, set_max, doc="""Returns the maximun value.""")

    def get_min(self):
        if self.counter.value > 0:
            return self.minimum.value
        return 0.0

    def set_min(self, potential_min):
        done = False
        while not done:
            current_min = self.minimum.value
            done = (current_min is not None and current_min <= potential_min) \
                or self.minimum.compare_and_swap(current_min, potential_min)

    min = property(get_min, set_min, doc="""Returns the minimum value.""")

    @property
    def mean(self):
        """Returns the mean value."""
        if self.counter.value > 0:
            return self.sum.value / self.counter.value
        return 0.0

    @property
    def stddev(self):
        """Returns the standard deviation."""
        if self.counter.value > 0:
            return self.variance**.5
        return 0.0

    @property
    def variance(self):
        """Returns variance"""
        if self.counter.value <= 1:
            return 0.0
        return self.var.value[1] / (self.counter.value - 1)

    def update_variance(self, value):
        def variance(old_values):
            if old_values[0] == -1:
                new_values = (value, 0)
            else:
                old_m = old_values[0]
                old_s = old_values[1]

                new_m = old_m + ((value - old_m) / self.counter.value)
                new_s = old_s + ((value - old_m) * (value - new_m))

                new_values = (new_m, new_s)
            return new_values

        self.var.value = variance(self.var.value)
Esempio n. 35
0
 def test_sub(self):
     atomic = AtomicLong(1000)
     atomic -= 1
     self.assertEqual(999, atomic.value)
Esempio n. 36
0
 def test_value(self):
     atomic = AtomicLong(0)
     atomic.value = 1
     self.assertEqual(1, atomic.value)