コード例 #1
0
ファイル: connection.py プロジェクト: Xooxy/lymph
class Connection(object):
    def __init__(self, server, endpoint, heartbeat_interval=1, timeout=3, idle_timeout=10, unresponsive_disconnect=30, idle_disconnect=60):
        assert heartbeat_interval < timeout < idle_timeout
        self.server = server
        self.endpoint = endpoint
        self.timeout = timeout
        self.heartbeat_interval = heartbeat_interval
        self.idle_timeout = idle_timeout
        self.unresponsive_disconnect = unresponsive_disconnect
        self.idle_disconnect = idle_disconnect

        now = time.monotonic()
        self.last_seen = 0
        self.idle_since = 0
        self.last_message = now
        self.created_at = now
        self.heartbeat_samples = SampleWindow(100, factor=1000)  # milliseconds
        self.explicit_heartbeat_count = 0
        self.status = UNKNOWN

        self.received_message_count = 0
        self.sent_message_count = 0

        self.heartbeat_loop_greenlet = self.server.spawn(self.heartbeat_loop)
        self.live_check_loop_greenlet = self.server.spawn(self.live_check_loop)

        self.pid = os.getpid()

    def __str__(self):
        return "connection to=%s last_seen=%s" % (self.endpoint, self._dt())

    def _dt(self):
        return time.monotonic() - self.last_seen

    @property
    def phi(self):
        p = self.heartbeat_samples.p(self._dt())
        if p == 0:
            return float('inf')
        return -math.log10(p)

    def set_status(self, status):
        self.status = status

    def heartbeat_loop(self):
        while True:
            start = time.monotonic()
            channel = self.server.ping(self.endpoint)
            error = False
            try:
                channel.get(timeout=self.heartbeat_interval)
            except RpcError as e:
                logger.debug('hearbeat error on %s: %r', self, e)
                error = True
            took = time.monotonic() - start
            if not error:
                self.heartbeat_samples.add(took)
                self.explicit_heartbeat_count += 1
            gevent.sleep(max(0, self.heartbeat_interval - took))

    def live_check_loop(self):
        while True:
            self.update_status()
            self.log_stats()
            gevent.sleep(1)

    def update_status(self):
        if self.last_seen:
            now = time.monotonic()
            if now - self.last_seen >= self.timeout:
                self.set_status(UNRESPONSIVE)
            elif now - self.last_message >= self.idle_timeout:
                self.set_status(IDLE)
                self.idle_since = now
            else:
                self.set_status(RESPONSIVE)

    def log_stats(self):
        roundtrip_stats = 'window (mean rtt={mean:.1f} ms; stddev rtt={stddev:.1f})'.format(**self.heartbeat_samples.stats)
        roundtrip_total_stats = 'total (mean rtt={mean:.1f} ms; stddev rtt={stddev:.1f})'.format(**self.heartbeat_samples.total.stats)
        logger.debug("pid=%s; endpoint=%s; %s; %s; phi=%.3f; ping/s=%.2f; status=%s" % (
            self.pid,
            self.endpoint,
            roundtrip_stats,
            roundtrip_total_stats,
            self.phi,
            self.explicit_heartbeat_count / max(1, time.monotonic() - self.created_at),
            self.status,
        ))

    def close(self):
        if self.status == CLOSED:
            return
        self.status = CLOSED
        self.heartbeat_loop_greenlet.kill()
        self.live_check_loop_greenlet.kill()
        self.server.disconnect(self.endpoint)

    def on_recv(self, msg):
        now = time.monotonic()
        self.last_seen = now
        if not msg.is_idle_chatter():
            self.last_message = now
        self.received_message_count += 1

    def on_send(self, msg):
        if not msg.is_idle_chatter():
            self.last_message = time.monotonic()
        self.sent_message_count += 1

    def is_alive(self):
        return self.status in (RESPONSIVE, IDLE, UNKNOWN)

    def stats(self):
        # FIXME: rtt and phi should be recorded as summary/histogram for all connections
        return {
            'endpoint': self.endpoint,
            'rtt': self.heartbeat_samples.stats,
            'phi': self.phi,
            'status': self.status,
            'sent': self.sent_message_count,
            'received': self.received_message_count,
        }
コード例 #2
0
class Connection(object):
    def __init__(self, server, endpoint, heartbeat_interval=1, timeout=3, idle_timeout=10, unresponsive_disconnect=30, idle_disconnect=60):
        assert heartbeat_interval < timeout < idle_timeout
        self.server = server
        self.endpoint = endpoint
        self.timeout = timeout
        self.heartbeat_interval = heartbeat_interval
        self.idle_timeout = idle_timeout
        self.unresponsive_disconnect = unresponsive_disconnect
        self.idle_disconnect = idle_disconnect

        now = time.monotonic()
        self.last_seen = 0
        self.idle_since = 0
        self.last_message = now
        self.created_at = now
        self.heartbeat_samples = SampleWindow(100, factor=1000)  # milliseconds
        self.explicit_heartbeat_count = 0
        self.status = UNKNOWN

        self.received_message_count = 0
        self.sent_message_count = 0

        if self.heartbeat_interval:
            self.heartbeat_loop_greenlet = self.server.spawn(self.heartbeat_loop)
            self.live_check_loop_greenlet = self.server.spawn(self.live_check_loop)
        else:
            self.heartbeat_loop_greenlet = None
            self.live_check_loop_greenlet = None

        self.pid = os.getpid()

    def __str__(self):
        return "connection to=%s last_seen=%s" % (self.endpoint, self._dt())

    def _dt(self):
        return time.monotonic() - self.last_seen

    @property
    def phi(self):
        p = self.heartbeat_samples.p(self._dt())
        if p == 0:
            return float('inf')
        return -math.log10(p)

    def set_status(self, status):
        if status != self.status:
            logger.info('changing connection status to %r endpoint=%s', status, self.endpoint)
        self.status = status

    def heartbeat_loop(self):
        while True:
            start = time.monotonic()
            channel = self.server.ping(self.endpoint)
            error = False
            try:
                channel.get(timeout=self.heartbeat_interval)
            except RpcError as e:
                logger.debug('hearbeat error on %s: %r', self, e)
                error = True
            took = time.monotonic() - start
            if not error:
                self.heartbeat_samples.add(took)
                self.explicit_heartbeat_count += 1
            gevent.sleep(max(0, self.heartbeat_interval - took))

    def live_check_loop(self):
        while True:
            self.update_status()
            self.log_stats()
            gevent.sleep(1)

    def update_status(self):
        if self.last_seen:
            now = time.monotonic()
            if now - self.last_seen >= self.timeout:
                self.set_status(UNRESPONSIVE)
            elif now - self.last_message >= self.idle_timeout:
                self.set_status(IDLE)
                self.idle_since = now
            else:
                self.set_status(RESPONSIVE)

    def log_stats(self):
        roundtrip_stats = 'window (mean rtt={mean:.1f} ms; stddev rtt={stddev:.1f})'.format(**self.heartbeat_samples.stats)
        roundtrip_total_stats = 'total (mean rtt={mean:.1f} ms; stddev rtt={stddev:.1f})'.format(**self.heartbeat_samples.total.stats)
        logger.debug("pid=%s; endpoint=%s; %s; %s; phi=%.3f; ping/s=%.2f; status=%s" % (
            self.pid,
            self.endpoint,
            roundtrip_stats,
            roundtrip_total_stats,
            self.phi,
            self.explicit_heartbeat_count / max(1, time.monotonic() - self.created_at),
            self.status,
        ))

    def close(self):
        if self.status == CLOSED:
            return
        self.status = CLOSED
        if self.heartbeat_loop_greenlet:
            self.heartbeat_loop_greenlet.kill()
            self.live_check_loop_greenlet.kill()
        self.server.disconnect(self.endpoint)

    def on_recv(self, msg):
        now = time.monotonic()
        self.last_seen = now
        if not msg.is_idle_chatter():
            self.last_message = now
        self.received_message_count += 1

    def on_send(self, msg):
        if not msg.is_idle_chatter():
            self.last_message = time.monotonic()
        self.sent_message_count += 1

    def is_alive(self):
        return self.status in (RESPONSIVE, IDLE, UNKNOWN)

    def stats(self):
        # FIXME: rtt and phi should be recorded as summary/histogram for all connections
        return {
            'endpoint': self.endpoint,
            'rtt': self.heartbeat_samples.stats,
            'phi': self.phi,
            'status': self.status,
            'sent': self.sent_message_count,
            'received': self.received_message_count,
        }
コード例 #3
0
ファイル: connection.py プロジェクト: TimBeyer/lymph
class Connection(object):
    def __init__(self, container, endpoint, heartbeat_interval=1, timeout=1, idle_timeout=10, unresponsive_disconnect=30, idle_disconnect=60):
        self.container = container
        self.endpoint = endpoint
        self.timeout = timeout
        self.heartbeat_interval = heartbeat_interval
        self.idle_timeout = idle_timeout
        self.unresponsive_disconnect = unresponsive_disconnect
        self.idle_disconnect = idle_disconnect

        now = time.monotonic()
        self.last_seen = 0
        self.idle_since = 0
        self.last_message = now
        self.created_at = now
        self.heartbeat_samples = SampleWindow(100, factor=1000)  # milliseconds
        self.roundtrip_samples = SampleWindow(100, factor=1000)  # milliseconds
        self.explicit_heartbeat_count = 0
        self.status = UNKNOWN

        self.received_message_count = 0
        self.sent_message_count = 0

        self.heartbeat_loop_greenlet = self.container.spawn(self.heartbeat_loop)
        self.live_check_loop_greenlet = self.container.spawn(self.live_check_loop)

    def __str__(self):
        return "connection to=%s last_seen=%s" % (self.endpoint, self._dt())

    def _dt(self):
        return time.monotonic() - self.last_seen

    @property
    def phi(self):
        p = self.heartbeat_samples.p(self._dt())
        if p == 0:
            return float('inf')
        return -math.log10(p)

    def set_status(self, status):
        self.status = status

    def heartbeat_loop(self):
        while True:
            start = time.monotonic()
            channel = self.container.ping(self.endpoint)
            try:
                channel.get(timeout=self.heartbeat_interval)
            except RpcError:
                pass
            else:
                self.roundtrip_samples.add(time.monotonic() - start)
                self.explicit_heartbeat_count += 1
            gevent.sleep(max(self.heartbeat_interval - .001 * self.roundtrip_samples.mean, MIN_HEARTBEAT_INTERVAL))

    def live_check_loop(self):
        while True:
            if self.last_seen:
                now = time.monotonic()
                if now - self.last_seen >= self.timeout:
                    self.set_status(UNRESPONSIVE)
                elif now - self.last_message >= self.idle_timeout:
                    self.set_status(IDLE)
                    self.idle_since = now
                else:
                    self.set_status(RESPONSIVE)
            heartbeat_stats = 'window (mean ♡ = {mean:.1f} ms; stddev ♡ = {stddev:.1f})'.format(**self.heartbeat_samples.stats)
            heartbeat_total_stats = 'total (mean ♡ = {mean:.1f} ms; stddev ♡ = {stddev:.1f})'.format(**self.heartbeat_samples.total.stats)
            roundtrip_stats = 'mean rtt = {mean:.3f} ms; stddev rtt = {stddev:.3f}'.format(**self.roundtrip_samples.stats)
            logger.debug("pid=%s; %s; %s; %s; φ = %.3f; ping/s = %.2f; status=%s" % (
                os.getpid(),
                roundtrip_stats,
                heartbeat_stats,
                heartbeat_total_stats,
                self.phi,
                self.explicit_heartbeat_count / (time.monotonic() - self.created_at),
                self.status,
            ))
            gevent.sleep(self.timeout)

    def close(self):
        if self.status == CLOSED:
            return
        self.status = CLOSED
        self.heartbeat_loop_greenlet.kill()
        self.live_check_loop_greenlet.kill()
        self.container.disconnect(self.endpoint)

    def on_recv(self, msg):
        now = time.monotonic()
        if self.last_seen:
            self.heartbeat_samples.add(now - self.last_seen)
        self.last_seen = now
        if not msg.is_idle_chatter():
            self.last_message = now
        self.received_message_count += 1

    def on_send(self, msg):
        if not msg.is_idle_chatter():
            self.last_message = time.monotonic()
        self.sent_message_count += 1

    def is_alive(self):
        return self.status in (RESPONSIVE, IDLE)

    def stats(self):
        return {
            'endpoint': self.endpoint,
            'rtt': self.roundtrip_samples.stats,
            'heartbeat': self.heartbeat_samples.stats,
            'phi': self.phi,
            'status': self.status,
            'sent': self.sent_message_count,
            'received': self.received_message_count,
        }
コード例 #4
0
class Connection(object):
    def __init__(self,
                 container,
                 endpoint,
                 heartbeat_interval=1,
                 timeout=1,
                 idle_timeout=10,
                 unresponsive_disconnect=30,
                 idle_disconnect=60):
        self.container = container
        self.endpoint = endpoint
        self.timeout = timeout
        self.heartbeat_interval = heartbeat_interval
        self.idle_timeout = idle_timeout
        self.unresponsive_disconnect = unresponsive_disconnect
        self.idle_disconnect = idle_disconnect

        now = time.monotonic()
        self.last_seen = 0
        self.idle_since = 0
        self.last_message = now
        self.created_at = now
        self.heartbeat_samples = SampleWindow(100, factor=1000)  # milliseconds
        self.roundtrip_samples = SampleWindow(100, factor=1000)  # milliseconds
        self.explicit_heartbeat_count = 0
        self.status = UNKNOWN

        self.received_message_count = 0
        self.sent_message_count = 0

        self.heartbeat_loop_greenlet = self.container.spawn(
            self.heartbeat_loop)
        self.live_check_loop_greenlet = self.container.spawn(
            self.live_check_loop)

    def __str__(self):
        return "connection to=%s last_seen=%s" % (self.endpoint, self._dt())

    def _dt(self):
        return time.monotonic() - self.last_seen

    @property
    def phi(self):
        p = self.heartbeat_samples.p(self._dt())
        if p == 0:
            return float('inf')
        return -math.log10(p)

    def set_status(self, status):
        self.status = status

    def heartbeat_loop(self):
        while True:
            start = time.monotonic()
            channel = self.container.ping(self.endpoint)
            try:
                channel.get(timeout=self.heartbeat_interval)
            except RpcError:
                pass
            else:
                self.roundtrip_samples.add(time.monotonic() - start)
                self.explicit_heartbeat_count += 1
            gevent.sleep(
                max(
                    self.heartbeat_interval -
                    .001 * self.roundtrip_samples.mean,
                    MIN_HEARTBEAT_INTERVAL))

    def live_check_loop(self):
        while True:
            if self.last_seen:
                now = time.monotonic()
                if now - self.last_seen >= self.timeout:
                    self.set_status(UNRESPONSIVE)
                elif now - self.last_message >= self.idle_timeout:
                    self.set_status(IDLE)
                    self.idle_since = now
                else:
                    self.set_status(RESPONSIVE)
            heartbeat_stats = 'window (mean ♡ = {mean:.1f} ms; stddev ♡ = {stddev:.1f})'.format(
                **self.heartbeat_samples.stats)
            heartbeat_total_stats = 'total (mean ♡ = {mean:.1f} ms; stddev ♡ = {stddev:.1f})'.format(
                **self.heartbeat_samples.total.stats)
            roundtrip_stats = 'mean rtt = {mean:.3f} ms; stddev rtt = {stddev:.3f}'.format(
                **self.roundtrip_samples.stats)
            logger.debug(
                "pid=%s; %s; %s; %s; φ = %.3f; ping/s = %.2f; status=%s" % (
                    os.getpid(),
                    roundtrip_stats,
                    heartbeat_stats,
                    heartbeat_total_stats,
                    self.phi,
                    self.explicit_heartbeat_count /
                    (time.monotonic() - self.created_at),
                    self.status,
                ))
            gevent.sleep(self.timeout)

    def close(self):
        if self.status == CLOSED:
            return
        self.status = CLOSED
        self.heartbeat_loop_greenlet.kill()
        self.live_check_loop_greenlet.kill()
        self.container.disconnect(self.endpoint)

    def on_recv(self, msg):
        now = time.monotonic()
        if self.last_seen:
            self.heartbeat_samples.add(now - self.last_seen)
        self.last_seen = now
        if not msg.is_idle_chatter():
            self.last_message = now
        self.received_message_count += 1

    def on_send(self, msg):
        if not msg.is_idle_chatter():
            self.last_message = time.monotonic()
        self.sent_message_count += 1

    def is_alive(self):
        return self.status in (RESPONSIVE, IDLE)

    def stats(self):
        return {
            'endpoint': self.endpoint,
            'rtt': self.roundtrip_samples.stats,
            'heartbeat': self.heartbeat_samples.stats,
            'phi': self.phi,
            'status': self.status,
            'sent': self.sent_message_count,
            'received': self.received_message_count,
        }