def __init__(self, container, endpoint, heartbeat_interval=1, timeout=1, idle_timeout=10, unresponsive_disconnect=30, idle_disconnect=60):
self.container = container
self.endpoint = endpoint
self.timeout = timeout
self.heartbeat_interval = heartbeat_interval
self.idle_timeout = idle_timeout
self.unresponsive_disconnect = unresponsive_disconnect
self.idle_disconnect = idle_disconnect
now = time.monotonic()
self.last_seen = 0
self.idle_since = 0
self.last_message = now
self.created_at = now
self.heartbeat_samples = SampleWindow(100, factor=1000) # milliseconds
self.roundtrip_samples = SampleWindow(100, factor=1000) # milliseconds
self.explicit_heartbeat_count = 0
self.status = UNKNOWN
self.received_message_count = 0
self.sent_message_count = 0
self.heartbeat_loop_greenlet = gevent.spawn(self.heartbeat_loop)
self.live_check_loop_greenlet = gevent.spawn(self.live_check_loop)
class Connection(object):
def __init__(self, container, endpoint, heartbeat_interval=1, timeout=1, idle_timeout=10, unresponsive_disconnect=30, idle_disconnect=60):
self.container = container
self.endpoint = endpoint
self.timeout = timeout
self.heartbeat_interval = heartbeat_interval
self.idle_timeout = idle_timeout
self.unresponsive_disconnect = unresponsive_disconnect
self.idle_disconnect = idle_disconnect
now = time.monotonic()
self.last_seen = 0
self.idle_since = 0
self.last_message = now
self.created_at = now
self.heartbeat_samples = SampleWindow(100, factor=1000) # milliseconds
self.roundtrip_samples = SampleWindow(100, factor=1000) # milliseconds
self.explicit_heartbeat_count = 0
self.status = UNKNOWN
self.received_message_count = 0
self.sent_message_count = 0
self.heartbeat_loop_greenlet = gevent.spawn(self.heartbeat_loop)
self.live_check_loop_greenlet = gevent.spawn(self.live_check_loop)
def __str__(self):
return "connection to=%s last_seen=%s" % (self.endpoint, self._dt())
def _dt(self):
return time.monotonic() - self.last_seen
@property
def phi(self):
p = self.heartbeat_samples.p(self._dt())
if p == 0:
return float('inf')
return -math.log10(p)
def set_status(self, status):
self.status = status
def heartbeat_loop(self):
while True:
start = time.monotonic()
channel = self.container.ping(self.endpoint)
try:
channel.get(timeout=self.heartbeat_interval)
except RpcError:
pass
else:
self.roundtrip_samples.add(time.monotonic() - start)
self.explicit_heartbeat_count += 1
gevent.sleep(max(self.heartbeat_interval - .001 * self.roundtrip_samples.mean, MIN_HEARTBEAT_INTERVAL))
def live_check_loop(self):
while True:
if self.last_seen:
now = time.monotonic()
if now - self.last_seen >= self.timeout:
self.set_status(UNRESPONSIVE)
elif now - self.last_message >= self.idle_timeout:
self.set_status(IDLE)
self.idle_since = now
else:
self.set_status(RESPONSIVE)
heartbeat_stats = 'window (mean ♡ = {mean:.1f} ms; stddev ♡ = {stddev:.1f})'.format(**self.heartbeat_samples.stats)
heartbeat_total_stats = 'total (mean ♡ = {mean:.1f} ms; stddev ♡ = {stddev:.1f})'.format(**self.heartbeat_samples.total.stats)
roundtrip_stats = 'mean rtt = {mean:.3f} ms; stddev rtt = {stddev:.3f}'.format(**self.roundtrip_samples.stats)
logger.debug("pid=%s; %s; %s; %s; φ = %.3f; ping/s = %.2f; status=%s" % (
os.getpid(),
roundtrip_stats,
heartbeat_stats,
heartbeat_total_stats,
self.phi,
self.explicit_heartbeat_count / (time.monotonic() - self.created_at),
self.status,
))
gevent.sleep(self.timeout)
def close(self):
if self.status == CLOSED:
return
self.status = CLOSED
self.heartbeat_loop_greenlet.kill()
self.live_check_loop_greenlet.kill()
self.container.disconnect(self.endpoint)
def on_recv(self, msg):
now = time.monotonic()
if self.last_seen:
self.heartbeat_samples.add(now - self.last_seen)
self.last_seen = now
if not msg.is_idle_chatter():
self.last_message = now
self.received_message_count += 1
def on_send(self, msg):
if not msg.is_idle_chatter():
self.last_message = time.monotonic()
self.sent_message_count += 1
def is_alive(self):
return self.status in (RESPONSIVE, IDLE)
def stats(self):
return {
'endpoint': self.endpoint,
'rtt': self.roundtrip_samples.stats,
'heartbeat': self.heartbeat_samples.stats,
'phi': self.phi,
'status': self.status,
'sent': self.sent_message_count,
'received': self.received_message_count,
}