def __heartbeat_loop(self):
"""
Main loop for sending (and monitoring received) heartbeats.
"""
now = monotonic()
# Setup the initial due time for the outbound heartbeat
if self.send_sleep != 0:
self.next_outbound_heartbeat = now + self.send_sleep
while self.running:
now = monotonic()
next_events = []
if self.next_outbound_heartbeat is not None:
next_events.append(self.next_outbound_heartbeat - now)
if self.receive_sleep != 0:
t = self.received_heartbeat + self.receive_sleep - now
if t > 0:
next_events.append(t)
sleep_time = min(next_events)
if sleep_time > 0:
time.sleep(sleep_time)
now = monotonic()
if not self.transport.is_connected():
time.sleep(self.send_sleep)
continue
if self.send_sleep != 0 and now > self.next_outbound_heartbeat:
log.debug("Sending a heartbeat message at %s", now)
try:
self.transport.transmit(utils.Frame(None, {}, None))
except exception.NotConnectedException:
log.debug("Lost connection, unable to send heartbeat")
except Exception:
_, e, _ = sys.exc_info()
log.debug("Unable to send heartbeat, due to: %s", e)
if self.receive_sleep != 0:
diff_receive = now - self.received_heartbeat
if diff_receive > self.receive_sleep:
# heartbeat timeout
log.warning(
"Heartbeat timeout: diff_receive=%s, time=%s, lastrec=%s",
diff_receive, now, self.received_heartbeat)
self.transport.set_connected(False)
self.transport.disconnect_socket()
self.transport.stop()
for listener in self.transport.listeners.values():
listener.on_heartbeat_timeout()
self.heartbeat_thread = None
def __update_heartbeat(self):
# Honour any grace that has been already included
if self.received_heartbeat is None:
return
now = monotonic()
if now > self.received_heartbeat:
self.received_heartbeat = now
def on_connected(self, headers, body):
"""
Once the connection is established, and 'heart-beat' is found in the headers, we calculate the real
heartbeat numbers (based on what the server sent and what was specified by the client) - if the heartbeats
are not 0, we start up the heartbeat loop accordingly.
:param dict headers: headers in the connection message
:param body: the message body
"""
if 'heart-beat' in headers:
self.heartbeats = utils.calculate_heartbeats(
headers['heart-beat'].replace(' ', '').split(','),
self.heartbeats)
if self.heartbeats != (0, 0):
self.send_sleep = self.heartbeats[0] / 1000
# receive gets an additional grace of 50%
self.receive_sleep = (self.heartbeats[1] / 1000) * 1.5
# Give grace of receiving the first heartbeat
self.received_heartbeat = monotonic() + self.receive_sleep
self.running = True
if self.heartbeat_thread is None:
self.heartbeat_thread = utils.default_create_thread(
self.__heartbeat_loop)
self.heartbeat_thread.name = "StompHeartbeat%s" % \
getattr(self.heartbeat_thread, "name", "Thread")
def on_send(self, frame):
"""
Add the heartbeat header to the frame when connecting, and bump
next outbound heartbeat timestamp.
:param Frame frame: the Frame object
"""
if frame.cmd == CMD_CONNECT or frame.cmd == CMD_STOMP:
if self.heartbeats != (0, 0):
frame.headers[HDR_HEARTBEAT] = '%s,%s' % self.heartbeats
if self.next_outbound_heartbeat is not None:
self.next_outbound_heartbeat = monotonic() + self.send_sleep
def attempt_connection(self):
"""
Try connecting to the (host, port) tuples specified at construction time.
"""
self.connection_error = False
sleep_exp = 1
connect_count = 0
while self.running and self.socket is None and connect_count < self.__reconnect_attempts_max:
for host_and_port in self.__host_and_ports:
try:
log.info("Attempting connection to host %s, port %s",
host_and_port[0], host_and_port[1])
self.socket = get_socket(host_and_port[0],
host_and_port[1], self.__timeout)
self.__enable_keepalive()
need_ssl = self.__need_ssl(host_and_port)
if need_ssl: # wrap socket
ssl_params = self.get_ssl(host_and_port)
if ssl_params['ca_certs']:
cert_validation = ssl.CERT_REQUIRED
else:
cert_validation = ssl.CERT_NONE
try:
tls_context = ssl.create_default_context(
cafile=ssl_params['ca_certs'])
except AttributeError:
tls_context = None
if tls_context:
# Wrap the socket for TLS
certfile = ssl_params['cert_file']
keyfile = ssl_params['key_file']
password = ssl_params.get('password')
if certfile and not keyfile:
keyfile = certfile
if certfile:
tls_context.load_cert_chain(
certfile, keyfile, password)
if cert_validation is None or cert_validation == ssl.CERT_NONE:
tls_context.check_hostname = False
tls_context.verify_mode = cert_validation
self.socket = tls_context.wrap_socket(
self.socket, server_hostname=host_and_port[0])
else:
# Old-style wrap_socket where we don't have a modern SSLContext (so no SNI)
self.socket = ssl.wrap_socket(
self.socket,
keyfile=ssl_params['key_file'],
certfile=ssl_params['cert_file'],
cert_reqs=cert_validation,
ca_certs=ssl_params['ca_certs'],
ssl_version=ssl_params['ssl_version'])
self.socket.settimeout(self.__timeout)
if self.blocking is not None:
self.socket.setblocking(self.blocking)
#
# Validate server cert
#
if need_ssl and ssl_params['cert_validator']:
cert = self.socket.getpeercert()
(ok, errmsg) = ssl_params['cert_validator'](
cert, host_and_port[0])
if not ok:
raise SSLError(
"Server certificate validation failed: %s",
errmsg)
self.current_host_and_port = host_and_port
log.info("Established connection to host %s, port %s",
host_and_port[0], host_and_port[1])
break
except socket.error:
self.socket = None
connect_count += 1
log.warning("Could not connect to host %s, port %s",
host_and_port[0],
host_and_port[1],
exc_info=1)
if self.socket is None:
sleep_duration = (min(self.__reconnect_sleep_max, (
(self.__reconnect_sleep_initial /
(1.0 + self.__reconnect_sleep_increase)) *
math.pow(1.0 + self.__reconnect_sleep_increase, sleep_exp)
)) * (1.0 + random.random() * self.__reconnect_sleep_jitter))
sleep_end = monotonic() + sleep_duration
log.debug(
"Sleeping for %.1f seconds before attempting reconnect",
sleep_duration)
while self.running and monotonic() < sleep_end:
time.sleep(0.2)
if sleep_duration < self.__reconnect_sleep_max:
sleep_exp += 1
if not self.socket:
raise exception.ConnectFailedException()