def __heartbeat_loop(self):
"""
Main loop for sending (and monitoring received) heartbeats.
"""
logging.info('Starting heartbeat loop')
now = monotonic()
# Setup the initial due time for the outbound heartbeat
if self.send_sleep != 0:
self.next_outbound_heartbeat = now + self.send_sleep
while self.running:
now = monotonic()
next_events = []
if self.next_outbound_heartbeat is not None:
next_events.append(self.next_outbound_heartbeat - now)
if self.receive_sleep != 0:
t = self.received_heartbeat + self.receive_sleep - now
if t > 0:
next_events.append(t)
sleep_time = min(next_events)
if sleep_time > 0:
terminate = self.heartbeat_terminate_event.wait(sleep_time)
if terminate:
break
now = monotonic()
if not self.transport.is_connected():
time.sleep(self.send_sleep)
continue
if self.send_sleep != 0 and now > self.next_outbound_heartbeat:
logging.debug("Sending a heartbeat message at %s", now)
try:
self.transport.transmit(utils.Frame(None, {}, None))
except exception.NotConnectedException:
logging.debug("Lost connection, unable to send heartbeat")
except Exception:
_, e, _ = sys.exc_info()
logging.debug("Unable to send heartbeat, due to: %s", e)
if self.receive_sleep != 0:
diff_receive = now - self.received_heartbeat
if diff_receive > self.receive_sleep:
# heartbeat timeout
logging.warning(
"Heartbeat timeout: diff_receive=%s, time=%s, lastrec=%s",
diff_receive, now, self.received_heartbeat)
self.transport.set_connected(False)
self.transport.disconnect_socket()
self.transport.stop()
for listener in self.transport.listeners.values():
listener.on_heartbeat_timeout()
self.heartbeat_thread = None
self.heartbeat_terminate_event.clear()
logging.info('Heartbeat loop ended')
def process_frame(self, f, frame_str):
"""
:param Frame f: Frame object
:param bytes frame_str: raw frame content
"""
frame_type = f.cmd.lower()
if frame_type in ["connected", "message", "receipt", "error", "heartbeat"]:
if frame_type == "message":
self.notify("before_message", f)
if logging.isEnabledFor(logging.DEBUG):
logging.debug("Received frame: %r, headers=%r, body=%r", f.cmd, f.headers, f.body)
else:
logging.info("Received frame: %r, len(body)=%r", f.cmd, length(f.body))
self.notify(frame_type, f)
else:
logging.warning("Unknown response frame type: '%s' (frame length was %d)", frame_type, length(frame_str))
def disconnect_socket(self):
"""
Disconnect the underlying socket connection
"""
self.running = False
if self.socket is not None:
if self.__need_ssl():
#
# Even though we don't want to use the socket, unwrap is the only API method which does a proper SSL
# shutdown
#
try:
self.socket = self.socket.unwrap()
except Exception:
#
# unwrap seems flaky on Win with the back-ported ssl mod, so catch any exception and log it
#
_, e, _ = sys.exc_info()
logging.warning(e)
elif hasattr(socket, 'SHUT_RDWR'):
try:
self.socket.shutdown(socket.SHUT_RDWR)
except socket.error:
_, e, _ = sys.exc_info()
# ignore when socket already closed
if get_errno(e) != errno.ENOTCONN:
logging.warning(
"Unable to issue SHUT_RDWR on socket because of error '%s'",
e)
#
# split this into a separate check, because sometimes the socket is nulled between shutdown and this call
#
if self.socket is not None:
try:
self.socket.close()
except socket.error:
_, e, _ = sys.exc_info()
logging.warning("Unable to close socket because of error '%s'",
e)
self.current_host_and_port = None
self.socket = None
self.notify('disconnected')
def attempt_connection(self):
"""
Try connecting to the (host, port) tuples specified at construction time.
"""
self.connection_error = False
sleep_exp = 1
connect_count = 0
while self.running and self.socket is None and (
connect_count < self.__reconnect_attempts_max
or self.__reconnect_attempts_max == -1):
for host_and_port in self.__host_and_ports:
try:
logging.info("Attempting connection to host %s, port %s",
host_and_port[0], host_and_port[1])
self.socket = socket.create_connection(
host_and_port, self.__timeout)
self.__enable_keepalive()
need_ssl = self.__need_ssl(host_and_port)
if need_ssl: # wrap socket
ssl_params = self.get_ssl(host_and_port)
if ssl_params['ca_certs']:
cert_validation = ssl.CERT_REQUIRED
else:
cert_validation = ssl.CERT_NONE
try:
tls_context = ssl.create_default_context(
cafile=ssl_params['ca_certs'])
except AttributeError:
tls_context = None
if tls_context:
# Wrap the socket for TLS
certfile = ssl_params['cert_file']
keyfile = ssl_params['key_file']
password = ssl_params.get('password')
if certfile and not keyfile:
keyfile = certfile
if certfile:
tls_context.load_cert_chain(
certfile, keyfile, password)
if cert_validation is None or cert_validation == ssl.CERT_NONE:
tls_context.check_hostname = False
tls_context.verify_mode = cert_validation
self.socket = tls_context.wrap_socket(
self.socket, server_hostname=host_and_port[0])
else:
# Old-style wrap_socket where we don't have a modern SSLContext (so no SNI)
self.socket = ssl.wrap_socket(
self.socket,
keyfile=ssl_params['key_file'],
certfile=ssl_params['cert_file'],
cert_reqs=cert_validation,
ca_certs=ssl_params['ca_certs'],
ssl_version=ssl_params['ssl_version'])
self.socket.settimeout(self.__timeout)
if self.blocking is not None:
self.socket.setblocking(self.blocking)
#
# Validate server cert
#
if need_ssl and ssl_params['cert_validator']:
cert = self.socket.getpeercert()
(ok, errmsg) = ssl_params['cert_validator'](
cert, host_and_port[0])
if not ok:
raise SSLError(
"Server certificate validation failed: %s",
errmsg)
self.current_host_and_port = host_and_port
logging.info("Established connection to host %s, port %s",
host_and_port[0], host_and_port[1])
break
except socket.error:
self.socket = None
connect_count += 1
logging.warning("Could not connect to host %s, port %s",
host_and_port[0],
host_and_port[1],
exc_info=1)
if self.socket is None:
sleep_duration = (min(self.__reconnect_sleep_max, (
(self.__reconnect_sleep_initial /
(1.0 + self.__reconnect_sleep_increase)) *
math.pow(1.0 + self.__reconnect_sleep_increase, sleep_exp)
)) * (1.0 + random.random() * self.__reconnect_sleep_jitter))
sleep_end = monotonic() + sleep_duration
logging.debug(
"Sleeping for %.1f seconds before attempting reconnect",
sleep_duration)
while self.running and monotonic() < sleep_end:
time.sleep(0.2)
if sleep_duration < self.__reconnect_sleep_max:
sleep_exp += 1
if not self.socket:
raise exception.ConnectFailedException()