def __heartbeat_loop(self):
"""
Main loop for sending (and monitoring received) heartbeats.
"""
send_time = monotonic()
while self.running:
time.sleep(self.sleep_time)
now = monotonic()
if self.send_sleep != 0 and now - send_time > self.send_sleep:
send_time = now
log.debug("Sending a heartbeat message at %s", now)
try:
self.transport.transmit(utils.Frame(None, {}, None))
except exception.NotConnectedException:
log.debug("Lost connection, unable to send heartbeat")
except Exception:
_, e, _ = sys.exc_info()
log.debug("Unable to send heartbeat, due to: %s", e)
if self.receive_sleep != 0:
diff_receive = now - self.received_heartbeat
if diff_receive > self.receive_sleep:
# heartbeat timeout
log.warning("Heartbeat timeout: diff_receive=%s, time=%s, lastrec=%s", diff_receive, now, self.received_heartbeat)
self.transport.disconnect_socket()
self.transport.set_connected(False)
for listener in self.transport.listeners.values():
listener.on_heartbeat_timeout()
def __heartbeat_loop(self):
"""
Main loop for sending (and monitoring received) heartbeats.
"""
log.info('Starting heartbeat loop')
now = monotonic()
# Setup the initial due time for the outbound heartbeat
if self.send_sleep != 0:
self.next_outbound_heartbeat = now + self.send_sleep
while self.running:
now = monotonic()
next_events = []
if self.next_outbound_heartbeat is not None:
next_events.append(self.next_outbound_heartbeat - now)
if self.receive_sleep != 0:
t = self.received_heartbeat + self.receive_sleep - now
if t > 0:
next_events.append(t)
sleep_time = min(next_events)
if sleep_time > 0:
terminate = self.heartbeat_terminate_event.wait(sleep_time)
if terminate:
break
now = monotonic()
if not self.transport.is_connected():
time.sleep(self.send_sleep)
continue
if self.send_sleep != 0 and now > self.next_outbound_heartbeat:
log.debug("Sending a heartbeat message at %s", now)
try:
self.transport.transmit(utils.Frame(None, {}, None))
except exception.NotConnectedException:
log.debug("Lost connection, unable to send heartbeat")
except Exception:
_, e, _ = sys.exc_info()
log.debug("Unable to send heartbeat, due to: %s", e)
if self.receive_sleep != 0:
diff_receive = now - self.received_heartbeat
if diff_receive > self.receive_sleep:
# heartbeat timeout
log.warning(
"Heartbeat timeout: diff_receive=%s, time=%s, lastrec=%s",
diff_receive, now, self.received_heartbeat)
self.transport.set_connected(False)
self.transport.disconnect_socket()
self.transport.stop()
for listener in self.transport.listeners.values():
listener.on_heartbeat_timeout()
self.heartbeat_thread = None
log.info('Heartbeat loop ended')
def __heartbeat_loop(self):
"""
Main loop for sending (and monitoring received) heartbeats.
"""
log.info('Starting heartbeat loop')
now = monotonic()
# Setup the initial due time for the outbound heartbeat
if self.send_sleep != 0:
self.next_outbound_heartbeat = now + self.send_sleep
while self.running:
now = monotonic()
next_events = []
if self.next_outbound_heartbeat is not None:
next_events.append(self.next_outbound_heartbeat - now)
if self.receive_sleep != 0:
t = self.received_heartbeat + self.receive_sleep - now
if t > 0:
next_events.append(t)
sleep_time = min(next_events)
if sleep_time > 0:
terminate = self.heartbeat_terminate_event.wait(sleep_time)
if terminate:
break
now = monotonic()
if not self.transport.is_connected():
time.sleep(self.send_sleep)
continue
if self.send_sleep != 0 and now > self.next_outbound_heartbeat:
log.debug("Sending a heartbeat message at %s", now)
try:
self.transport.transmit(utils.Frame(None, {}, None))
except exception.NotConnectedException:
log.debug("Lost connection, unable to send heartbeat")
except Exception:
_, e, _ = sys.exc_info()
log.debug("Unable to send heartbeat, due to: %s", e)
if self.receive_sleep != 0:
diff_receive = now - self.received_heartbeat
if diff_receive > self.receive_sleep:
# heartbeat timeout
log.warning("Heartbeat timeout: diff_receive=%s, time=%s, lastrec=%s",
diff_receive, now, self.received_heartbeat)
self.transport.set_connected(False)
self.transport.disconnect_socket()
self.transport.stop()
for listener in self.transport.listeners.values():
listener.on_heartbeat_timeout()
self.heartbeat_thread = None
log.info('Heartbeat loop ended')
def test_threads_dont_wedge(self):
for t in self.threads:
t.start()
start = monotonic()
while monotonic() - start < self.runfor:
try:
self.Q.put(1, False)
time.sleep(1.0)
except Full:
assert False, "Failed: 'request' queue filled up"
print("passed")
def test_threads_dont_wedge(self):
for t in self.threads:
t.start()
start = monotonic()
while monotonic() - start < self.runfor:
try:
self.Q.put(1, False)
time.sleep(1.0)
except Full:
assert False, "Failed: 'request' queue filled up"
print("passed")
def test_timeout(self):
conn = stomp.Connection([('192.0.2.0', 60000)], timeout=5, reconnect_attempts_max=1)
conn.set_listener('', self.listener)
try:
ms = monotonic()
conn.connect("test", "test")
self.fail("shouldn't happen")
except exception.ConnectFailedException:
pass # success!
ms = monotonic() - ms
self.assertTrue(ms > 5.0, 'connection timeout should have been at least 5 seconds')
def test_timeout(self):
conn = stomp.Connection([('192.0.2.0', 60000)], timeout=5, reconnect_attempts_max=1)
conn.set_listener('', self.listener)
try:
ms = monotonic()
conn.start()
self.fail("shouldn't happen")
except exception.ConnectFailedException:
pass # success!
ms = monotonic() - ms
self.assertTrue(ms > 5.0, 'connection timeout should have been at least 5 seconds')
def __update_heartbeat(self):
# Honour any grace that has been already included
if self.received_heartbeat is None:
return
now = monotonic()
if now > self.received_heartbeat:
self.received_heartbeat = now
def on_connected(self, headers, body):
"""
Once the connection is established, and 'heart-beat' is found in the headers, we calculate the real
heartbeat numbers (based on what the server sent and what was specified by the client) - if the heartbeats
are not 0, we start up the heartbeat loop accordingly.
:param dict headers: headers in the connection message
:param body: the message body
"""
if 'heart-beat' in headers:
self.heartbeats = utils.calculate_heartbeats(
headers['heart-beat'].replace(' ', '').split(','),
self.heartbeats)
if self.heartbeats != (0, 0):
self.send_sleep = self.heartbeats[0] / 1000
# by default, receive gets an additional grace of 50%
# set a different heart-beat-receive-scale when creating the connection to override that
self.receive_sleep = (self.heartbeats[1] /
1000) * self.heart_beat_receive_scale
log.debug("Setting receive_sleep to %s", self.receive_sleep)
# Give grace of receiving the first heartbeat
self.received_heartbeat = monotonic() + self.receive_sleep
self.running = True
if self.heartbeat_thread is None:
self.heartbeat_thread = utils.default_create_thread(
self.__heartbeat_loop)
self.heartbeat_thread.name = "StompHeartbeat%s" % \
getattr(self.heartbeat_thread, "name", "Thread")
def on_connected(self, headers, body):
"""
Once the connection is established, and 'heart-beat' is found in the headers, we calculate the real
heartbeat numbers (based on what the server sent and what was specified by the client) - if the heartbeats
are not 0, we start up the heartbeat loop accordingly.
:param dict headers: headers in the connection message
:param body: the message body
"""
if 'heart-beat' in headers:
self.heartbeats = utils.calculate_heartbeats(
headers['heart-beat'].replace(' ', '').split(','), self.heartbeats)
if self.heartbeats != (0, 0):
self.send_sleep = self.heartbeats[0] / 1000
# by default, receive gets an additional grace of 50%
# set a different heart-beat-receive-scale when creating the connection to override that
self.receive_sleep = (self.heartbeats[1] / 1000) * self.heart_beat_receive_scale
log.debug("Setting receive_sleep to %s", self.receive_sleep)
# Give grace of receiving the first heartbeat
self.received_heartbeat = monotonic() + self.receive_sleep
self.running = True
if self.heartbeat_thread is None:
self.heartbeat_thread = utils.default_create_thread(
self.__heartbeat_loop)
self.heartbeat_thread.name = "StompHeartbeat%s" % \
getattr(self.heartbeat_thread, "name", "Thread")
def __update_heartbeat(self):
# Honour any grace that has been already included
if self.received_heartbeat is None:
return
now = monotonic()
if now > self.received_heartbeat:
self.received_heartbeat = now
def on_connected(self, headers, body):
"""
Once the connection is established, and 'heart-beat' is found in the headers, we calculate the real
heartbeat numbers (based on what the server sent and what was specified by the client) - if the heartbeats
are not 0, we start up the heartbeat loop accordingly.
:param headers: headers in the connection message
:param body: the message body
"""
if 'heart-beat' in headers:
self.heartbeats = utils.calculate_heartbeats(headers['heart-beat'].replace(' ', '').split(','), self.heartbeats)
if self.heartbeats != (0, 0):
self.send_sleep = self.heartbeats[0] / 1000
# receive gets an additional grace of 50%
self.receive_sleep = (self.heartbeats[1] / 1000) * 1.5
# Setup an initial grace period
if self.receive_sleep != 0:
self.received_heartbeat = monotonic() + \
2 * self.receive_sleep
if self.send_sleep == 0:
self.sleep_time = self.receive_sleep
elif self.receive_sleep == 0:
self.sleep_time = self.send_sleep
else:
# sleep is the GCD of the send and receive times
self.sleep_time = gcd(self.send_sleep, self.receive_sleep) / 2.0
self.running = True
if self.heartbeat_thread is None:
self.heartbeat_thread = utils.default_create_thread(self.__heartbeat_loop)
self.heartbeat_thread.name = "StompHeartbeat%s" % getattr(self.heartbeat_thread, "name", "Thread")
def attempt_connection(self):
"""
Try connecting to the (host, port) tuples specified at construction time.
"""
self.connection_error = False
sleep_exp = 1
connect_count = 0
while self.running and self.socket is None and connect_count < self.__reconnect_attempts_max:
for host_and_port in self.__hosts_and_ports:
try:
log.info("Attempting connection to websocket %s", host_and_port)
self.socket = websocket.WebSocket()
proto, host, port, path = host_and_port[3], host_and_port[0], host_and_port[1], host_and_port[2]
if port:
ws_uri = '{}://{}:{}/{}'.format(proto, host, port, path)
else:
ws_uri = '{}://{}/{}'.format(proto, host, path)
self.socket.connect(ws_uri,
timeout=self.__timeout)
self.current_host_and_port = host_and_port
log.info("Established connection to %s", ws_uri)
break
except WebSocketException:
self.socket = None
connect_count += 1
log.warning("Could not connect to host %s, port %s", host_and_port[0], host_and_port[1], exc_info=1)
if self.socket is None:
sleep_duration = (min(self.__reconnect_sleep_max,
((self.__reconnect_sleep_initial / (1.0 + self.__reconnect_sleep_increase))
* math.pow(1.0 + self.__reconnect_sleep_increase, sleep_exp)))
* (1.0 + random.random() * self.__reconnect_sleep_jitter))
sleep_end = monotonic() + sleep_duration
log.debug("Sleeping for %.1f seconds before attempting reconnect", sleep_duration)
while self.running and monotonic() < sleep_end:
time.sleep(0.2)
if sleep_duration < self.__reconnect_sleep_max:
sleep_exp += 1
if not self.socket:
raise exception.ConnectFailedException()
def on_message(self, headers, body):
"""
Reset the last received time whenever a message is received.
:param headers: headers in the message
:param body: the message content
"""
# reset the heartbeat for any received message
self.received_heartbeat = monotonic()
def on_send(self, frame):
"""
Add the heartbeat header to the frame when connecting, and bump
next outbound heartbeat timestamp.
:param Frame frame: the Frame object
"""
if frame.cmd == CMD_CONNECT or frame.cmd == CMD_STOMP:
if self.heartbeats != (0, 0):
frame.headers[HDR_HEARTBEAT] = '%s,%s' % self.heartbeats
if self.next_outbound_heartbeat is not None:
self.next_outbound_heartbeat = monotonic() + self.send_sleep
def on_send(self, frame):
"""
Add the heartbeat header to the frame when connecting, and bump
next outbound heartbeat timestamp.
:param Frame frame: the Frame object
"""
if frame.cmd == CMD_CONNECT or frame.cmd == CMD_STOMP:
if self.heartbeats != (0, 0):
frame.headers[HDR_HEARTBEAT] = '%s,%s' % self.heartbeats
if self.next_outbound_heartbeat is not None:
self.next_outbound_heartbeat = monotonic() + self.send_sleep
def attempt_connection(self):
"""
Try connecting to the (host, port) tuples specified at construction time.
"""
self.connection_error = False
sleep_exp = 1
connect_count = 0
while self.running and self.socket is None and connect_count < self.__reconnect_attempts_max:
for host_and_port in self.__host_and_ports:
try:
log.info("Attempting connection to host %s, port %s",
host_and_port[0], host_and_port[1])
self.socket = get_socket(host_and_port[0],
host_and_port[1], self.__timeout)
self.__enable_keepalive()
need_ssl = self.__need_ssl(host_and_port)
if need_ssl: # wrap socket
ssl_params = self.get_ssl(host_and_port)
if ssl_params['ca_certs']:
cert_validation = ssl.CERT_REQUIRED
else:
cert_validation = ssl.CERT_NONE
self.socket = ssl.wrap_socket(
self.socket,
keyfile=ssl_params['key_file'],
certfile=ssl_params['cert_file'],
cert_reqs=cert_validation,
ca_certs=ssl_params['ca_certs'],
ssl_version=ssl_params['ssl_version'])
self.socket.settimeout(self.__timeout)
if self.blocking is not None:
self.socket.setblocking(self.blocking)
#
# Validate server cert
#
if need_ssl and ssl_params['cert_validator']:
cert = self.socket.getpeercert()
(ok, errmsg) = ssl_params['cert_validator'](
cert, host_and_port[0])
if not ok:
raise SSLError(
"Server certificate validation failed: %s",
errmsg)
self.current_host_and_port = host_and_port
log.info("Established connection to host %s, port %s",
host_and_port[0], host_and_port[1])
break
except socket.error:
self.socket = None
connect_count += 1
log.info("Could not connect to host %s, port %s",
host_and_port[0],
host_and_port[1],
exc_info=1)
if self.socket is None:
sleep_duration = (min(self.__reconnect_sleep_max, (
(self.__reconnect_sleep_initial /
(1.0 + self.__reconnect_sleep_increase)) *
math.pow(1.0 + self.__reconnect_sleep_increase, sleep_exp)
)) * (1.0 + random.random() * self.__reconnect_sleep_jitter))
sleep_end = monotonic() + sleep_duration
log.debug(
"Sleeping for %.1f seconds before attempting reconnect",
sleep_duration)
while self.running and monotonic() < sleep_end:
time.sleep(0.2)
if sleep_duration < self.__reconnect_sleep_max:
sleep_exp += 1
if not self.socket:
raise exception.ConnectFailedException()
def attempt_connection(self):
"""
Try connecting to the (host, port) tuples specified at construction time.
"""
self.connection_error = False
sleep_exp = 1
connect_count = 0
while self.running and self.socket is None and connect_count < self.__reconnect_attempts_max:
for host_and_port in self.__host_and_ports:
try:
log.info("Attempting connection to host %s, port %s", host_and_port[0], host_and_port[1])
self.socket = get_socket(host_and_port[0], host_and_port[1], self.__timeout)
self.__enable_keepalive()
need_ssl = self.__need_ssl(host_and_port)
if need_ssl: # wrap socket
ssl_params = self.get_ssl(host_and_port)
if ssl_params['ca_certs']:
cert_validation = ssl.CERT_REQUIRED
else:
cert_validation = ssl.CERT_NONE
try:
tls_context = ssl.create_default_context(cafile=ssl_params['ca_certs'])
except AttributeError:
tls_context = None
if tls_context:
# Wrap the socket for TLS
certfile = ssl_params['cert_file']
keyfile = ssl_params['key_file']
if certfile and not keyfile:
keyfile = certfile
if certfile:
tls_context.load_cert_chain(certfile, keyfile)
if cert_validation is None or cert_validation == ssl.CERT_NONE:
tls_context.check_hostname = False
tls_context.verify_mode = cert_validation
self.socket = tls_context.wrap_socket(self.socket, server_hostname=host_and_port[0])
else:
# Old-style wrap_socket where we don't have a modern SSLContext (so no SNI)
self.socket = ssl.wrap_socket(
self.socket,
keyfile=ssl_params['key_file'],
certfile=ssl_params['cert_file'],
cert_reqs=cert_validation,
ca_certs=ssl_params['ca_certs'],
ssl_version=ssl_params['ssl_version'])
self.socket.settimeout(self.__timeout)
if self.blocking is not None:
self.socket.setblocking(self.blocking)
#
# Validate server cert
#
if need_ssl and ssl_params['cert_validator']:
cert = self.socket.getpeercert()
(ok, errmsg) = ssl_params['cert_validator'](cert, host_and_port[0])
if not ok:
raise SSLError("Server certificate validation failed: %s", errmsg)
self.current_host_and_port = host_and_port
log.info("Established connection to host %s, port %s", host_and_port[0], host_and_port[1])
break
except socket.error:
self.socket = None
connect_count += 1
log.warning("Could not connect to host %s, port %s", host_and_port[0], host_and_port[1], exc_info=1)
if self.socket is None:
sleep_duration = (min(self.__reconnect_sleep_max,
((self.__reconnect_sleep_initial / (1.0 + self.__reconnect_sleep_increase))
* math.pow(1.0 + self.__reconnect_sleep_increase, sleep_exp)))
* (1.0 + random.random() * self.__reconnect_sleep_jitter))
sleep_end = monotonic() + sleep_duration
log.debug("Sleeping for %.1f seconds before attempting reconnect", sleep_duration)
while self.running and monotonic() < sleep_end:
time.sleep(0.2)
if sleep_duration < self.__reconnect_sleep_max:
sleep_exp += 1
if not self.socket:
raise exception.ConnectFailedException()
def attempt_connection(self):
"""
Try connecting to the (host, port) tuples specified at construction time.
"""
self.connection_error = False
sleep_exp = 1
connect_count = 0
while self.running and self.socket is None and connect_count < self.__reconnect_attempts_max:
for host_and_port in self.__host_and_ports:
try:
log.info("Attempting connection to host %s, port %s", host_and_port[0], host_and_port[1])
self.socket = get_socket(host_and_port[0], host_and_port[1], self.__timeout)
self.__enable_keepalive()
need_ssl = self.__need_ssl(host_and_port)
if need_ssl: # wrap socket
ssl_params = self.get_ssl(host_and_port)
if ssl_params["ca_certs"]:
cert_validation = ssl.CERT_REQUIRED
else:
cert_validation = ssl.CERT_NONE
self.socket = ssl.wrap_socket(
self.socket,
keyfile=ssl_params["key_file"],
certfile=ssl_params["cert_file"],
cert_reqs=cert_validation,
ca_certs=ssl_params["ca_certs"],
ssl_version=ssl_params["ssl_version"],
)
self.socket.settimeout(self.__timeout)
if self.blocking is not None:
self.socket.setblocking(self.blocking)
#
# Validate server cert
#
if need_ssl and ssl_params["cert_validator"]:
cert = self.socket.getpeercert()
(ok, errmsg) = ssl_params["cert_validator"](cert, host_and_port[0])
if not ok:
raise SSLError("Server certificate validation failed: %s", errmsg)
self.current_host_and_port = host_and_port
log.info("Established connection to host %s, port %s", host_and_port[0], host_and_port[1])
break
except socket.error:
self.socket = None
connect_count += 1
log.info("Could not connect to host %s, port %s", host_and_port[0], host_and_port[1], exc_info=1)
if self.socket is None:
sleep_duration = min(
self.__reconnect_sleep_max,
(
(self.__reconnect_sleep_initial / (1.0 + self.__reconnect_sleep_increase))
* math.pow(1.0 + self.__reconnect_sleep_increase, sleep_exp)
),
) * (1.0 + random.random() * self.__reconnect_sleep_jitter)
sleep_end = monotonic() + sleep_duration
log.debug("Sleeping for %.1f seconds before attempting reconnect", sleep_duration)
while self.running and monotonic() < sleep_end:
time.sleep(0.2)
if sleep_duration < self.__reconnect_sleep_max:
sleep_exp += 1
if not self.socket:
raise exception.ConnectFailedException()
def attempt_connection(self):
"""
Try connecting to the (host, port) tuples specified at construction time.
"""
self.connection_error = False
sleep_exp = 1
connect_count = 0
while self.running and self.socket is None and connect_count < self.__reconnect_attempts_max:
for host_and_port in self.__host_and_ports:
try:
log.info("Attempting connection to host %s, port %s",
host_and_port[0], host_and_port[1])
self.socket = get_socket(host_and_port[0],
host_and_port[1], self.__timeout)
self.__enable_keepalive()
need_ssl = self.__need_ssl(host_and_port)
if need_ssl: # wrap socket
ssl_params = self.get_ssl(host_and_port)
if ssl_params['ca_certs']:
cert_validation = ssl.CERT_REQUIRED
else:
cert_validation = ssl.CERT_NONE
try:
tls_context = ssl.create_default_context(
cafile=ssl_params['ca_certs'])
except AttributeError:
tls_context = None
if tls_context:
# Wrap the socket for TLS
certfile = ssl_params['cert_file']
keyfile = ssl_params['key_file']
password = ssl_params.get('password')
if certfile and not keyfile:
keyfile = certfile
if certfile:
tls_context.load_cert_chain(
certfile, keyfile, password)
if cert_validation is None or cert_validation == ssl.CERT_NONE:
tls_context.check_hostname = False
tls_context.verify_mode = cert_validation
self.socket = tls_context.wrap_socket(
self.socket, server_hostname=host_and_port[0])
else:
# Old-style wrap_socket where we don't have a modern SSLContext (so no SNI)
self.socket = ssl.wrap_socket(
self.socket,
keyfile=ssl_params['key_file'],
certfile=ssl_params['cert_file'],
cert_reqs=cert_validation,
ca_certs=ssl_params['ca_certs'],
ssl_version=ssl_params['ssl_version'])
self.socket.settimeout(self.__timeout)
if self.blocking is not None:
self.socket.setblocking(self.blocking)
#
# Validate server cert
#
if need_ssl and ssl_params['cert_validator']:
cert = self.socket.getpeercert()
(ok, errmsg) = ssl_params['cert_validator'](
cert, host_and_port[0])
if not ok:
raise SSLError(
"Server certificate validation failed: %s",
errmsg)
self.current_host_and_port = host_and_port
log.info("Established connection to host %s, port %s",
host_and_port[0], host_and_port[1])
break
except socket.error:
self.socket = None
connect_count += 1
log.warning("Could not connect to host %s, port %s",
host_and_port[0],
host_and_port[1],
exc_info=1)
if self.socket is None:
sleep_duration = (min(self.__reconnect_sleep_max, (
(self.__reconnect_sleep_initial /
(1.0 + self.__reconnect_sleep_increase)) *
math.pow(1.0 + self.__reconnect_sleep_increase, sleep_exp)
)) * (1.0 + random.random() * self.__reconnect_sleep_jitter))
sleep_end = monotonic() + sleep_duration
log.debug(
"Sleeping for %.1f seconds before attempting reconnect",
sleep_duration)
while self.running and monotonic() < sleep_end:
time.sleep(0.2)
if sleep_duration < self.__reconnect_sleep_max:
sleep_exp += 1
if not self.socket:
raise exception.ConnectFailedException()
def on_heartbeat(self):
"""
Reset the last received time whenever a heartbeat message is received.
"""
self.received_heartbeat = monotonic()
def on_error(self, *_):
"""
Reset the last received time whenever an error is received.
"""
self.received_heartbeat = monotonic()
def on_receipt(self, *_):
"""
Reset the last received time whenever a receipt is received.
"""
self.received_heartbeat = monotonic()
