def __heartbeat_loop(self): """ Main loop for sending (and monitoring received) heartbeats. """ logging.info('Starting heartbeat loop') now = monotonic() # Setup the initial due time for the outbound heartbeat if self.send_sleep != 0: self.next_outbound_heartbeat = now + self.send_sleep while self.running: now = monotonic() next_events = [] if self.next_outbound_heartbeat is not None: next_events.append(self.next_outbound_heartbeat - now) if self.receive_sleep != 0: t = self.received_heartbeat + self.receive_sleep - now if t > 0: next_events.append(t) sleep_time = min(next_events) if sleep_time > 0: terminate = self.heartbeat_terminate_event.wait(sleep_time) if terminate: break now = monotonic() if not self.transport.is_connected(): time.sleep(self.send_sleep) continue if self.send_sleep != 0 and now > self.next_outbound_heartbeat: logging.debug("Sending a heartbeat message at %s", now) try: self.transport.transmit(utils.Frame(None, {}, None)) except exception.NotConnectedException: logging.debug("Lost connection, unable to send heartbeat") except Exception: _, e, _ = sys.exc_info() logging.debug("Unable to send heartbeat, due to: %s", e) if self.receive_sleep != 0: diff_receive = now - self.received_heartbeat if diff_receive > self.receive_sleep: # heartbeat timeout logging.warning( "Heartbeat timeout: diff_receive=%s, time=%s, lastrec=%s", diff_receive, now, self.received_heartbeat) self.transport.set_connected(False) self.transport.disconnect_socket() self.transport.stop() for listener in self.transport.listeners.values(): listener.on_heartbeat_timeout() self.heartbeat_thread = None self.heartbeat_terminate_event.clear() logging.info('Heartbeat loop ended')
def process_frame(self, f, frame_str): """ :param Frame f: Frame object :param bytes frame_str: raw frame content """ frame_type = f.cmd.lower() if frame_type in ["connected", "message", "receipt", "error", "heartbeat"]: if frame_type == "message": self.notify("before_message", f) if logging.isEnabledFor(logging.DEBUG): logging.debug("Received frame: %r, headers=%r, body=%r", f.cmd, f.headers, f.body) else: logging.info("Received frame: %r, len(body)=%r", f.cmd, length(f.body)) self.notify(frame_type, f) else: logging.warning("Unknown response frame type: '%s' (frame length was %d)", frame_type, length(frame_str))
def disconnect_socket(self): """ Disconnect the underlying socket connection """ self.running = False if self.socket is not None: if self.__need_ssl(): # # Even though we don't want to use the socket, unwrap is the only API method which does a proper SSL # shutdown # try: self.socket = self.socket.unwrap() except Exception: # # unwrap seems flaky on Win with the back-ported ssl mod, so catch any exception and log it # _, e, _ = sys.exc_info() logging.warning(e) elif hasattr(socket, 'SHUT_RDWR'): try: self.socket.shutdown(socket.SHUT_RDWR) except socket.error: _, e, _ = sys.exc_info() # ignore when socket already closed if get_errno(e) != errno.ENOTCONN: logging.warning( "Unable to issue SHUT_RDWR on socket because of error '%s'", e) # # split this into a separate check, because sometimes the socket is nulled between shutdown and this call # if self.socket is not None: try: self.socket.close() except socket.error: _, e, _ = sys.exc_info() logging.warning("Unable to close socket because of error '%s'", e) self.current_host_and_port = None self.socket = None self.notify('disconnected')
def attempt_connection(self): """ Try connecting to the (host, port) tuples specified at construction time. """ self.connection_error = False sleep_exp = 1 connect_count = 0 while self.running and self.socket is None and ( connect_count < self.__reconnect_attempts_max or self.__reconnect_attempts_max == -1): for host_and_port in self.__host_and_ports: try: logging.info("Attempting connection to host %s, port %s", host_and_port[0], host_and_port[1]) self.socket = socket.create_connection( host_and_port, self.__timeout) self.__enable_keepalive() need_ssl = self.__need_ssl(host_and_port) if need_ssl: # wrap socket ssl_params = self.get_ssl(host_and_port) if ssl_params['ca_certs']: cert_validation = ssl.CERT_REQUIRED else: cert_validation = ssl.CERT_NONE try: tls_context = ssl.create_default_context( cafile=ssl_params['ca_certs']) except AttributeError: tls_context = None if tls_context: # Wrap the socket for TLS certfile = ssl_params['cert_file'] keyfile = ssl_params['key_file'] password = ssl_params.get('password') if certfile and not keyfile: keyfile = certfile if certfile: tls_context.load_cert_chain( certfile, keyfile, password) if cert_validation is None or cert_validation == ssl.CERT_NONE: tls_context.check_hostname = False tls_context.verify_mode = cert_validation self.socket = tls_context.wrap_socket( self.socket, server_hostname=host_and_port[0]) else: # Old-style wrap_socket where we don't have a modern SSLContext (so no SNI) self.socket = ssl.wrap_socket( self.socket, keyfile=ssl_params['key_file'], certfile=ssl_params['cert_file'], cert_reqs=cert_validation, ca_certs=ssl_params['ca_certs'], ssl_version=ssl_params['ssl_version']) self.socket.settimeout(self.__timeout) if self.blocking is not None: self.socket.setblocking(self.blocking) # # Validate server cert # if need_ssl and ssl_params['cert_validator']: cert = self.socket.getpeercert() (ok, errmsg) = ssl_params['cert_validator']( cert, host_and_port[0]) if not ok: raise SSLError( "Server certificate validation failed: %s", errmsg) self.current_host_and_port = host_and_port logging.info("Established connection to host %s, port %s", host_and_port[0], host_and_port[1]) break except socket.error: self.socket = None connect_count += 1 logging.warning("Could not connect to host %s, port %s", host_and_port[0], host_and_port[1], exc_info=1) if self.socket is None: sleep_duration = (min(self.__reconnect_sleep_max, ( (self.__reconnect_sleep_initial / (1.0 + self.__reconnect_sleep_increase)) * math.pow(1.0 + self.__reconnect_sleep_increase, sleep_exp) )) * (1.0 + random.random() * self.__reconnect_sleep_jitter)) sleep_end = monotonic() + sleep_duration logging.debug( "Sleeping for %.1f seconds before attempting reconnect", sleep_duration) while self.running and monotonic() < sleep_end: time.sleep(0.2) if sleep_duration < self.__reconnect_sleep_max: sleep_exp += 1 if not self.socket: raise exception.ConnectFailedException()