def _send_request(self, read_timeout, connect_timeout): """Called when we have something to send out on the socket""" client = self.client try: request, async_object = client._queue[0] except IndexError: # Not actually something on the queue, this can occur if # something happens to cancel the request such that we # don't clear the pipe below after sending try: # Clear possible inconsistence (no request in the queue # but have data in the read pipe), which causes cpu to spin. os.read(self._read_pipe, 1) except OSError: pass return # Special case for testing, if this is a _SessionExpire object # then throw a SessionExpiration error as if we were dropped if request is _SESSION_EXPIRED: raise SessionExpiredError("Session expired: Testing") if request is _CONNECTION_DROP: raise ConnectionDropped("Connection dropped: Testing") # Special case for auth packets if request.type == Auth.type: xid = AUTH_XID else: self._xid += 1 xid = self._xid self._submit(request, connect_timeout, xid) client._queue.popleft() os.read(self._read_pipe, 1) client._pending.append((request, async_object, xid))
def _send_request(self, read_timeout, connect_timeout): """Called when we have something to send out on the socket""" client = self.client try: request, async_object = client._queue[0] except IndexError: # Not actually something on the queue, this can occur if # something happens to cancel the request such that we # don't clear the pipe below after sending return # Special case for testing, if this is a _SessionExpire object # then throw a SessionExpiration error as if we were dropped if request is _SESSION_EXPIRED: raise SessionExpiredError("Session expired: Testing") if request is _CONNECTION_DROP: raise ConnectionDropped("Connection dropped: Testing") # Special case for auth packets if request.type == Auth.type: self._submit(request, connect_timeout, AUTH_XID) client._queue.popleft() os.read(self._read_pipe, 1) return self._xid += 1 if self.log_debug: log.debug('xid: %r', self._xid) self._submit(request, connect_timeout, self._xid) client._queue.popleft() os.read(self._read_pipe, 1) client._pending.append((request, async_object, self._xid))
def _read(self, length, timeout): msgparts = [] remaining = length with self._socket_error_handling(): while remaining > 0: # Because of SSL framing, a select may not return when using # an SSL socket because the underlying physical socket may not # have anything to select, but the wrapped object may still # have something to read as it has previously gotten enough # data from the underlying socket. if (hasattr(self._socket, "pending") and self._socket.pending() > 0): pass else: s = self.handler.select([self._socket], [], [], timeout)[0] if not s: # pragma: nocover # If the read list is empty, we got a timeout. We don't # have to check wlist and xlist as we don't set any raise self.handler.timeout_exception( "socket time-out during read") chunk = self._socket.recv(remaining) if chunk == b'': raise ConnectionDropped('socket connection broken') msgparts.append(chunk) remaining -= len(chunk) return b"".join(msgparts)
def _write(self, msg, timeout): """Write a raw msg to the socket""" sent = 0 msg_length = len(msg) with self._socket_error_handling(): while sent < msg_length: s = self.handler.select([], [self._socket], [], timeout)[1] if not s: # pragma: nocover # If the write list is empty, we got a timeout. We don't # have to check rlist and xlist as we don't set any raise self.handler.timeout_exception("socket time-out") msg_slice = buffer(msg, sent) bytes_sent = self._socket.send(msg_slice) if not bytes_sent: raise ConnectionDropped('socket connection broken') sent += bytes_sent
def _read(self, length, timeout): msgparts = [] remaining = length with self._socket_error_handling(): while remaining > 0: s = self.handler.select([self._socket], [], [], timeout)[0] if not s: # pragma: nocover # If the read list is empty, we got a timeout. We don't # have to check wlist and xlist as we don't set any raise self.handler.timeout_exception("socket time-out") chunk = self._socket.recv(remaining) if chunk == b'': raise ConnectionDropped('socket connection broken') msgparts.append(chunk) remaining -= len(chunk) return b"".join(msgparts)
def _read_socket(self, read_timeout): """Called when there's something to read on the socket""" client = self.client header, buffer, offset = self._read_header(read_timeout) if header.xid == PING_XID: self.logger.log(BLATHER, 'Received Ping') self.ping_outstanding.clear() elif header.xid == AUTH_XID: self.logger.log(BLATHER, 'Received AUTH') request, async_object, xid = client._pending.popleft() if header.err: async_object.set_exception(AuthFailedError()) client._session_callback(KeeperState.AUTH_FAILED) else: async_object.set(True) elif header.xid == WATCH_XID: self._read_watch_event(buffer, offset) elif self.sasl_cli and not self.sasl_cli.complete: # SASL authentication is not yet finished, this can only # be a SASL packet self.logger.log(BLATHER, 'Received SASL') try: challenge, _ = SASL.deserialize(buffer, offset) except Exception: raise ConnectionDropped('error while SASL authentication.') response = self.sasl_cli.process(challenge) if response: # authentication not yet finished, answering the challenge self._send_sasl_request(challenge=response, timeout=client._session_timeout) else: # authentication is ok, state is CONNECTED # remove sensible information from the object client._session_callback(KeeperState.CONNECTED) self.sasl_cli.dispose() else: self.logger.log(BLATHER, 'Reading for header %r', header) return self._read_response(header, buffer, offset)
def _invoke(self, timeout, request, xid=None): """A special writer used during connection establishment only""" self._submit(request, timeout, xid) zxid = None if xid: header, buffer, offset = self._read_header(timeout) if header.xid != xid: raise RuntimeError( 'xids do not match, expected %r received %r', xid, header.xid) if header.zxid > 0: zxid = header.zxid if header.err: callback_exception = EXCEPTIONS[header.err]() if self.log_debug: log.debug('Received error %r', callback_exception) raise callback_exception return zxid msg = self._read(4, timeout) length = int_struct.unpack(msg)[0] msg = self._read(length, timeout) if hasattr(request, 'deserialize'): try: obj, _ = request.deserialize(msg, 0) except Exception as exc: if self.log_debug: log.debug( "Exception raised during deserialization" " of request: %s", request) log.exception(exc) # raise ConnectionDropped so connect loop will retry raise ConnectionDropped('invalid server response') log.debug('Read response %s', obj) return obj, zxid return zxid
def _write(self, msg, timeout): """Write a raw msg to the socket""" sent = 0 msg_length = len(msg) with self._socket_error_handling(): while sent < msg_length: s = self.handler.select([], [self._socket], [], timeout)[1] if not s: # pragma: nocover # If the write list is empty, we got a timeout. We don't # have to check rlist and xlist as we don't set any raise self.handler.timeout_exception("socket time-out" " during write") msg_slice = buffer(msg, sent) try: bytes_sent = self._socket.send(msg_slice) except ssl.SSLError as e: if e.errno in (ssl.SSL_ERROR_WANT_READ, ssl.SSL_ERROR_WANT_WRITE): continue else: raise if not bytes_sent: raise ConnectionDropped('socket connection broken') sent += bytes_sent
def _socket_error_handling(self): try: yield except (socket.error, select.error) as e: err = getattr(e, 'strerror', e) raise ConnectionDropped("socket connection error: %s", err)
def _connect_attempt(self, host, port, retry): client = self.client TimeoutError = self.handler.timeout_exception close_connection = False self._socket = None # Were we given a r/w server? If so, use that instead if self._rw_server: self.logger.log(BLATHER, "Found r/w server to use, %s:%s", host, port) host, port = self._rw_server self._rw_server = None if client._state != KeeperState.CONNECTING: client._session_callback(KeeperState.CONNECTING) try: read_timeout, connect_timeout = self._connect(host, port) read_timeout = read_timeout / 1000.0 connect_timeout = connect_timeout / 1000.0 retry.reset() self._xid = 0 while not close_connection: # Watch for something to read or send jitter_time = random.randint(0, 40) / 100.0 # Ensure our timeout is positive timeout = max([read_timeout / 2.0 - jitter_time, jitter_time]) s = self.handler.select([self._socket, self._read_pipe], [], [], timeout)[0] if not s: if self.ping_outstanding.is_set(): self.ping_outstanding.clear() raise ConnectionDropped( "outstanding heartbeat ping not received") self._send_ping(connect_timeout) elif s[0] == self._socket: response = self._read_socket(read_timeout) close_connection = response == CLOSE_RESPONSE else: self._send_request(read_timeout, connect_timeout) self.logger.info('Closing connection to %s:%s', host, port) client._session_callback(KeeperState.CLOSED) return STOP_CONNECTING except (ConnectionDropped, TimeoutError) as e: if isinstance(e, ConnectionDropped): self.logger.warning('Connection dropped: %s', e) else: self.logger.warning('Connection time-out') if client._state != KeeperState.CONNECTING: self.logger.warning("Transition to CONNECTING") client._session_callback(KeeperState.CONNECTING) except AuthFailedError: retry.reset() self.logger.warning('AUTH_FAILED closing') client._session_callback(KeeperState.AUTH_FAILED) return STOP_CONNECTING except SessionExpiredError: retry.reset() self.logger.warning('Session has expired') client._session_callback(KeeperState.EXPIRED_SESSION) except RWServerAvailable: retry.reset() self.logger.warning('Found a RW server, dropping connection') client._session_callback(KeeperState.CONNECTING) except Exception: self.logger.exception('Unhandled exception in connection loop') raise finally: if self._socket is not None: self._socket.close()
def socket_error_handling(): try: yield except (socket.error, select.error) as e: raise ConnectionDropped("socket connection error: %s", e.strerror)
def _connect_attempt(self, host, hostip, port, retry): client = self.client KazooTimeoutError = self.handler.timeout_exception self._socket = None # Were we given a r/w server? If so, use that instead if self._rw_server: self.logger.log(BLATHER, "Found r/w server to use, %s:%s", host, port) host, port = self._rw_server self._rw_server = None if client._state != KeeperState.CONNECTING: client._session_callback(KeeperState.CONNECTING) try: self._xid = 0 read_timeout, connect_timeout = self._connect(host, hostip, port) read_timeout = read_timeout / 1000.0 connect_timeout = connect_timeout / 1000.0 retry.reset() self.ping_outstanding.clear() last_send = time.time() with self._socket_error_handling(): while True: # Watch for something to read or send jitter_time = random.randint(1, 40) / 100.0 deadline = last_send + read_timeout / 2.0 - jitter_time # Ensure our timeout is positive timeout = max([deadline - time.time(), jitter_time]) s = self.handler.select([self._socket, self._read_sock], [], [], timeout)[0] if not s: if self.ping_outstanding.is_set(): self.ping_outstanding.clear() raise ConnectionDropped( "outstanding heartbeat ping not received") else: if self._socket in s: response = self._read_socket(read_timeout) if response == CLOSE_RESPONSE: break # Check if any requests need sending before proceeding # to process more responses. Otherwise the responses # may choke out the requests. See PR#633. if self._read_sock in s: self._send_request(read_timeout, connect_timeout) # Requests act as implicit pings. last_send = time.time() continue if time.time() >= deadline: self._send_ping(connect_timeout) last_send = time.time() self.logger.info('Closing connection to %s:%s', host, port) client._session_callback(KeeperState.CLOSED) return STOP_CONNECTING except (ConnectionDropped, KazooTimeoutError) as e: if isinstance(e, ConnectionDropped): self.logger.warning('Connection dropped: %s', e) else: self.logger.warning('Connection time-out: %s', e) if client._state != KeeperState.CONNECTING: self.logger.warning("Transition to CONNECTING") client._session_callback(KeeperState.CONNECTING) except AuthFailedError as err: retry.reset() self.logger.warning('AUTH_FAILED closing: %s', err) client._session_callback(KeeperState.AUTH_FAILED) return STOP_CONNECTING except SessionExpiredError: retry.reset() self.logger.warning('Session has expired') client._session_callback(KeeperState.EXPIRED_SESSION) except RWServerAvailable: retry.reset() self.logger.warning('Found a RW server, dropping connection') client._session_callback(KeeperState.CONNECTING) except Exception: self.logger.exception('Unhandled exception in connection loop') raise finally: if self._socket is not None: self._socket.close()