def _send_request(self, read_timeout, connect_timeout):
"""Called when we have something to send out on the socket"""
client = self.client
try:
request, async_object = client._queue[0]
except IndexError:
# Not actually something on the queue, this can occur if
# something happens to cancel the request such that we
# don't clear the pipe below after sending
try:
# Clear possible inconsistence (no request in the queue
# but have data in the read pipe), which causes cpu to spin.
os.read(self._read_pipe, 1)
except OSError:
pass
return
# Special case for testing, if this is a _SessionExpire object
# then throw a SessionExpiration error as if we were dropped
if request is _SESSION_EXPIRED:
raise SessionExpiredError("Session expired: Testing")
if request is _CONNECTION_DROP:
raise ConnectionDropped("Connection dropped: Testing")
# Special case for auth packets
if request.type == Auth.type:
xid = AUTH_XID
else:
self._xid += 1
xid = self._xid
self._submit(request, connect_timeout, xid)
client._queue.popleft()
os.read(self._read_pipe, 1)
client._pending.append((request, async_object, xid))
def _send_request(self, read_timeout, connect_timeout):
"""Called when we have something to send out on the socket"""
client = self.client
try:
request, async_object = client._queue[0]
except IndexError:
# Not actually something on the queue, this can occur if
# something happens to cancel the request such that we
# don't clear the pipe below after sending
return
# Special case for testing, if this is a _SessionExpire object
# then throw a SessionExpiration error as if we were dropped
if request is _SESSION_EXPIRED:
raise SessionExpiredError("Session expired: Testing")
if request is _CONNECTION_DROP:
raise ConnectionDropped("Connection dropped: Testing")
# Special case for auth packets
if request.type == Auth.type:
self._submit(request, connect_timeout, AUTH_XID)
client._queue.popleft()
os.read(self._read_pipe, 1)
return
self._xid += 1
if self.log_debug:
log.debug('xid: %r', self._xid)
self._submit(request, connect_timeout, self._xid)
client._queue.popleft()
os.read(self._read_pipe, 1)
client._pending.append((request, async_object, self._xid))
def _read(self, length, timeout):
msgparts = []
remaining = length
with self._socket_error_handling():
while remaining > 0:
# Because of SSL framing, a select may not return when using
# an SSL socket because the underlying physical socket may not
# have anything to select, but the wrapped object may still
# have something to read as it has previously gotten enough
# data from the underlying socket.
if (hasattr(self._socket, "pending")
and self._socket.pending() > 0):
pass
else:
s = self.handler.select([self._socket], [], [], timeout)[0]
if not s: # pragma: nocover
# If the read list is empty, we got a timeout. We don't
# have to check wlist and xlist as we don't set any
raise self.handler.timeout_exception(
"socket time-out during read")
chunk = self._socket.recv(remaining)
if chunk == b'':
raise ConnectionDropped('socket connection broken')
msgparts.append(chunk)
remaining -= len(chunk)
return b"".join(msgparts)
def _write(self, msg, timeout):
"""Write a raw msg to the socket"""
sent = 0
msg_length = len(msg)
with self._socket_error_handling():
while sent < msg_length:
s = self.handler.select([], [self._socket], [], timeout)[1]
if not s: # pragma: nocover
# If the write list is empty, we got a timeout. We don't
# have to check rlist and xlist as we don't set any
raise self.handler.timeout_exception("socket time-out")
msg_slice = buffer(msg, sent)
bytes_sent = self._socket.send(msg_slice)
if not bytes_sent:
raise ConnectionDropped('socket connection broken')
sent += bytes_sent
def _read(self, length, timeout):
msgparts = []
remaining = length
with self._socket_error_handling():
while remaining > 0:
s = self.handler.select([self._socket], [], [], timeout)[0]
if not s: # pragma: nocover
# If the read list is empty, we got a timeout. We don't
# have to check wlist and xlist as we don't set any
raise self.handler.timeout_exception("socket time-out")
chunk = self._socket.recv(remaining)
if chunk == b'':
raise ConnectionDropped('socket connection broken')
msgparts.append(chunk)
remaining -= len(chunk)
return b"".join(msgparts)
def _read_socket(self, read_timeout):
"""Called when there's something to read on the socket"""
client = self.client
header, buffer, offset = self._read_header(read_timeout)
if header.xid == PING_XID:
self.logger.log(BLATHER, 'Received Ping')
self.ping_outstanding.clear()
elif header.xid == AUTH_XID:
self.logger.log(BLATHER, 'Received AUTH')
request, async_object, xid = client._pending.popleft()
if header.err:
async_object.set_exception(AuthFailedError())
client._session_callback(KeeperState.AUTH_FAILED)
else:
async_object.set(True)
elif header.xid == WATCH_XID:
self._read_watch_event(buffer, offset)
elif self.sasl_cli and not self.sasl_cli.complete:
# SASL authentication is not yet finished, this can only
# be a SASL packet
self.logger.log(BLATHER, 'Received SASL')
try:
challenge, _ = SASL.deserialize(buffer, offset)
except Exception:
raise ConnectionDropped('error while SASL authentication.')
response = self.sasl_cli.process(challenge)
if response:
# authentication not yet finished, answering the challenge
self._send_sasl_request(challenge=response,
timeout=client._session_timeout)
else:
# authentication is ok, state is CONNECTED
# remove sensible information from the object
client._session_callback(KeeperState.CONNECTED)
self.sasl_cli.dispose()
else:
self.logger.log(BLATHER, 'Reading for header %r', header)
return self._read_response(header, buffer, offset)
def _invoke(self, timeout, request, xid=None):
"""A special writer used during connection establishment
only"""
self._submit(request, timeout, xid)
zxid = None
if xid:
header, buffer, offset = self._read_header(timeout)
if header.xid != xid:
raise RuntimeError(
'xids do not match, expected %r received %r', xid,
header.xid)
if header.zxid > 0:
zxid = header.zxid
if header.err:
callback_exception = EXCEPTIONS[header.err]()
if self.log_debug:
log.debug('Received error %r', callback_exception)
raise callback_exception
return zxid
msg = self._read(4, timeout)
length = int_struct.unpack(msg)[0]
msg = self._read(length, timeout)
if hasattr(request, 'deserialize'):
try:
obj, _ = request.deserialize(msg, 0)
except Exception as exc:
if self.log_debug:
log.debug(
"Exception raised during deserialization"
" of request: %s", request)
log.exception(exc)
# raise ConnectionDropped so connect loop will retry
raise ConnectionDropped('invalid server response')
log.debug('Read response %s', obj)
return obj, zxid
return zxid
def _write(self, msg, timeout):
"""Write a raw msg to the socket"""
sent = 0
msg_length = len(msg)
with self._socket_error_handling():
while sent < msg_length:
s = self.handler.select([], [self._socket], [], timeout)[1]
if not s: # pragma: nocover
# If the write list is empty, we got a timeout. We don't
# have to check rlist and xlist as we don't set any
raise self.handler.timeout_exception("socket time-out"
" during write")
msg_slice = buffer(msg, sent)
try:
bytes_sent = self._socket.send(msg_slice)
except ssl.SSLError as e:
if e.errno in (ssl.SSL_ERROR_WANT_READ,
ssl.SSL_ERROR_WANT_WRITE):
continue
else:
raise
if not bytes_sent:
raise ConnectionDropped('socket connection broken')
sent += bytes_sent
def _socket_error_handling(self):
try:
yield
except (socket.error, select.error) as e:
err = getattr(e, 'strerror', e)
raise ConnectionDropped("socket connection error: %s", err)
def _connect_attempt(self, host, port, retry):
client = self.client
TimeoutError = self.handler.timeout_exception
close_connection = False
self._socket = None
# Were we given a r/w server? If so, use that instead
if self._rw_server:
self.logger.log(BLATHER, "Found r/w server to use, %s:%s", host,
port)
host, port = self._rw_server
self._rw_server = None
if client._state != KeeperState.CONNECTING:
client._session_callback(KeeperState.CONNECTING)
try:
read_timeout, connect_timeout = self._connect(host, port)
read_timeout = read_timeout / 1000.0
connect_timeout = connect_timeout / 1000.0
retry.reset()
self._xid = 0
while not close_connection:
# Watch for something to read or send
jitter_time = random.randint(0, 40) / 100.0
# Ensure our timeout is positive
timeout = max([read_timeout / 2.0 - jitter_time, jitter_time])
s = self.handler.select([self._socket, self._read_pipe], [],
[], timeout)[0]
if not s:
if self.ping_outstanding.is_set():
self.ping_outstanding.clear()
raise ConnectionDropped(
"outstanding heartbeat ping not received")
self._send_ping(connect_timeout)
elif s[0] == self._socket:
response = self._read_socket(read_timeout)
close_connection = response == CLOSE_RESPONSE
else:
self._send_request(read_timeout, connect_timeout)
self.logger.info('Closing connection to %s:%s', host, port)
client._session_callback(KeeperState.CLOSED)
return STOP_CONNECTING
except (ConnectionDropped, TimeoutError) as e:
if isinstance(e, ConnectionDropped):
self.logger.warning('Connection dropped: %s', e)
else:
self.logger.warning('Connection time-out')
if client._state != KeeperState.CONNECTING:
self.logger.warning("Transition to CONNECTING")
client._session_callback(KeeperState.CONNECTING)
except AuthFailedError:
retry.reset()
self.logger.warning('AUTH_FAILED closing')
client._session_callback(KeeperState.AUTH_FAILED)
return STOP_CONNECTING
except SessionExpiredError:
retry.reset()
self.logger.warning('Session has expired')
client._session_callback(KeeperState.EXPIRED_SESSION)
except RWServerAvailable:
retry.reset()
self.logger.warning('Found a RW server, dropping connection')
client._session_callback(KeeperState.CONNECTING)
except Exception:
self.logger.exception('Unhandled exception in connection loop')
raise
finally:
if self._socket is not None:
self._socket.close()
def socket_error_handling():
try:
yield
except (socket.error, select.error) as e:
raise ConnectionDropped("socket connection error: %s", e.strerror)
def _connect_attempt(self, host, hostip, port, retry):
client = self.client
KazooTimeoutError = self.handler.timeout_exception
self._socket = None
# Were we given a r/w server? If so, use that instead
if self._rw_server:
self.logger.log(BLATHER, "Found r/w server to use, %s:%s", host,
port)
host, port = self._rw_server
self._rw_server = None
if client._state != KeeperState.CONNECTING:
client._session_callback(KeeperState.CONNECTING)
try:
self._xid = 0
read_timeout, connect_timeout = self._connect(host, hostip, port)
read_timeout = read_timeout / 1000.0
connect_timeout = connect_timeout / 1000.0
retry.reset()
self.ping_outstanding.clear()
last_send = time.time()
with self._socket_error_handling():
while True:
# Watch for something to read or send
jitter_time = random.randint(1, 40) / 100.0
deadline = last_send + read_timeout / 2.0 - jitter_time
# Ensure our timeout is positive
timeout = max([deadline - time.time(), jitter_time])
s = self.handler.select([self._socket, self._read_sock],
[], [], timeout)[0]
if not s:
if self.ping_outstanding.is_set():
self.ping_outstanding.clear()
raise ConnectionDropped(
"outstanding heartbeat ping not received")
else:
if self._socket in s:
response = self._read_socket(read_timeout)
if response == CLOSE_RESPONSE:
break
# Check if any requests need sending before proceeding
# to process more responses. Otherwise the responses
# may choke out the requests. See PR#633.
if self._read_sock in s:
self._send_request(read_timeout, connect_timeout)
# Requests act as implicit pings.
last_send = time.time()
continue
if time.time() >= deadline:
self._send_ping(connect_timeout)
last_send = time.time()
self.logger.info('Closing connection to %s:%s', host, port)
client._session_callback(KeeperState.CLOSED)
return STOP_CONNECTING
except (ConnectionDropped, KazooTimeoutError) as e:
if isinstance(e, ConnectionDropped):
self.logger.warning('Connection dropped: %s', e)
else:
self.logger.warning('Connection time-out: %s', e)
if client._state != KeeperState.CONNECTING:
self.logger.warning("Transition to CONNECTING")
client._session_callback(KeeperState.CONNECTING)
except AuthFailedError as err:
retry.reset()
self.logger.warning('AUTH_FAILED closing: %s', err)
client._session_callback(KeeperState.AUTH_FAILED)
return STOP_CONNECTING
except SessionExpiredError:
retry.reset()
self.logger.warning('Session has expired')
client._session_callback(KeeperState.EXPIRED_SESSION)
except RWServerAvailable:
retry.reset()
self.logger.warning('Found a RW server, dropping connection')
client._session_callback(KeeperState.CONNECTING)
except Exception:
self.logger.exception('Unhandled exception in connection loop')
raise
finally:
if self._socket is not None:
self._socket.close()