def _call(self, request, async_object):
"""Ensure there's an active connection and put the request in
the queue if there is.
Returns False if the call short circuits due to AUTH_FAILED,
CLOSED, EXPIRED_SESSION or CONNECTING state.
"""
if self._state == KeeperState.AUTH_FAILED:
async_object.set_exception(AuthFailedError())
return False
elif self._state == KeeperState.CLOSED:
async_object.set_exception(
ConnectionClosedError("Connection has been closed"))
return False
elif self._state in (KeeperState.EXPIRED_SESSION,
KeeperState.CONNECTING):
async_object.set_exception(SessionExpiredError())
return False
self._queue.append((request, async_object))
# wake the connection, guarding against a race with close()
write_pipe = self._connection._write_pipe
if write_pipe is None:
async_object.set_exception(
ConnectionClosedError("Connection has been closed"))
try:
os.write(write_pipe, b'\0')
except:
async_object.set_exception(
ConnectionClosedError("Connection has been closed"))
def _send_request(self, read_timeout, connect_timeout):
"""Called when we have something to send out on the socket"""
client = self.client
try:
request, async_object = client._queue[0]
except IndexError:
# Not actually something on the queue, this can occur if
# something happens to cancel the request such that we
# don't clear the pipe below after sending
return
# Special case for testing, if this is a _SessionExpire object
# then throw a SessionExpiration error as if we were dropped
if request is _SESSION_EXPIRED:
raise SessionExpiredError("Session expired: Testing")
if request is _CONNECTION_DROP:
raise ConnectionDropped("Connection dropped: Testing")
# Special case for auth packets
if request.type == Auth.type:
self._submit(request, connect_timeout, AUTH_XID)
client._queue.popleft()
os.read(self._read_pipe, 1)
return
self._xid += 1
if self.log_debug:
log.debug('xid: %r', self._xid)
self._submit(request, connect_timeout, self._xid)
client._queue.popleft()
os.read(self._read_pipe, 1)
client._pending.append((request, async_object, self._xid))
def _notify_pending(self, state):
"""Used to clear a pending response queue and request queue
during connection drops."""
if state == KeeperState.AUTH_FAILED:
exc = AuthFailedError()
elif state == KeeperState.EXPIRED_SESSION:
exc = SessionExpiredError()
else:
exc = ConnectionLoss()
while True:
try:
request, async_object, xid = self._pending.popleft()
if async_object:
async_object.set_exception(exc)
except IndexError:
break
while True:
try:
request, async_object = self._queue.popleft()
if async_object:
async_object.set_exception(exc)
except IndexError:
break
def _send_request(self, read_timeout, connect_timeout):
"""Called when we have something to send out on the socket"""
client = self.client
try:
request, async_object = client._queue[0]
except IndexError:
# Not actually something on the queue, this can occur if
# something happens to cancel the request such that we
# don't clear the pipe below after sending
try:
# Clear possible inconsistence (no request in the queue
# but have data in the read pipe), which causes cpu to spin.
os.read(self._read_pipe, 1)
except OSError:
pass
return
# Special case for testing, if this is a _SessionExpire object
# then throw a SessionExpiration error as if we were dropped
if request is _SESSION_EXPIRED:
raise SessionExpiredError("Session expired: Testing")
if request is _CONNECTION_DROP:
raise ConnectionDropped("Connection dropped: Testing")
# Special case for auth packets
if request.type == Auth.type:
xid = AUTH_XID
else:
self._xid += 1
xid = self._xid
self._submit(request, connect_timeout, xid)
client._queue.popleft()
os.read(self._read_pipe, 1)
client._pending.append((request, async_object, xid))
def _connect(self, host, port):
client = self.client
self.logger.info('Connecting to %s:%s', host, port)
self.logger.log(BLATHER, ' Using session_id: %r session_passwd: %s',
client._session_id, hexlify(client._session_passwd))
with self._socket_error_handling():
self._socket = self.handler.create_connection(
(host, port), client._session_timeout / 1000.0)
self._socket.setblocking(0)
connect = Connect(0, client.last_zxid, client._session_timeout,
client._session_id or 0, client._session_passwd,
client.read_only)
connect_result, zxid = self._invoke(client._session_timeout, connect)
if connect_result.time_out <= 0:
raise SessionExpiredError("Session has expired")
if zxid:
client.last_zxid = zxid
# Load return values
client._session_id = connect_result.session_id
client._protocol_version = connect_result.protocol_version
negotiated_session_timeout = connect_result.time_out
connect_timeout = negotiated_session_timeout / len(client.hosts)
read_timeout = negotiated_session_timeout * 2.0 / 3.0
client._session_passwd = connect_result.passwd
self.logger.log(
BLATHER, 'Session created, session_id: %r session_passwd: %s\n'
' negotiated session timeout: %s\n'
' connect timeout: %s\n'
' read timeout: %s', client._session_id,
hexlify(client._session_passwd), negotiated_session_timeout,
connect_timeout, read_timeout)
if self.sasl_server_principal:
self._authenticate_with_sasl(host, connect_timeout / 1000.0)
if connect_result.read_only:
client._session_callback(KeeperState.CONNECTED_RO)
self._ro_mode = iter(self._server_pinger())
else:
client._session_callback(KeeperState.CONNECTED)
self._ro_mode = None
for scheme, auth in client.auth_data:
ap = Auth(0, scheme, auth)
zxid = self._invoke(connect_timeout, ap, xid=AUTH_XID)
if zxid:
client.last_zxid = zxid
return read_timeout, connect_timeout
def test_acquire_other_error(self):
"""
If acquire_eff internally raises any error then it tries to delete
child node before returning.
"""
seq = [(Constant(None), noop),
(zk.CreateNode("/testlock"), const("/testlock")),
(Func(uuid.uuid4), const("prefix")),
(zk.CreateNode("/testlock/prefix",
value="id",
ephemeral=True,
sequence=True),
const("/testlock/prefix0000000001")),
(GetChildren("/testlock"), conste(SessionExpiredError())),
(DeleteNode(path="/testlock/prefix0000000001",
version=-1), conste(SessionExpiredError()))]
self.assertRaises(SessionExpiredError, perform_sequence, seq,
self.lock.acquire_eff(True, 0.3))
def _call(self, request, async_object):
# Before kazoo==2.7.0 it wasn't possible to send requests to zookeeper if
# the connection is in the SUSPENDED state and Patroni was strongly relying on it.
# The https://github.com/python-zk/kazoo/pull/588 changed it, and now such requests are queued.
# We override the `_call()` method in order to keep the old behavior.
if self._state == KeeperState.CONNECTING:
async_object.set_exception(SessionExpiredError())
return False
return super(PatroniKazooClient, self)._call(request, async_object)
def _connect(self, host, port):
client = self.client
log.info('Connecting to %s:%s', host, port)
if self.log_debug:
log.debug(' Using session_id: %r session_passwd: %s',
client._session_id,
hexlify(client._session_passwd))
with socket_error_handling():
self._socket.connect((host, port))
self._socket.setblocking(0)
connect = Connect(0, client.last_zxid, client._session_timeout,
client._session_id or 0, client._session_passwd,
client.read_only)
connect_result, zxid = self._invoke(client._session_timeout, connect)
if connect_result.time_out <= 0:
raise SessionExpiredError("Session has expired")
if zxid:
client.last_zxid = zxid
# Load return values
client._session_id = connect_result.session_id
negotiated_session_timeout = connect_result.time_out
connect_timeout = negotiated_session_timeout / len(client.hosts)
read_timeout = negotiated_session_timeout * 2.0 / 3.0
client._session_passwd = connect_result.passwd
if self.log_debug:
log.debug('Session created, session_id: %r session_passwd: %s\n'
' negotiated session timeout: %s\n'
' connect timeout: %s\n'
' read timeout: %s', client._session_id,
hexlify(client._session_passwd),
negotiated_session_timeout, connect_timeout,
read_timeout)
if connect_result.read_only:
client._session_callback(KeeperState.CONNECTED_RO)
self._ro_mode = iter(self._server_pinger())
else:
client._session_callback(KeeperState.CONNECTED)
self._ro_mode = None
for scheme, auth in client.auth_data:
ap = Auth(0, scheme, auth)
zxid = self._invoke(connect_timeout, ap, xid=-4)
if zxid:
client.last_zxid = zxid
return read_timeout, connect_timeout
def test_release_no_node_reset(self):
"""
If release_eff fails to delete child node, it will not set self._node
to None
"""
node = self.lock._node = "/testlock/prefix0000000001"
seq = [(DeleteNode(path=self.lock._node,
version=-1), conste(SessionExpiredError()))]
self.assertRaises(SessionExpiredError, perform_sequence, seq,
self.lock.release_eff())
self.assertIs(self.lock._node, node)
def _call(self, request, async_object):
"""Ensure there's an active connection and put the request in
the queue if there is."""
with self._state_lock:
if self._state == KeeperState.AUTH_FAILED:
raise AuthFailedError()
elif self._state == KeeperState.CLOSED:
raise ConnectionClosedError("Connection has been closed")
elif self._state in (KeeperState.EXPIRED_SESSION,
KeeperState.CONNECTING):
raise SessionExpiredError()
self._queue.put((request, async_object))
def _call(self, request, async_object):
"""Ensure there's an active connection and put the request in
the queue if there is."""
if self._state == KeeperState.AUTH_FAILED:
raise AuthFailedError()
elif self._state == KeeperState.CLOSED:
raise ConnectionClosedError("Connection has been closed")
elif self._state in (KeeperState.EXPIRED_SESSION,
KeeperState.CONNECTING):
raise SessionExpiredError()
self._queue.append((request, async_object))
# wake the connection, guarding against a race with close()
write_pipe = self._connection._write_pipe
if write_pipe is None:
raise ConnectionClosedError("Connection has been closed")
try:
os.write(write_pipe, b'\0')
except OSError as e:
if e.errno == errno.EBADF:
raise ConnectionClosedError("Connection has been closed")
raise
def testit():
raise SessionExpiredError()
def _connect(self, host, port):
client = self.client
self.logger.info('Connecting to %s:%s', host, port)
self.logger.log(BLATHER, ' Using session_id: %r session_passwd: %s',
client._session_id, hexlify(client._session_passwd))
with self._socket_error_handling():
self._socket = self.handler.create_connection(
(host, port), client._session_timeout / 1000.0)
self._socket.setblocking(0)
connect = Connect(0, client.last_zxid, client._session_timeout,
client._session_id or 0, client._session_passwd,
client.read_only)
# save the client's last_zxid before it gets overwritten by the server's.
# we'll need this to reset watches via SetWatches further below.
last_zxid = client.last_zxid
connect_result, zxid = self._invoke(client._session_timeout / 1000.0,
connect)
if connect_result.time_out <= 0:
raise SessionExpiredError("Session has expired")
if zxid:
client.last_zxid = zxid
# Load return values
client._session_id = connect_result.session_id
client._protocol_version = connect_result.protocol_version
negotiated_session_timeout = connect_result.time_out
connect_timeout = negotiated_session_timeout / len(client.hosts)
read_timeout = negotiated_session_timeout * 2.0 / 3.0
client._session_passwd = connect_result.passwd
self.logger.log(
BLATHER, 'Session created, session_id: %r session_passwd: %s\n'
' negotiated session timeout: %s\n'
' connect timeout: %s\n'
' read timeout: %s', client._session_id,
hexlify(client._session_passwd), negotiated_session_timeout,
connect_timeout, read_timeout)
if connect_result.read_only:
client._session_callback(KeeperState.CONNECTED_RO)
self._ro_mode = iter(self._server_pinger())
else:
client._session_callback(KeeperState.CONNECTED)
self._ro_mode = None
for scheme, auth in client.auth_data:
ap = Auth(0, scheme, auth)
zxid = self._invoke(connect_timeout / 1000.0, ap, xid=AUTH_XID)
if zxid:
client.last_zxid = zxid
# TODO: separate exist from data watches
if client._data_watchers or client._child_watchers.keys():
sw = SetWatches(last_zxid, client._data_watchers.keys(),
client._data_watchers.keys(),
client._child_watchers.keys())
zxid = self._invoke(connect_timeout / 1000.0,
sw,
xid=SET_WATCHES_XID)
if zxid:
client.last_zxid = zxid
return read_timeout, connect_timeout
def _connect(self, host, hostip, port):
client = self.client
self.logger.info('Connecting to %s(%s):%s, use_ssl: %r', host, hostip,
port, self.client.use_ssl)
self.logger.log(BLATHER, ' Using session_id: %r session_passwd: %s',
client._session_id, hexlify(client._session_passwd))
with self._socket_error_handling():
self._socket = self.handler.create_connection(
address=(hostip, port),
timeout=client._session_timeout / 1000.0,
use_ssl=self.client.use_ssl,
keyfile=self.client.keyfile,
certfile=self.client.certfile,
ca=self.client.ca,
keyfile_password=self.client.keyfile_password,
verify_certs=self.client.verify_certs,
)
self._socket.setblocking(0)
connect = Connect(0, client.last_zxid, client._session_timeout,
client._session_id or 0, client._session_passwd,
client.read_only)
connect_result, zxid = self._invoke(
client._session_timeout / 1000.0 / len(client.hosts), connect)
if connect_result.time_out <= 0:
raise SessionExpiredError("Session has expired")
if zxid:
client.last_zxid = zxid
# Load return values
client._session_id = connect_result.session_id
client._protocol_version = connect_result.protocol_version
negotiated_session_timeout = connect_result.time_out
connect_timeout = negotiated_session_timeout / len(client.hosts)
read_timeout = negotiated_session_timeout * 2.0 / 3.0
client._session_passwd = connect_result.passwd
self.logger.log(
BLATHER, 'Session created, session_id: %r session_passwd: %s\n'
' negotiated session timeout: %s\n'
' connect timeout: %s\n'
' read timeout: %s', client._session_id,
hexlify(client._session_passwd), negotiated_session_timeout,
connect_timeout, read_timeout)
if connect_result.read_only:
client._session_callback(KeeperState.CONNECTED_RO)
self._ro_mode = iter(self._server_pinger())
else:
client._session_callback(KeeperState.CONNECTED)
self._ro_mode = None
if self.sasl_options is not None:
self._authenticate_with_sasl(host, connect_timeout / 1000.0)
# Get a copy of the auth data before iterating, in case it is
# changed.
client_auth_data_copy = copy.copy(client.auth_data)
for scheme, auth in client_auth_data_copy:
ap = Auth(0, scheme, auth)
zxid = self._invoke(connect_timeout / 1000.0, ap, xid=AUTH_XID)
if zxid:
client.last_zxid = zxid
return read_timeout, connect_timeout
def _connect(self, host, port):
client = self.client
self.logger.info('Connecting to %s:%s, use_ssl: %r', host, port,
self.client.use_ssl)
self.logger.log(BLATHER, ' Using session_id: %r session_passwd: %s',
client._session_id, hexlify(client._session_passwd))
with self._socket_error_handling():
self._socket = self.handler.create_connection(
address=(host, port),
timeout=client._session_timeout / 1000.0,
use_ssl=self.client.use_ssl,
keyfile=self.client.keyfile,
certfile=self.client.certfile,
ca=self.client.ca,
keyfile_password=self.client.keyfile_password,
verify_certs=self.client.verify_certs,
)
self._socket.setblocking(0)
connect = Connect(0, client.last_zxid, client._session_timeout,
client._session_id or 0, client._session_passwd,
client.read_only)
connect_result, zxid = self._invoke(client._session_timeout / 1000.0,
connect)
if connect_result.time_out <= 0:
raise SessionExpiredError("Session has expired")
if zxid:
client.last_zxid = zxid
# Load return values
client._session_id = connect_result.session_id
client._protocol_version = connect_result.protocol_version
negotiated_session_timeout = connect_result.time_out
connect_timeout = negotiated_session_timeout / len(client.hosts)
read_timeout = negotiated_session_timeout * 2.0 / 3.0
client._session_passwd = connect_result.passwd
self.logger.log(
BLATHER, 'Session created, session_id: %r session_passwd: %s\n'
' negotiated session timeout: %s\n'
' connect timeout: %s\n'
' read timeout: %s', client._session_id,
hexlify(client._session_passwd), negotiated_session_timeout,
connect_timeout, read_timeout)
if connect_result.read_only:
client._session_callback(KeeperState.CONNECTED_RO)
self._ro_mode = iter(self._server_pinger())
else:
self._ro_mode = None
# Get a copy of the auth data before iterating, in case it is
# changed.
client_auth_data_copy = copy.copy(client.auth_data)
if client.use_sasl and self.sasl_cli is None:
if PURESASL_AVAILABLE:
for scheme, auth in client_auth_data_copy:
if scheme == 'sasl':
username, password = auth.split(":")
self.sasl_cli = SASLClient(
host=client.sasl_server_principal,
service='zookeeper',
mechanism='DIGEST-MD5',
username=username,
password=password)
break
# As described in rfc
# https://tools.ietf.org/html/rfc2831#section-2.1
# sending empty challenge
self._send_sasl_request(challenge=b'',
timeout=connect_timeout)
else:
self.logger.warn('Pure-sasl library is missing while sasl'
' authentification is configured. Please'
' install pure-sasl library to connect '
'using sasl. Now falling back '
'connecting WITHOUT any '
'authentification.')
client.use_sasl = False
client._session_callback(KeeperState.CONNECTED)
else:
client._session_callback(KeeperState.CONNECTED)
for scheme, auth in client_auth_data_copy:
if scheme == "digest":
ap = Auth(0, scheme, auth)
zxid = self._invoke(connect_timeout / 1000.0,
ap,
xid=AUTH_XID)
if zxid:
client.last_zxid = zxid
return read_timeout, connect_timeout