def _getconnection(self,
container,
host,
path,
https=False,
forcecreate=False,
cafile=None,
key=None,
certificate=None,
timeout=None):
if not host:
raise ValueError
matcher = WebClientRequestDoneEvent.createMatcher(host, path, https)
while self.sameurllimit and (host, path, https) in self._requesting:
self._pathwaiting.add((host, path, https))
yield (matcher, )
# Lock the path
if self.sameurllimit:
self._requesting.add((host, path, https))
# connmap format: (free, free_ssl, workingcount)
conns = self._connmap.setdefault(host, [[], [], 0])
conns[0] = [c for c in conns[0] if c.connected]
conns[1] = [c for c in conns[1] if c.connected]
myset = conns[1 if https else 0]
if not forcecreate and myset:
# There are free connections, reuse them
conn = myset.pop()
conn.setdaemon(False)
container.retvalue = (conn, False)
conns[2] += 1
return
matcher = WebClientRequestDoneEvent.createMatcher(host)
while self.samehostlimit and len(conns[0]) + len(
conns[1]) + conns[2] >= self.samehostlimit:
if myset:
# Close a old connection
conn = myset.pop()
for m in conn.shutdown():
yield m
else:
# Wait for free connections
self._hostwaiting.add(host)
yield (matcher, )
conns = self._connmap.setdefault(host, [[], [], 0])
myset = conns[1 if https else 0]
if not forcecreate and myset:
conn = myset.pop()
conn.setdaemon(False)
container.retvalue = (conn, False)
conns[2] += 1
return
# Create new connection
conns[2] += 1
conn = Client(
urlunsplit(('ssl' if https else 'tcp', host, '/', '', '')),
self._protocol, container.scheduler, key, certificate, cafile)
if timeout is not None:
conn.connect_timeout = timeout
conn.start()
connected = self._protocol.statematcher(
conn, HttpConnectionStateEvent.CLIENT_CONNECTED, False)
notconnected = self._protocol.statematcher(
conn, HttpConnectionStateEvent.CLIENT_NOTCONNECTED, False)
yield (connected, notconnected)
if container.matcher is notconnected:
conns[2] -= 1
for m in conn.shutdown(True):
yield m
raise IOError('Failed to connect to %r' % (conn.rawurl, ))
if https and cafile and self.verifyhost:
try:
# TODO: check with SSLContext
hostcheck = re.sub(r':\d+$', '', host)
if host == conn.socket.remoteaddr[0]:
# IP Address is currently now allowed
for m in conn.shutdown(True):
yield m
raise CertificateException(
'Cannot verify host with IP address')
match_hostname(conn.socket.getpeercert(False), hostcheck)
except:
conns[2] -= 1
raise
container.retvalue = (conn, True)
def _connection_manage(self):
try:
failed = 0
self._last_zxid = last_zxid = 0
session_id = 0
passwd = b'\x00' * 16
last_conn_time = None
while True:
self.currentserver = self.serverlist[self.nextptr]
np = self.nextptr + 1
if np >= len(self.serverlist):
np = 0
self.nextptr = np
conn = Client(self.currentserver, self.protocol,
self._container.scheduler, self.key,
self.certificate, self.ca_certs)
self.current_connection = conn
conn_up = ZooKeeperConnectionStateEvent.createMatcher(
ZooKeeperConnectionStateEvent.UP, conn)
conn_nc = ZooKeeperConnectionStateEvent.createMatcher(
ZooKeeperConnectionStateEvent.NOTCONNECTED, conn)
conn.start()
try:
yield (conn_up, conn_nc)
if self._container.matcher is conn_nc:
self._logger.warning(
'Connect to %r failed, try next server',
self.currentserver)
if failed > 5:
# Wait for a small amount of time to prevent a busy loop
# Socket may be rejected, it may fail very quick
for m in self._container.waitWithTimeout(
min((failed - 5) * 0.1, 1.0)):
yield m
failed += 1
continue
try:
# Handshake
set_watches = []
if self.session_state == ZooKeeperSessionStateChanged.DISCONNECTED:
for m in self._container.waitForSend(
ZooKeeperRestoreWatches(
self,
self.session_id,
True,
restore_watches=(set(), set(),
set()))):
yield m
yield (
ZooKeeperRestoreWatches.createMatcher(self), )
data_watches, exists_watches, child_watches = \
self._container.event.restore_watches
if data_watches or exists_watches or child_watches:
current_set_watches = zk.SetWatches(
relativeZxid=last_zxid)
current_length = 0
for d, e, c in izip_longest(
data_watches, exists_watches,
child_watches):
if d is not None:
current_set_watches.dataWatches.append(
d)
current_length += 4 + len(d)
if e is not None:
current_set_watches.existWatches.append(
e)
current_length += 4 + len(e)
if c is not None:
current_set_watches.childWatches.append(
c)
current_length += 4 + len(c)
if current_length > _MAX_SETWATCHES_SIZE:
# Split set_watches
set_watches.append(current_set_watches)
current_set_watches = zk.SetWatches(
relativeZxid=last_zxid)
if current_set_watches.dataWatches or current_set_watches.existWatches \
or current_set_watches.childWatches:
set_watches.append(current_set_watches)
auth_list = list(self.auth_set)
with closing(
self._container.executeWithTimeout(
10,
self.protocol.handshake(
conn,
zk.ConnectRequest(
lastZxidSeen=last_zxid,
timeOut=int(self.sessiontimeout *
1000.0),
sessionId=session_id,
passwd=passwd,
readOnly=self.readonly),
self._container, [
zk.AuthPacket(scheme=a[0],
auth=a[1])
for a in auth_list
] + set_watches))) as g:
for m in g:
yield m
if self._container.timeout:
raise IOError
except ZooKeeperSessionExpiredException:
self._logger.warning('Session expired.')
# Session expired
self.session_state = ZooKeeperSessionStateChanged.EXPIRED
for m in self._container.waitForSend(
ZooKeeperSessionStateChanged(
ZooKeeperSessionStateChanged.EXPIRED, self,
session_id)):
yield m
if self.restart_session:
failed = 0
last_zxid = 0
session_id = 0
passwd = b'\x00' * 16
last_conn_time = None
continue
else:
break
except Exception:
self._logger.warning(
'Handshake failed to %r, try next server',
self.currentserver)
if failed > 5:
# There is a bug ZOOKEEPER-1159 that ZooKeeper server does not respond
# for session expiration, but directly close the connection.
# This is a workaround: we store the time that we disconnected from the server,
# if we have exceeded the session expiration time, we declare the session is expired
if last_conn_time is not None and last_conn_time + self.sessiontimeout * 2 < time(
):
self._logger.warning(
'Session expired detected from client time.'
)
# Session expired
self.session_state = ZooKeeperSessionStateChanged.EXPIRED
for m in self._container.waitForSend(
ZooKeeperSessionStateChanged(
ZooKeeperSessionStateChanged.
EXPIRED, self, session_id)):
yield m
if self.restart_session:
failed = 0
last_zxid = 0
session_id = 0
passwd = b'\x00' * 16
last_conn_time = None
continue
else:
break
else:
# Wait for a small amount of time to prevent a busy loop
for m in self._container.waitWithTimeout(
min((failed - 5) * 0.1, 1.0)):
yield m
failed += 1
else:
failed = 0
conn_resp, auth_resp = self._container.retvalue
if conn_resp.timeOut <= 0:
# Session expired
# Currently should not happen because handshake() should raise an exception
self._logger.warning(
'Session expired detected from handshake packet'
)
self.session_state = ZooKeeperSessionStateChanged.EXPIRED
for m in self._container.waitForSend(
ZooKeeperSessionStateChanged(
ZooKeeperSessionStateChanged.EXPIRED,
self, session_id)):
yield m
if self.restart_session:
failed = 0
last_zxid = 0
last_conn_time = None
session_id = 0
passwd = b'\x00' * 16
continue
else:
break
else:
session_id = conn_resp.sessionId
passwd = conn_resp.passwd
# Authentication result check
auth_failed = any(a.err == zk.ZOO_ERR_AUTHFAILED
for a in auth_resp)
if auth_failed:
self._logger.warning(
'ZooKeeper authentication failed for following auth: %r',
[
a
for a, r in zip(auth_list, auth_resp)
if r.err == zk.ZOO_ERR_AUTHFAILED
])
self.session_state = ZooKeeperSessionStateChanged.AUTHFAILED
for m in self._container.waitForSend(
ZooKeeperSessionStateChanged(
ZooKeeperSessionStateChanged.
AUTHFAILED, self, session_id)):
yield m
# Not retrying
break
else:
self.session_readonly = getattr(
conn_resp, 'readOnly', False)
self.session_id = session_id
if self.session_state == ZooKeeperSessionStateChanged.EXPIRED:
for m in self._container.waitForSend(
ZooKeeperSessionStateChanged(
ZooKeeperSessionStateChanged.
CREATED, self, session_id)):
yield m
else:
for m in self._container.waitForSend(
ZooKeeperSessionStateChanged(
ZooKeeperSessionStateChanged.
RECONNECTED, self,
session_id)):
yield m
self.session_state = ZooKeeperSessionStateChanged.CREATED
if conn.connected:
conn_down = ZooKeeperConnectionStateEvent.createMatcher(
ZooKeeperConnectionStateEvent.DOWN, conn,
conn.connmark)
auth_failed = ZooKeeperResponseEvent.createMatcher(
zk.AUTH_XID,
conn,
conn.connmark,
_ismatch=lambda x: x.message.err ==
ZOO_ERR_AUTHFAILED)
while True:
rebalancetime = self.rebalancetime
if rebalancetime is not None:
rebalancetime += random() * 60
for m in self._container.waitWithTimeout(
rebalancetime, conn_down, auth_failed):
yield m
if self._container.timeout:
# Rebalance
if conn.zookeeper_requests:
# There are still requests not processed, wait longer
for _ in range(0, 3):
longer_time = random() * 10
for m in self._container.waitWithTimeout(
longer_time, conn_down,
auth_failed):
yield m
if not self._container.timeout:
# Connection is down, or auth failed
break
if not conn.zookeeper_requests:
break
else:
# There is still requests, skip for this time
continue
# Rebalance to a random server
if self._container.timeout:
self.nextptr = randrange(
len(self.serverlist))
break
if self._container.matcher is auth_failed:
self._logger.warning(
'ZooKeeper authentication failed, shutdown the connection'
)
self.session_state = ZooKeeperSessionStateChanged.AUTHFAILED
for m in self._container.waitForSend(
ZooKeeperSessionStateChanged(
ZooKeeperSessionStateChanged.
AUTHFAILED, self, session_id)):
yield m
# Not retrying
break
else:
# Connection is down, try other servers
if not self._container.timeout:
self._logger.warning(
'Connection lost to %r, try next server',
self.currentserver)
else:
self._logger.info(
'Rebalance to next server')
self._last_zxid = last_zxid = conn.zookeeper_lastzxid
last_conn_time = time()
self.session_state = ZooKeeperSessionStateChanged.DISCONNECTED
for m in self._container.waitForSend(
ZooKeeperSessionStateChanged(
ZooKeeperSessionStateChanged.
DISCONNECTED, self, session_id)):
yield m
finally:
conn.subroutine(conn.shutdown(True), False)
self.current_connection = None
finally:
self._shutdown = True
if self.session_state != ZooKeeperSessionStateChanged.EXPIRED and self.session_state != ZooKeeperSessionStateChanged.AUTHFAILED:
self.session_state = ZooKeeperSessionStateChanged.EXPIRED
self._container.scheduler.emergesend(
ZooKeeperSessionStateChanged(
ZooKeeperSessionStateChanged.EXPIRED, self,
session_id))
def _connection_manage(self):
try:
failed = 0
self._last_zxid = last_zxid = 0
session_id = 0
passwd = b'\x00' * 16
last_conn_time = None
while True:
self.currentserver = self.serverlist[self.nextptr]
np = self.nextptr + 1
if np >= len(self.serverlist):
np = 0
self.nextptr = np
conn = Client(self.currentserver, self.protocol, self._container.scheduler,
self.key, self.certificate, self.ca_certs)
self.current_connection = conn
conn_up = ZooKeeperConnectionStateEvent.createMatcher(ZooKeeperConnectionStateEvent.UP,
conn)
conn_nc = ZooKeeperConnectionStateEvent.createMatcher(ZooKeeperConnectionStateEvent.NOTCONNECTED,
conn)
conn.start()
try:
yield (conn_up, conn_nc)
if self._container.matcher is conn_nc:
self._logger.warning('Connect to %r failed, try next server', self.currentserver)
if failed > 5:
# Wait for a small amount of time to prevent a busy loop
# Socket may be rejected, it may fail very quick
for m in self._container.waitWithTimeout(min((failed - 5) * 0.1, 1.0)):
yield m
failed += 1
continue
try:
# Handshake
set_watches = []
if self.session_state == ZooKeeperSessionStateChanged.DISCONNECTED:
for m in self._container.waitForSend(ZooKeeperRestoreWatches(self,
self.session_id,
True,
restore_watches = (set(), set(), set()))):
yield m
yield (ZooKeeperRestoreWatches.createMatcher(self),)
data_watches, exists_watches, child_watches = \
self._container.event.restore_watches
if data_watches or exists_watches or child_watches:
current_set_watches = zk.SetWatches(relativeZxid = last_zxid)
current_length = 0
for d, e, c in izip_longest(data_watches, exists_watches, child_watches):
if d is not None:
current_set_watches.dataWatches.append(d)
current_length += 4 + len(d)
if e is not None:
current_set_watches.existWatches.append(e)
current_length += 4 + len(e)
if c is not None:
current_set_watches.childWatches.append(c)
current_length += 4 + len(c)
if current_length > _MAX_SETWATCHES_SIZE:
# Split set_watches
set_watches.append(current_set_watches)
current_set_watches = zk.SetWatches(relativeZxid = last_zxid)
if current_set_watches.dataWatches or current_set_watches.existWatches \
or current_set_watches.childWatches:
set_watches.append(current_set_watches)
auth_list = list(self.auth_set)
with closing(self._container.executeWithTimeout(10,
self.protocol.handshake(conn,
zk.ConnectRequest(lastZxidSeen = last_zxid,
timeOut = int(self.sessiontimeout * 1000.0),
sessionId = session_id,
passwd = passwd,
readOnly = self.readonly),
self._container,
[zk.AuthPacket(scheme = a[0], auth = a[1]) for a in auth_list] +
set_watches))) as g:
for m in g:
yield m
if self._container.timeout:
raise IOError
except ZooKeeperSessionExpiredException:
self._logger.warning('Session expired.')
# Session expired
self.session_state = ZooKeeperSessionStateChanged.EXPIRED
for m in self._container.waitForSend(ZooKeeperSessionStateChanged(
ZooKeeperSessionStateChanged.EXPIRED,
self,
session_id)):
yield m
if self.restart_session:
failed = 0
last_zxid = 0
session_id = 0
passwd = b'\x00' * 16
last_conn_time = None
continue
else:
break
except Exception:
self._logger.warning('Handshake failed to %r, try next server', self.currentserver)
if failed > 5:
# There is a bug ZOOKEEPER-1159 that ZooKeeper server does not respond
# for session expiration, but directly close the connection.
# This is a workaround: we store the time that we disconnected from the server,
# if we have exceeded the session expiration time, we declare the session is expired
if last_conn_time is not None and last_conn_time + self.sessiontimeout * 2 < time():
self._logger.warning('Session expired detected from client time.')
# Session expired
self.session_state = ZooKeeperSessionStateChanged.EXPIRED
for m in self._container.waitForSend(ZooKeeperSessionStateChanged(
ZooKeeperSessionStateChanged.EXPIRED,
self,
session_id)):
yield m
if self.restart_session:
failed = 0
last_zxid = 0
session_id = 0
passwd = b'\x00' * 16
last_conn_time = None
continue
else:
break
else:
# Wait for a small amount of time to prevent a busy loop
for m in self._container.waitWithTimeout(min((failed - 5) * 0.1, 1.0)):
yield m
failed += 1
else:
failed = 0
conn_resp, auth_resp = self._container.retvalue
if conn_resp.timeOut <= 0:
# Session expired
# Currently should not happen because handshake() should raise an exception
self._logger.warning('Session expired detected from handshake packet')
self.session_state = ZooKeeperSessionStateChanged.EXPIRED
for m in self._container.waitForSend(ZooKeeperSessionStateChanged(
ZooKeeperSessionStateChanged.EXPIRED,
self,
session_id)):
yield m
if self.restart_session:
failed = 0
last_zxid = 0
last_conn_time = None
session_id = 0
passwd = b'\x00' * 16
continue
else:
break
else:
session_id = conn_resp.sessionId
passwd = conn_resp.passwd
# Authentication result check
auth_failed = any(a.err == zk.ZOO_ERR_AUTHFAILED for a in auth_resp)
if auth_failed:
self._logger.warning('ZooKeeper authentication failed for following auth: %r',
[a for a,r in zip(auth_list, auth_resp) if r.err == zk.ZOO_ERR_AUTHFAILED])
self.session_state = ZooKeeperSessionStateChanged.AUTHFAILED
for m in self._container.waitForSend(ZooKeeperSessionStateChanged(
ZooKeeperSessionStateChanged.AUTHFAILED,
self,
session_id
)):
yield m
# Not retrying
break
else:
self.session_readonly = getattr(conn_resp, 'readOnly', False)
self.session_id = session_id
if self.session_state == ZooKeeperSessionStateChanged.EXPIRED:
for m in self._container.waitForSend(ZooKeeperSessionStateChanged(
ZooKeeperSessionStateChanged.CREATED,
self,
session_id
)):
yield m
else:
for m in self._container.waitForSend(ZooKeeperSessionStateChanged(
ZooKeeperSessionStateChanged.RECONNECTED,
self,
session_id
)):
yield m
self.session_state = ZooKeeperSessionStateChanged.CREATED
if conn.connected:
conn_down = ZooKeeperConnectionStateEvent.createMatcher(ZooKeeperConnectionStateEvent.DOWN,
conn,
conn.connmark
)
auth_failed = ZooKeeperResponseEvent.createMatcher(zk.AUTH_XID,
conn,
conn.connmark,
_ismatch = lambda x: x.message.err == ZOO_ERR_AUTHFAILED)
while True:
rebalancetime = self.rebalancetime
if rebalancetime is not None:
rebalancetime += random() * 60
for m in self._container.waitWithTimeout(rebalancetime, conn_down, auth_failed):
yield m
if self._container.timeout:
# Rebalance
if conn.zookeeper_requests:
# There are still requests not processed, wait longer
for _ in range(0, 3):
longer_time = random() * 10
for m in self._container.waitWithTimeout(longer_time, conn_down, auth_failed):
yield m
if not self._container.timeout:
# Connection is down, or auth failed
break
if not conn.zookeeper_requests:
break
else:
# There is still requests, skip for this time
continue
# Rebalance to a random server
if self._container.timeout:
self.nextptr = randrange(len(self.serverlist))
break
if self._container.matcher is auth_failed:
self._logger.warning('ZooKeeper authentication failed, shutdown the connection')
self.session_state = ZooKeeperSessionStateChanged.AUTHFAILED
for m in self._container.waitForSend(ZooKeeperSessionStateChanged(
ZooKeeperSessionStateChanged.AUTHFAILED,
self,
session_id
)):
yield m
# Not retrying
break
else:
# Connection is down, try other servers
if not self._container.timeout:
self._logger.warning('Connection lost to %r, try next server', self.currentserver)
else:
self._logger.info('Rebalance to next server')
self._last_zxid = last_zxid = conn.zookeeper_lastzxid
last_conn_time = time()
self.session_state = ZooKeeperSessionStateChanged.DISCONNECTED
for m in self._container.waitForSend(ZooKeeperSessionStateChanged(
ZooKeeperSessionStateChanged.DISCONNECTED,
self,
session_id
)):
yield m
finally:
conn.subroutine(conn.shutdown(True), False)
self.current_connection = None
finally:
self._shutdown = True
if self.session_state != ZooKeeperSessionStateChanged.EXPIRED and self.session_state != ZooKeeperSessionStateChanged.AUTHFAILED:
self.session_state = ZooKeeperSessionStateChanged.EXPIRED
self._container.scheduler.emergesend(ZooKeeperSessionStateChanged(
ZooKeeperSessionStateChanged.EXPIRED,
self,
session_id
))
def _getconnection(self, container, host, path, https = False, forcecreate = False, cafile = None, key = None, certificate = None,
timeout = None):
if not host:
raise ValueError
matcher = WebClientRequestDoneEvent.createMatcher(host, path, https)
while self.sameurllimit and (host, path, https) in self._requesting:
self._pathwaiting.add((host, path, https))
yield (matcher,)
# Lock the path
if self.sameurllimit:
self._requesting.add((host, path, https))
# connmap format: (free, free_ssl, workingcount)
conns = self._connmap.setdefault(host, [[],[], 0])
conns[0] = [c for c in conns[0] if c.connected]
conns[1] = [c for c in conns[1] if c.connected]
myset = conns[1 if https else 0]
if not forcecreate and myset:
# There are free connections, reuse them
conn = myset.pop()
conn.setdaemon(False)
container.retvalue = (conn, False)
conns[2] += 1
return
matcher = WebClientRequestDoneEvent.createMatcher(host)
while self.samehostlimit and len(conns[0]) + len(conns[1]) + conns[2] >= self.samehostlimit:
if myset:
# Close a old connection
conn = myset.pop()
for m in conn.shutdown():
yield m
else:
# Wait for free connections
self._hostwaiting.add(host)
yield (matcher,)
conns = self._connmap.setdefault(host, [[],[], 0])
myset = conns[1 if https else 0]
if not forcecreate and myset:
conn = myset.pop()
conn.setdaemon(False)
container.retvalue = (conn, False)
conns[2] += 1
return
# Create new connection
conns[2] += 1
conn = Client(urlunsplit(('ssl' if https else 'tcp', host, '/', '', '')), self._protocol, container.scheduler,
key, certificate, cafile)
if timeout is not None:
conn.connect_timeout = timeout
conn.start()
connected = self._protocol.statematcher(conn, HttpConnectionStateEvent.CLIENT_CONNECTED, False)
notconnected = self._protocol.statematcher(conn, HttpConnectionStateEvent.CLIENT_NOTCONNECTED, False)
yield (connected, notconnected)
if container.matcher is notconnected:
conns[2] -= 1
for m in conn.shutdown(True):
yield m
raise IOError('Failed to connect to %r' % (conn.rawurl,))
if https and cafile and self.verifyhost:
try:
# TODO: check with SSLContext
hostcheck = re.sub(r':\d+$', '', host)
if host == conn.socket.remoteaddr[0]:
# IP Address is currently now allowed
for m in conn.shutdown(True):
yield m
raise CertificateException('Cannot verify host with IP address')
match_hostname(conn.socket.getpeercert(False), hostcheck)
except:
conns[2] -= 1
raise
container.retvalue = (conn, True)
