def _getconnection(self, container, host, path, https=False, forcecreate=False, cafile=None, key=None, certificate=None, timeout=None): if not host: raise ValueError matcher = WebClientRequestDoneEvent.createMatcher(host, path, https) while self.sameurllimit and (host, path, https) in self._requesting: self._pathwaiting.add((host, path, https)) yield (matcher, ) # Lock the path if self.sameurllimit: self._requesting.add((host, path, https)) # connmap format: (free, free_ssl, workingcount) conns = self._connmap.setdefault(host, [[], [], 0]) conns[0] = [c for c in conns[0] if c.connected] conns[1] = [c for c in conns[1] if c.connected] myset = conns[1 if https else 0] if not forcecreate and myset: # There are free connections, reuse them conn = myset.pop() conn.setdaemon(False) container.retvalue = (conn, False) conns[2] += 1 return matcher = WebClientRequestDoneEvent.createMatcher(host) while self.samehostlimit and len(conns[0]) + len( conns[1]) + conns[2] >= self.samehostlimit: if myset: # Close a old connection conn = myset.pop() for m in conn.shutdown(): yield m else: # Wait for free connections self._hostwaiting.add(host) yield (matcher, ) conns = self._connmap.setdefault(host, [[], [], 0]) myset = conns[1 if https else 0] if not forcecreate and myset: conn = myset.pop() conn.setdaemon(False) container.retvalue = (conn, False) conns[2] += 1 return # Create new connection conns[2] += 1 conn = Client( urlunsplit(('ssl' if https else 'tcp', host, '/', '', '')), self._protocol, container.scheduler, key, certificate, cafile) if timeout is not None: conn.connect_timeout = timeout conn.start() connected = self._protocol.statematcher( conn, HttpConnectionStateEvent.CLIENT_CONNECTED, False) notconnected = self._protocol.statematcher( conn, HttpConnectionStateEvent.CLIENT_NOTCONNECTED, False) yield (connected, notconnected) if container.matcher is notconnected: conns[2] -= 1 for m in conn.shutdown(True): yield m raise IOError('Failed to connect to %r' % (conn.rawurl, )) if https and cafile and self.verifyhost: try: # TODO: check with SSLContext hostcheck = re.sub(r':\d+$', '', host) if host == conn.socket.remoteaddr[0]: # IP Address is currently now allowed for m in conn.shutdown(True): yield m raise CertificateException( 'Cannot verify host with IP address') match_hostname(conn.socket.getpeercert(False), hostcheck) except: conns[2] -= 1 raise container.retvalue = (conn, True)
def _connection_manage(self): try: failed = 0 self._last_zxid = last_zxid = 0 session_id = 0 passwd = b'\x00' * 16 last_conn_time = None while True: self.currentserver = self.serverlist[self.nextptr] np = self.nextptr + 1 if np >= len(self.serverlist): np = 0 self.nextptr = np conn = Client(self.currentserver, self.protocol, self._container.scheduler, self.key, self.certificate, self.ca_certs) self.current_connection = conn conn_up = ZooKeeperConnectionStateEvent.createMatcher( ZooKeeperConnectionStateEvent.UP, conn) conn_nc = ZooKeeperConnectionStateEvent.createMatcher( ZooKeeperConnectionStateEvent.NOTCONNECTED, conn) conn.start() try: yield (conn_up, conn_nc) if self._container.matcher is conn_nc: self._logger.warning( 'Connect to %r failed, try next server', self.currentserver) if failed > 5: # Wait for a small amount of time to prevent a busy loop # Socket may be rejected, it may fail very quick for m in self._container.waitWithTimeout( min((failed - 5) * 0.1, 1.0)): yield m failed += 1 continue try: # Handshake set_watches = [] if self.session_state == ZooKeeperSessionStateChanged.DISCONNECTED: for m in self._container.waitForSend( ZooKeeperRestoreWatches( self, self.session_id, True, restore_watches=(set(), set(), set()))): yield m yield ( ZooKeeperRestoreWatches.createMatcher(self), ) data_watches, exists_watches, child_watches = \ self._container.event.restore_watches if data_watches or exists_watches or child_watches: current_set_watches = zk.SetWatches( relativeZxid=last_zxid) current_length = 0 for d, e, c in izip_longest( data_watches, exists_watches, child_watches): if d is not None: current_set_watches.dataWatches.append( d) current_length += 4 + len(d) if e is not None: current_set_watches.existWatches.append( e) current_length += 4 + len(e) if c is not None: current_set_watches.childWatches.append( c) current_length += 4 + len(c) if current_length > _MAX_SETWATCHES_SIZE: # Split set_watches set_watches.append(current_set_watches) current_set_watches = zk.SetWatches( relativeZxid=last_zxid) if current_set_watches.dataWatches or current_set_watches.existWatches \ or current_set_watches.childWatches: set_watches.append(current_set_watches) auth_list = list(self.auth_set) with closing( self._container.executeWithTimeout( 10, self.protocol.handshake( conn, zk.ConnectRequest( lastZxidSeen=last_zxid, timeOut=int(self.sessiontimeout * 1000.0), sessionId=session_id, passwd=passwd, readOnly=self.readonly), self._container, [ zk.AuthPacket(scheme=a[0], auth=a[1]) for a in auth_list ] + set_watches))) as g: for m in g: yield m if self._container.timeout: raise IOError except ZooKeeperSessionExpiredException: self._logger.warning('Session expired.') # Session expired self.session_state = ZooKeeperSessionStateChanged.EXPIRED for m in self._container.waitForSend( ZooKeeperSessionStateChanged( ZooKeeperSessionStateChanged.EXPIRED, self, session_id)): yield m if self.restart_session: failed = 0 last_zxid = 0 session_id = 0 passwd = b'\x00' * 16 last_conn_time = None continue else: break except Exception: self._logger.warning( 'Handshake failed to %r, try next server', self.currentserver) if failed > 5: # There is a bug ZOOKEEPER-1159 that ZooKeeper server does not respond # for session expiration, but directly close the connection. # This is a workaround: we store the time that we disconnected from the server, # if we have exceeded the session expiration time, we declare the session is expired if last_conn_time is not None and last_conn_time + self.sessiontimeout * 2 < time( ): self._logger.warning( 'Session expired detected from client time.' ) # Session expired self.session_state = ZooKeeperSessionStateChanged.EXPIRED for m in self._container.waitForSend( ZooKeeperSessionStateChanged( ZooKeeperSessionStateChanged. EXPIRED, self, session_id)): yield m if self.restart_session: failed = 0 last_zxid = 0 session_id = 0 passwd = b'\x00' * 16 last_conn_time = None continue else: break else: # Wait for a small amount of time to prevent a busy loop for m in self._container.waitWithTimeout( min((failed - 5) * 0.1, 1.0)): yield m failed += 1 else: failed = 0 conn_resp, auth_resp = self._container.retvalue if conn_resp.timeOut <= 0: # Session expired # Currently should not happen because handshake() should raise an exception self._logger.warning( 'Session expired detected from handshake packet' ) self.session_state = ZooKeeperSessionStateChanged.EXPIRED for m in self._container.waitForSend( ZooKeeperSessionStateChanged( ZooKeeperSessionStateChanged.EXPIRED, self, session_id)): yield m if self.restart_session: failed = 0 last_zxid = 0 last_conn_time = None session_id = 0 passwd = b'\x00' * 16 continue else: break else: session_id = conn_resp.sessionId passwd = conn_resp.passwd # Authentication result check auth_failed = any(a.err == zk.ZOO_ERR_AUTHFAILED for a in auth_resp) if auth_failed: self._logger.warning( 'ZooKeeper authentication failed for following auth: %r', [ a for a, r in zip(auth_list, auth_resp) if r.err == zk.ZOO_ERR_AUTHFAILED ]) self.session_state = ZooKeeperSessionStateChanged.AUTHFAILED for m in self._container.waitForSend( ZooKeeperSessionStateChanged( ZooKeeperSessionStateChanged. AUTHFAILED, self, session_id)): yield m # Not retrying break else: self.session_readonly = getattr( conn_resp, 'readOnly', False) self.session_id = session_id if self.session_state == ZooKeeperSessionStateChanged.EXPIRED: for m in self._container.waitForSend( ZooKeeperSessionStateChanged( ZooKeeperSessionStateChanged. CREATED, self, session_id)): yield m else: for m in self._container.waitForSend( ZooKeeperSessionStateChanged( ZooKeeperSessionStateChanged. RECONNECTED, self, session_id)): yield m self.session_state = ZooKeeperSessionStateChanged.CREATED if conn.connected: conn_down = ZooKeeperConnectionStateEvent.createMatcher( ZooKeeperConnectionStateEvent.DOWN, conn, conn.connmark) auth_failed = ZooKeeperResponseEvent.createMatcher( zk.AUTH_XID, conn, conn.connmark, _ismatch=lambda x: x.message.err == ZOO_ERR_AUTHFAILED) while True: rebalancetime = self.rebalancetime if rebalancetime is not None: rebalancetime += random() * 60 for m in self._container.waitWithTimeout( rebalancetime, conn_down, auth_failed): yield m if self._container.timeout: # Rebalance if conn.zookeeper_requests: # There are still requests not processed, wait longer for _ in range(0, 3): longer_time = random() * 10 for m in self._container.waitWithTimeout( longer_time, conn_down, auth_failed): yield m if not self._container.timeout: # Connection is down, or auth failed break if not conn.zookeeper_requests: break else: # There is still requests, skip for this time continue # Rebalance to a random server if self._container.timeout: self.nextptr = randrange( len(self.serverlist)) break if self._container.matcher is auth_failed: self._logger.warning( 'ZooKeeper authentication failed, shutdown the connection' ) self.session_state = ZooKeeperSessionStateChanged.AUTHFAILED for m in self._container.waitForSend( ZooKeeperSessionStateChanged( ZooKeeperSessionStateChanged. AUTHFAILED, self, session_id)): yield m # Not retrying break else: # Connection is down, try other servers if not self._container.timeout: self._logger.warning( 'Connection lost to %r, try next server', self.currentserver) else: self._logger.info( 'Rebalance to next server') self._last_zxid = last_zxid = conn.zookeeper_lastzxid last_conn_time = time() self.session_state = ZooKeeperSessionStateChanged.DISCONNECTED for m in self._container.waitForSend( ZooKeeperSessionStateChanged( ZooKeeperSessionStateChanged. DISCONNECTED, self, session_id)): yield m finally: conn.subroutine(conn.shutdown(True), False) self.current_connection = None finally: self._shutdown = True if self.session_state != ZooKeeperSessionStateChanged.EXPIRED and self.session_state != ZooKeeperSessionStateChanged.AUTHFAILED: self.session_state = ZooKeeperSessionStateChanged.EXPIRED self._container.scheduler.emergesend( ZooKeeperSessionStateChanged( ZooKeeperSessionStateChanged.EXPIRED, self, session_id))
def _connection_manage(self): try: failed = 0 self._last_zxid = last_zxid = 0 session_id = 0 passwd = b'\x00' * 16 last_conn_time = None while True: self.currentserver = self.serverlist[self.nextptr] np = self.nextptr + 1 if np >= len(self.serverlist): np = 0 self.nextptr = np conn = Client(self.currentserver, self.protocol, self._container.scheduler, self.key, self.certificate, self.ca_certs) self.current_connection = conn conn_up = ZooKeeperConnectionStateEvent.createMatcher(ZooKeeperConnectionStateEvent.UP, conn) conn_nc = ZooKeeperConnectionStateEvent.createMatcher(ZooKeeperConnectionStateEvent.NOTCONNECTED, conn) conn.start() try: yield (conn_up, conn_nc) if self._container.matcher is conn_nc: self._logger.warning('Connect to %r failed, try next server', self.currentserver) if failed > 5: # Wait for a small amount of time to prevent a busy loop # Socket may be rejected, it may fail very quick for m in self._container.waitWithTimeout(min((failed - 5) * 0.1, 1.0)): yield m failed += 1 continue try: # Handshake set_watches = [] if self.session_state == ZooKeeperSessionStateChanged.DISCONNECTED: for m in self._container.waitForSend(ZooKeeperRestoreWatches(self, self.session_id, True, restore_watches = (set(), set(), set()))): yield m yield (ZooKeeperRestoreWatches.createMatcher(self),) data_watches, exists_watches, child_watches = \ self._container.event.restore_watches if data_watches or exists_watches or child_watches: current_set_watches = zk.SetWatches(relativeZxid = last_zxid) current_length = 0 for d, e, c in izip_longest(data_watches, exists_watches, child_watches): if d is not None: current_set_watches.dataWatches.append(d) current_length += 4 + len(d) if e is not None: current_set_watches.existWatches.append(e) current_length += 4 + len(e) if c is not None: current_set_watches.childWatches.append(c) current_length += 4 + len(c) if current_length > _MAX_SETWATCHES_SIZE: # Split set_watches set_watches.append(current_set_watches) current_set_watches = zk.SetWatches(relativeZxid = last_zxid) if current_set_watches.dataWatches or current_set_watches.existWatches \ or current_set_watches.childWatches: set_watches.append(current_set_watches) auth_list = list(self.auth_set) with closing(self._container.executeWithTimeout(10, self.protocol.handshake(conn, zk.ConnectRequest(lastZxidSeen = last_zxid, timeOut = int(self.sessiontimeout * 1000.0), sessionId = session_id, passwd = passwd, readOnly = self.readonly), self._container, [zk.AuthPacket(scheme = a[0], auth = a[1]) for a in auth_list] + set_watches))) as g: for m in g: yield m if self._container.timeout: raise IOError except ZooKeeperSessionExpiredException: self._logger.warning('Session expired.') # Session expired self.session_state = ZooKeeperSessionStateChanged.EXPIRED for m in self._container.waitForSend(ZooKeeperSessionStateChanged( ZooKeeperSessionStateChanged.EXPIRED, self, session_id)): yield m if self.restart_session: failed = 0 last_zxid = 0 session_id = 0 passwd = b'\x00' * 16 last_conn_time = None continue else: break except Exception: self._logger.warning('Handshake failed to %r, try next server', self.currentserver) if failed > 5: # There is a bug ZOOKEEPER-1159 that ZooKeeper server does not respond # for session expiration, but directly close the connection. # This is a workaround: we store the time that we disconnected from the server, # if we have exceeded the session expiration time, we declare the session is expired if last_conn_time is not None and last_conn_time + self.sessiontimeout * 2 < time(): self._logger.warning('Session expired detected from client time.') # Session expired self.session_state = ZooKeeperSessionStateChanged.EXPIRED for m in self._container.waitForSend(ZooKeeperSessionStateChanged( ZooKeeperSessionStateChanged.EXPIRED, self, session_id)): yield m if self.restart_session: failed = 0 last_zxid = 0 session_id = 0 passwd = b'\x00' * 16 last_conn_time = None continue else: break else: # Wait for a small amount of time to prevent a busy loop for m in self._container.waitWithTimeout(min((failed - 5) * 0.1, 1.0)): yield m failed += 1 else: failed = 0 conn_resp, auth_resp = self._container.retvalue if conn_resp.timeOut <= 0: # Session expired # Currently should not happen because handshake() should raise an exception self._logger.warning('Session expired detected from handshake packet') self.session_state = ZooKeeperSessionStateChanged.EXPIRED for m in self._container.waitForSend(ZooKeeperSessionStateChanged( ZooKeeperSessionStateChanged.EXPIRED, self, session_id)): yield m if self.restart_session: failed = 0 last_zxid = 0 last_conn_time = None session_id = 0 passwd = b'\x00' * 16 continue else: break else: session_id = conn_resp.sessionId passwd = conn_resp.passwd # Authentication result check auth_failed = any(a.err == zk.ZOO_ERR_AUTHFAILED for a in auth_resp) if auth_failed: self._logger.warning('ZooKeeper authentication failed for following auth: %r', [a for a,r in zip(auth_list, auth_resp) if r.err == zk.ZOO_ERR_AUTHFAILED]) self.session_state = ZooKeeperSessionStateChanged.AUTHFAILED for m in self._container.waitForSend(ZooKeeperSessionStateChanged( ZooKeeperSessionStateChanged.AUTHFAILED, self, session_id )): yield m # Not retrying break else: self.session_readonly = getattr(conn_resp, 'readOnly', False) self.session_id = session_id if self.session_state == ZooKeeperSessionStateChanged.EXPIRED: for m in self._container.waitForSend(ZooKeeperSessionStateChanged( ZooKeeperSessionStateChanged.CREATED, self, session_id )): yield m else: for m in self._container.waitForSend(ZooKeeperSessionStateChanged( ZooKeeperSessionStateChanged.RECONNECTED, self, session_id )): yield m self.session_state = ZooKeeperSessionStateChanged.CREATED if conn.connected: conn_down = ZooKeeperConnectionStateEvent.createMatcher(ZooKeeperConnectionStateEvent.DOWN, conn, conn.connmark ) auth_failed = ZooKeeperResponseEvent.createMatcher(zk.AUTH_XID, conn, conn.connmark, _ismatch = lambda x: x.message.err == ZOO_ERR_AUTHFAILED) while True: rebalancetime = self.rebalancetime if rebalancetime is not None: rebalancetime += random() * 60 for m in self._container.waitWithTimeout(rebalancetime, conn_down, auth_failed): yield m if self._container.timeout: # Rebalance if conn.zookeeper_requests: # There are still requests not processed, wait longer for _ in range(0, 3): longer_time = random() * 10 for m in self._container.waitWithTimeout(longer_time, conn_down, auth_failed): yield m if not self._container.timeout: # Connection is down, or auth failed break if not conn.zookeeper_requests: break else: # There is still requests, skip for this time continue # Rebalance to a random server if self._container.timeout: self.nextptr = randrange(len(self.serverlist)) break if self._container.matcher is auth_failed: self._logger.warning('ZooKeeper authentication failed, shutdown the connection') self.session_state = ZooKeeperSessionStateChanged.AUTHFAILED for m in self._container.waitForSend(ZooKeeperSessionStateChanged( ZooKeeperSessionStateChanged.AUTHFAILED, self, session_id )): yield m # Not retrying break else: # Connection is down, try other servers if not self._container.timeout: self._logger.warning('Connection lost to %r, try next server', self.currentserver) else: self._logger.info('Rebalance to next server') self._last_zxid = last_zxid = conn.zookeeper_lastzxid last_conn_time = time() self.session_state = ZooKeeperSessionStateChanged.DISCONNECTED for m in self._container.waitForSend(ZooKeeperSessionStateChanged( ZooKeeperSessionStateChanged.DISCONNECTED, self, session_id )): yield m finally: conn.subroutine(conn.shutdown(True), False) self.current_connection = None finally: self._shutdown = True if self.session_state != ZooKeeperSessionStateChanged.EXPIRED and self.session_state != ZooKeeperSessionStateChanged.AUTHFAILED: self.session_state = ZooKeeperSessionStateChanged.EXPIRED self._container.scheduler.emergesend(ZooKeeperSessionStateChanged( ZooKeeperSessionStateChanged.EXPIRED, self, session_id ))
def _getconnection(self, container, host, path, https = False, forcecreate = False, cafile = None, key = None, certificate = None, timeout = None): if not host: raise ValueError matcher = WebClientRequestDoneEvent.createMatcher(host, path, https) while self.sameurllimit and (host, path, https) in self._requesting: self._pathwaiting.add((host, path, https)) yield (matcher,) # Lock the path if self.sameurllimit: self._requesting.add((host, path, https)) # connmap format: (free, free_ssl, workingcount) conns = self._connmap.setdefault(host, [[],[], 0]) conns[0] = [c for c in conns[0] if c.connected] conns[1] = [c for c in conns[1] if c.connected] myset = conns[1 if https else 0] if not forcecreate and myset: # There are free connections, reuse them conn = myset.pop() conn.setdaemon(False) container.retvalue = (conn, False) conns[2] += 1 return matcher = WebClientRequestDoneEvent.createMatcher(host) while self.samehostlimit and len(conns[0]) + len(conns[1]) + conns[2] >= self.samehostlimit: if myset: # Close a old connection conn = myset.pop() for m in conn.shutdown(): yield m else: # Wait for free connections self._hostwaiting.add(host) yield (matcher,) conns = self._connmap.setdefault(host, [[],[], 0]) myset = conns[1 if https else 0] if not forcecreate and myset: conn = myset.pop() conn.setdaemon(False) container.retvalue = (conn, False) conns[2] += 1 return # Create new connection conns[2] += 1 conn = Client(urlunsplit(('ssl' if https else 'tcp', host, '/', '', '')), self._protocol, container.scheduler, key, certificate, cafile) if timeout is not None: conn.connect_timeout = timeout conn.start() connected = self._protocol.statematcher(conn, HttpConnectionStateEvent.CLIENT_CONNECTED, False) notconnected = self._protocol.statematcher(conn, HttpConnectionStateEvent.CLIENT_NOTCONNECTED, False) yield (connected, notconnected) if container.matcher is notconnected: conns[2] -= 1 for m in conn.shutdown(True): yield m raise IOError('Failed to connect to %r' % (conn.rawurl,)) if https and cafile and self.verifyhost: try: # TODO: check with SSLContext hostcheck = re.sub(r':\d+$', '', host) if host == conn.socket.remoteaddr[0]: # IP Address is currently now allowed for m in conn.shutdown(True): yield m raise CertificateException('Cannot verify host with IP address') match_hostname(conn.socket.getpeercert(False), hostcheck) except: conns[2] -= 1 raise container.retvalue = (conn, True)