def main(self): self.apiroutine.subroutine(self.watcher(), False, daemon=True) up = ZooKeeperConnectionStateEvent.createMatcher( ZooKeeperConnectionStateEvent.UP, self.client) notconn = ZooKeeperConnectionStateEvent.createMatcher( ZooKeeperConnectionStateEvent.NOTCONNECTED, self.client) yield (up, notconn) if self.apiroutine.matcher is notconn: print('Not connected') return else: print('Connection is up: %r' % (self.client, )) # Handshake for m in self.protocol.handshake( self.client, zk.ConnectRequest( timeOut=int(self.sessiontimeout * 1000), passwd=b'\x00' * 16, # Why is it necessary... ), self.apiroutine, []): yield m for m in self.protocol.requests( self.client, [zk.create(b'/vlcptest', b'test'), zk.getdata(b'/vlcptest', True)], self.apiroutine): yield m pprint(dump(self.apiroutine.retvalue[0])) for m in self.apiroutine.waitWithTimeout(0.2): yield m for m in self.protocol.requests( self.client, [zk.delete(b'/vlcptest'), zk.getdata(b'/vlcptest', watch=True)], self.apiroutine): yield m pprint(dump(self.apiroutine.retvalue[0])) for m in self.protocol.requests(self.client, [ zk.multi(zk.multi_create(b'/vlcptest2', b'test'), zk.multi_create(b'/vlcptest2/subtest', 'test2')), zk.getchildren2(b'/vlcptest2', True) ], self.apiroutine): yield m pprint(dump(self.apiroutine.retvalue[0])) for m in self.protocol.requests(self.client, [ zk.multi(zk.multi_delete(b'/vlcptest2/subtest'), zk.multi_delete(b'/vlcptest2')), zk.getchildren2(b'/vlcptest2', True) ], self.apiroutine): yield m pprint(dump(self.apiroutine.retvalue[0]))
def main(self): self.apiroutine.subroutine(self.watcher(), False, daemon = True) up = ZooKeeperConnectionStateEvent.createMatcher(ZooKeeperConnectionStateEvent.UP, self.client) notconn = ZooKeeperConnectionStateEvent.createMatcher(ZooKeeperConnectionStateEvent.NOTCONNECTED, self.client) yield (up, notconn) if self.apiroutine.matcher is notconn: print('Not connected') return else: print('Connection is up: %r' % (self.client,)) # Handshake for m in self.protocol.handshake(self.client, zk.ConnectRequest( timeOut = int(self.sessiontimeout * 1000), passwd = b'\x00' * 16, # Why is it necessary... ), self.apiroutine, []): yield m for m in self.protocol.requests(self.client, [zk.create(b'/vlcptest', b'test'), zk.getdata(b'/vlcptest', True)], self.apiroutine): yield m pprint(dump(self.apiroutine.retvalue[0])) for m in self.apiroutine.waitWithTimeout(0.2): yield m for m in self.protocol.requests(self.client, [zk.delete(b'/vlcptest'), zk.getdata(b'/vlcptest', watch = True)], self.apiroutine): yield m pprint(dump(self.apiroutine.retvalue[0])) for m in self.protocol.requests(self.client, [zk.multi( zk.multi_create(b'/vlcptest2', b'test'), zk.multi_create(b'/vlcptest2/subtest', 'test2') ), zk.getchildren2(b'/vlcptest2', True)], self.apiroutine): yield m pprint(dump(self.apiroutine.retvalue[0])) for m in self.protocol.requests(self.client, [zk.multi( zk.multi_delete(b'/vlcptest2/subtest'), zk.multi_delete(b'/vlcptest2')), zk.getchildren2(b'/vlcptest2', True)], self.apiroutine): yield m pprint(dump(self.apiroutine.retvalue[0]))
def _connection_manage(self): try: failed = 0 self._last_zxid = last_zxid = 0 session_id = 0 passwd = b'\x00' * 16 last_conn_time = None while True: self.currentserver = self.serverlist[self.nextptr] np = self.nextptr + 1 if np >= len(self.serverlist): np = 0 self.nextptr = np conn = Client(self.currentserver, self.protocol, self._container.scheduler, self.key, self.certificate, self.ca_certs) self.current_connection = conn conn_up = ZooKeeperConnectionStateEvent.createMatcher(ZooKeeperConnectionStateEvent.UP, conn) conn_nc = ZooKeeperConnectionStateEvent.createMatcher(ZooKeeperConnectionStateEvent.NOTCONNECTED, conn) conn.start() try: yield (conn_up, conn_nc) if self._container.matcher is conn_nc: self._logger.warning('Connect to %r failed, try next server', self.currentserver) if failed > 5: # Wait for a small amount of time to prevent a busy loop # Socket may be rejected, it may fail very quick for m in self._container.waitWithTimeout(min((failed - 5) * 0.1, 1.0)): yield m failed += 1 continue try: # Handshake set_watches = [] if self.session_state == ZooKeeperSessionStateChanged.DISCONNECTED: for m in self._container.waitForSend(ZooKeeperRestoreWatches(self, self.session_id, True, restore_watches = (set(), set(), set()))): yield m yield (ZooKeeperRestoreWatches.createMatcher(self),) data_watches, exists_watches, child_watches = \ self._container.event.restore_watches if data_watches or exists_watches or child_watches: current_set_watches = zk.SetWatches(relativeZxid = last_zxid) current_length = 0 for d, e, c in izip_longest(data_watches, exists_watches, child_watches): if d is not None: current_set_watches.dataWatches.append(d) current_length += 4 + len(d) if e is not None: current_set_watches.existWatches.append(e) current_length += 4 + len(e) if c is not None: current_set_watches.childWatches.append(c) current_length += 4 + len(c) if current_length > _MAX_SETWATCHES_SIZE: # Split set_watches set_watches.append(current_set_watches) current_set_watches = zk.SetWatches(relativeZxid = last_zxid) if current_set_watches.dataWatches or current_set_watches.existWatches \ or current_set_watches.childWatches: set_watches.append(current_set_watches) auth_list = list(self.auth_set) with closing(self._container.executeWithTimeout(10, self.protocol.handshake(conn, zk.ConnectRequest(lastZxidSeen = last_zxid, timeOut = int(self.sessiontimeout * 1000.0), sessionId = session_id, passwd = passwd, readOnly = self.readonly), self._container, [zk.AuthPacket(scheme = a[0], auth = a[1]) for a in auth_list] + set_watches))) as g: for m in g: yield m if self._container.timeout: raise IOError except ZooKeeperSessionExpiredException: self._logger.warning('Session expired.') # Session expired self.session_state = ZooKeeperSessionStateChanged.EXPIRED for m in self._container.waitForSend(ZooKeeperSessionStateChanged( ZooKeeperSessionStateChanged.EXPIRED, self, session_id)): yield m if self.restart_session: failed = 0 last_zxid = 0 session_id = 0 passwd = b'\x00' * 16 last_conn_time = None continue else: break except Exception: self._logger.warning('Handshake failed to %r, try next server', self.currentserver) if failed > 5: # There is a bug ZOOKEEPER-1159 that ZooKeeper server does not respond # for session expiration, but directly close the connection. # This is a workaround: we store the time that we disconnected from the server, # if we have exceeded the session expiration time, we declare the session is expired if last_conn_time is not None and last_conn_time + self.sessiontimeout * 2 < time(): self._logger.warning('Session expired detected from client time.') # Session expired self.session_state = ZooKeeperSessionStateChanged.EXPIRED for m in self._container.waitForSend(ZooKeeperSessionStateChanged( ZooKeeperSessionStateChanged.EXPIRED, self, session_id)): yield m if self.restart_session: failed = 0 last_zxid = 0 session_id = 0 passwd = b'\x00' * 16 last_conn_time = None continue else: break else: # Wait for a small amount of time to prevent a busy loop for m in self._container.waitWithTimeout(min((failed - 5) * 0.1, 1.0)): yield m failed += 1 else: failed = 0 conn_resp, auth_resp = self._container.retvalue if conn_resp.timeOut <= 0: # Session expired # Currently should not happen because handshake() should raise an exception self._logger.warning('Session expired detected from handshake packet') self.session_state = ZooKeeperSessionStateChanged.EXPIRED for m in self._container.waitForSend(ZooKeeperSessionStateChanged( ZooKeeperSessionStateChanged.EXPIRED, self, session_id)): yield m if self.restart_session: failed = 0 last_zxid = 0 last_conn_time = None session_id = 0 passwd = b'\x00' * 16 continue else: break else: session_id = conn_resp.sessionId passwd = conn_resp.passwd # Authentication result check auth_failed = any(a.err == zk.ZOO_ERR_AUTHFAILED for a in auth_resp) if auth_failed: self._logger.warning('ZooKeeper authentication failed for following auth: %r', [a for a,r in zip(auth_list, auth_resp) if r.err == zk.ZOO_ERR_AUTHFAILED]) self.session_state = ZooKeeperSessionStateChanged.AUTHFAILED for m in self._container.waitForSend(ZooKeeperSessionStateChanged( ZooKeeperSessionStateChanged.AUTHFAILED, self, session_id )): yield m # Not retrying break else: self.session_readonly = getattr(conn_resp, 'readOnly', False) self.session_id = session_id if self.session_state == ZooKeeperSessionStateChanged.EXPIRED: for m in self._container.waitForSend(ZooKeeperSessionStateChanged( ZooKeeperSessionStateChanged.CREATED, self, session_id )): yield m else: for m in self._container.waitForSend(ZooKeeperSessionStateChanged( ZooKeeperSessionStateChanged.RECONNECTED, self, session_id )): yield m self.session_state = ZooKeeperSessionStateChanged.CREATED if conn.connected: conn_down = ZooKeeperConnectionStateEvent.createMatcher(ZooKeeperConnectionStateEvent.DOWN, conn, conn.connmark ) auth_failed = ZooKeeperResponseEvent.createMatcher(zk.AUTH_XID, conn, conn.connmark, _ismatch = lambda x: x.message.err == ZOO_ERR_AUTHFAILED) while True: rebalancetime = self.rebalancetime if rebalancetime is not None: rebalancetime += random() * 60 for m in self._container.waitWithTimeout(rebalancetime, conn_down, auth_failed): yield m if self._container.timeout: # Rebalance if conn.zookeeper_requests: # There are still requests not processed, wait longer for _ in range(0, 3): longer_time = random() * 10 for m in self._container.waitWithTimeout(longer_time, conn_down, auth_failed): yield m if not self._container.timeout: # Connection is down, or auth failed break if not conn.zookeeper_requests: break else: # There is still requests, skip for this time continue # Rebalance to a random server if self._container.timeout: self.nextptr = randrange(len(self.serverlist)) break if self._container.matcher is auth_failed: self._logger.warning('ZooKeeper authentication failed, shutdown the connection') self.session_state = ZooKeeperSessionStateChanged.AUTHFAILED for m in self._container.waitForSend(ZooKeeperSessionStateChanged( ZooKeeperSessionStateChanged.AUTHFAILED, self, session_id )): yield m # Not retrying break else: # Connection is down, try other servers if not self._container.timeout: self._logger.warning('Connection lost to %r, try next server', self.currentserver) else: self._logger.info('Rebalance to next server') self._last_zxid = last_zxid = conn.zookeeper_lastzxid last_conn_time = time() self.session_state = ZooKeeperSessionStateChanged.DISCONNECTED for m in self._container.waitForSend(ZooKeeperSessionStateChanged( ZooKeeperSessionStateChanged.DISCONNECTED, self, session_id )): yield m finally: conn.subroutine(conn.shutdown(True), False) self.current_connection = None finally: self._shutdown = True if self.session_state != ZooKeeperSessionStateChanged.EXPIRED and self.session_state != ZooKeeperSessionStateChanged.AUTHFAILED: self.session_state = ZooKeeperSessionStateChanged.EXPIRED self._container.scheduler.emergesend(ZooKeeperSessionStateChanged( ZooKeeperSessionStateChanged.EXPIRED, self, session_id ))
def _connection_manage(self): try: failed = 0 self._last_zxid = last_zxid = 0 session_id = 0 passwd = b'\x00' * 16 last_conn_time = None while True: self.currentserver = self.serverlist[self.nextptr] np = self.nextptr + 1 if np >= len(self.serverlist): np = 0 self.nextptr = np conn = Client(self.currentserver, self.protocol, self._container.scheduler, self.key, self.certificate, self.ca_certs) self.current_connection = conn conn_up = ZooKeeperConnectionStateEvent.createMatcher( ZooKeeperConnectionStateEvent.UP, conn) conn_nc = ZooKeeperConnectionStateEvent.createMatcher( ZooKeeperConnectionStateEvent.NOTCONNECTED, conn) conn.start() try: yield (conn_up, conn_nc) if self._container.matcher is conn_nc: self._logger.warning( 'Connect to %r failed, try next server', self.currentserver) if failed > 5: # Wait for a small amount of time to prevent a busy loop # Socket may be rejected, it may fail very quick for m in self._container.waitWithTimeout( min((failed - 5) * 0.1, 1.0)): yield m failed += 1 continue try: # Handshake set_watches = [] if self.session_state == ZooKeeperSessionStateChanged.DISCONNECTED: for m in self._container.waitForSend( ZooKeeperRestoreWatches( self, self.session_id, True, restore_watches=(set(), set(), set()))): yield m yield ( ZooKeeperRestoreWatches.createMatcher(self), ) data_watches, exists_watches, child_watches = \ self._container.event.restore_watches if data_watches or exists_watches or child_watches: current_set_watches = zk.SetWatches( relativeZxid=last_zxid) current_length = 0 for d, e, c in izip_longest( data_watches, exists_watches, child_watches): if d is not None: current_set_watches.dataWatches.append( d) current_length += 4 + len(d) if e is not None: current_set_watches.existWatches.append( e) current_length += 4 + len(e) if c is not None: current_set_watches.childWatches.append( c) current_length += 4 + len(c) if current_length > _MAX_SETWATCHES_SIZE: # Split set_watches set_watches.append(current_set_watches) current_set_watches = zk.SetWatches( relativeZxid=last_zxid) if current_set_watches.dataWatches or current_set_watches.existWatches \ or current_set_watches.childWatches: set_watches.append(current_set_watches) auth_list = list(self.auth_set) with closing( self._container.executeWithTimeout( 10, self.protocol.handshake( conn, zk.ConnectRequest( lastZxidSeen=last_zxid, timeOut=int(self.sessiontimeout * 1000.0), sessionId=session_id, passwd=passwd, readOnly=self.readonly), self._container, [ zk.AuthPacket(scheme=a[0], auth=a[1]) for a in auth_list ] + set_watches))) as g: for m in g: yield m if self._container.timeout: raise IOError except ZooKeeperSessionExpiredException: self._logger.warning('Session expired.') # Session expired self.session_state = ZooKeeperSessionStateChanged.EXPIRED for m in self._container.waitForSend( ZooKeeperSessionStateChanged( ZooKeeperSessionStateChanged.EXPIRED, self, session_id)): yield m if self.restart_session: failed = 0 last_zxid = 0 session_id = 0 passwd = b'\x00' * 16 last_conn_time = None continue else: break except Exception: self._logger.warning( 'Handshake failed to %r, try next server', self.currentserver) if failed > 5: # There is a bug ZOOKEEPER-1159 that ZooKeeper server does not respond # for session expiration, but directly close the connection. # This is a workaround: we store the time that we disconnected from the server, # if we have exceeded the session expiration time, we declare the session is expired if last_conn_time is not None and last_conn_time + self.sessiontimeout * 2 < time( ): self._logger.warning( 'Session expired detected from client time.' ) # Session expired self.session_state = ZooKeeperSessionStateChanged.EXPIRED for m in self._container.waitForSend( ZooKeeperSessionStateChanged( ZooKeeperSessionStateChanged. EXPIRED, self, session_id)): yield m if self.restart_session: failed = 0 last_zxid = 0 session_id = 0 passwd = b'\x00' * 16 last_conn_time = None continue else: break else: # Wait for a small amount of time to prevent a busy loop for m in self._container.waitWithTimeout( min((failed - 5) * 0.1, 1.0)): yield m failed += 1 else: failed = 0 conn_resp, auth_resp = self._container.retvalue if conn_resp.timeOut <= 0: # Session expired # Currently should not happen because handshake() should raise an exception self._logger.warning( 'Session expired detected from handshake packet' ) self.session_state = ZooKeeperSessionStateChanged.EXPIRED for m in self._container.waitForSend( ZooKeeperSessionStateChanged( ZooKeeperSessionStateChanged.EXPIRED, self, session_id)): yield m if self.restart_session: failed = 0 last_zxid = 0 last_conn_time = None session_id = 0 passwd = b'\x00' * 16 continue else: break else: session_id = conn_resp.sessionId passwd = conn_resp.passwd # Authentication result check auth_failed = any(a.err == zk.ZOO_ERR_AUTHFAILED for a in auth_resp) if auth_failed: self._logger.warning( 'ZooKeeper authentication failed for following auth: %r', [ a for a, r in zip(auth_list, auth_resp) if r.err == zk.ZOO_ERR_AUTHFAILED ]) self.session_state = ZooKeeperSessionStateChanged.AUTHFAILED for m in self._container.waitForSend( ZooKeeperSessionStateChanged( ZooKeeperSessionStateChanged. AUTHFAILED, self, session_id)): yield m # Not retrying break else: self.session_readonly = getattr( conn_resp, 'readOnly', False) self.session_id = session_id if self.session_state == ZooKeeperSessionStateChanged.EXPIRED: for m in self._container.waitForSend( ZooKeeperSessionStateChanged( ZooKeeperSessionStateChanged. CREATED, self, session_id)): yield m else: for m in self._container.waitForSend( ZooKeeperSessionStateChanged( ZooKeeperSessionStateChanged. RECONNECTED, self, session_id)): yield m self.session_state = ZooKeeperSessionStateChanged.CREATED if conn.connected: conn_down = ZooKeeperConnectionStateEvent.createMatcher( ZooKeeperConnectionStateEvent.DOWN, conn, conn.connmark) auth_failed = ZooKeeperResponseEvent.createMatcher( zk.AUTH_XID, conn, conn.connmark, _ismatch=lambda x: x.message.err == ZOO_ERR_AUTHFAILED) while True: rebalancetime = self.rebalancetime if rebalancetime is not None: rebalancetime += random() * 60 for m in self._container.waitWithTimeout( rebalancetime, conn_down, auth_failed): yield m if self._container.timeout: # Rebalance if conn.zookeeper_requests: # There are still requests not processed, wait longer for _ in range(0, 3): longer_time = random() * 10 for m in self._container.waitWithTimeout( longer_time, conn_down, auth_failed): yield m if not self._container.timeout: # Connection is down, or auth failed break if not conn.zookeeper_requests: break else: # There is still requests, skip for this time continue # Rebalance to a random server if self._container.timeout: self.nextptr = randrange( len(self.serverlist)) break if self._container.matcher is auth_failed: self._logger.warning( 'ZooKeeper authentication failed, shutdown the connection' ) self.session_state = ZooKeeperSessionStateChanged.AUTHFAILED for m in self._container.waitForSend( ZooKeeperSessionStateChanged( ZooKeeperSessionStateChanged. AUTHFAILED, self, session_id)): yield m # Not retrying break else: # Connection is down, try other servers if not self._container.timeout: self._logger.warning( 'Connection lost to %r, try next server', self.currentserver) else: self._logger.info( 'Rebalance to next server') self._last_zxid = last_zxid = conn.zookeeper_lastzxid last_conn_time = time() self.session_state = ZooKeeperSessionStateChanged.DISCONNECTED for m in self._container.waitForSend( ZooKeeperSessionStateChanged( ZooKeeperSessionStateChanged. DISCONNECTED, self, session_id)): yield m finally: conn.subroutine(conn.shutdown(True), False) self.current_connection = None finally: self._shutdown = True if self.session_state != ZooKeeperSessionStateChanged.EXPIRED and self.session_state != ZooKeeperSessionStateChanged.AUTHFAILED: self.session_state = ZooKeeperSessionStateChanged.EXPIRED self._container.scheduler.emergesend( ZooKeeperSessionStateChanged( ZooKeeperSessionStateChanged.EXPIRED, self, session_id))