class ZookeeperServiceRegistry(BaseServiceRegistry): def __init__(self, hosts=DEFAULT_HOSTS, chroot=DEFAULT_CHROOT): super(ZookeeperServiceRegistry, self).__init__() self.chroot = chroot self.client = KazooClient( hosts=hosts, handler=SequentialGeventHandler(), ) self.client.add_listener(self.on_kazoo_state_change) self.start_count = 0 @classmethod def from_config(cls, config, **kwargs): return cls(hosts=config.get('hosts', DEFAULT_HOSTS), chroot=config.get('chroot', DEFAULT_CHROOT), **kwargs) def on_start(self, timeout=10): self.start_count += 1 if self.start_count > 1: return started = self.client.start_async() started.wait(timeout=timeout) if not self.client.connected: raise RuntimeError('could not connect to zookeeper') logger.debug('connected to zookeeper (version=%s)', '.'.join(map(str, self.client.server_version()))) def on_stop(self): self.start_count -= 1 if self.start_count != 0: return self.client.stop() def on_kazoo_state_change(self, state): logger.info('kazoo connection state changed to %s', state) def on_service_type_watch(self, service, event): try: if event.type == EventType.CHILD: # FIXME: figure out proper retry strategy self.client.retry(self.lookup, service.container, service) except Exception: logger.exception('error in service type watcher') def on_service_watch(self, service, event): try: prefix, service_type, identity = event.path.rsplit('/', 2) if event.type == EventType.DELETED: service.remove(identity) except Exception: logger.exception('error in service watcher') def _get_service_znode(self, service, service_type, identity): path = self._get_zk_path(service_type, identity) result = self.client.get_async(path, watch=functools.partial( self.on_service_watch, service)) value, znode = result.get() items = six.iteritems(json.loads(value.decode('utf-8'))) return {str(k): str(v) for k, v in items} def discover(self, container): result = self.client.get_children_async(path='%s/services' % self.chroot, ) return list(result.get()) def lookup(self, container, service, watch=True, timeout=1): def child_watch(event): print(event) service_type = service.service_type result = self.client.get_children_async( path='%s/services/%s' % (self.chroot, service_type), watch=functools.partial(self.on_service_type_watch, service), ) try: names = result.get(timeout=timeout) except NoNodeError: raise LookupFailure(None, "failed to resolve %s" % service.service_type) logger.info("lookup %s %r", service_type, names) identities = set(service.identities()) for name in names: kwargs = self._get_service_znode(service, service_type, name) identity = kwargs.pop('identity') service.update(identity, **kwargs) try: identities.remove(identity) except KeyError: pass for identity in identities: service.remove(identity) return service def _get_zk_path(self, service_type, identity): return '%s/services/%s/%s' % (self.chroot, service_type, identity) def register(self, container, service_type, timeout=1): path = self._get_zk_path(service_type, container.identity) value = json.dumps({ 'endpoint': container.endpoint, 'identity': container.identity, 'log_endpoint': container.log_endpoint, }) result = self.client.create_async(path, value.encode('utf-8'), ephemeral=True, makepath=True) # FIXME: result.set_exception(RegistrationFailure()) result.get(timeout=timeout) def unregister(self, container, service_type, timeout=1): path = self._get_zk_path(service_type, container.identity) result = self.client.delete_async(path) result.set_exception(RegistrationFailure()) result.get(timeout=timeout)
import sys from kazoo.client import KazooClient from kazoo.client import KazooState import logging logging.basicConfig() zk = KazooClient(hosts='127.0.0.1:2181') zk.start() from kazoo.exceptions import ConnectionLossException from kazoo.exceptions import NoAuthException def my_callback(async_obj): try: children = async_obj.get() print(children) except (ConnectionLossException, NoAuthException): sys.exit(1) # Both these statements return immediately, the second sets a callback # that will be run when get_children_async has its return value async_obj = zk.get_children_async("/zk-demo") async_obj.rawlink(my_callback)
class ZookeeperServiceRegistry(BaseServiceRegistry): def __init__(self, hosts=DEFAULT_HOSTS, chroot=DEFAULT_CHROOT): super(ZookeeperServiceRegistry, self).__init__() self.chroot = chroot self.client = KazooClient( hosts=hosts, handler=SequentialGeventHandler(), ) self.client.add_listener(self.on_kazoo_state_change) self.start_count = 0 @classmethod def from_config(cls, config, **kwargs): return cls( hosts=config.get('hosts', DEFAULT_HOSTS), chroot=config.get('chroot', DEFAULT_CHROOT), **kwargs ) def on_start(self, timeout=10): self.start_count += 1 if self.start_count > 1: return started = self.client.start_async() started.wait(timeout=timeout) if not self.client.connected: raise RuntimeError('could not connect to zookeeper') logger.debug('connected to zookeeper (version=%s)', '.'.join(map(str, self.client.server_version()))) def on_stop(self): self.start_count -= 1 if self.start_count != 0: return self.client.stop() def on_kazoo_state_change(self, state): logger.info('kazoo connection state changed to %s', state) def on_service_type_watch(self, service, event): try: if event.type == EventType.CHILD: # FIXME: figure out proper retry strategy self.client.retry(self.lookup, service.container, service) except Exception: logger.exception('error in service type watcher') def on_service_watch(self, service, event): try: prefix, service_type, identity = event.path.rsplit('/', 2) if event.type == EventType.DELETED: service.remove(identity) except Exception: logger.exception('error in service watcher') def _get_service_znode(self, service, service_type, identity): path = self._get_zk_path(service_type, identity) result = self.client.get_async( path, watch=functools.partial(self.on_service_watch, service)) value, znode = result.get() items = six.iteritems(json.loads(value.decode('utf-8'))) return {str(k): str(v) for k, v in items} def discover(self, container): result = self.client.get_children_async( path='%s/services' % self.chroot, ) return list(result.get()) def lookup(self, container, service, watch=True, timeout=1): def child_watch(event): print(event) service_type = service.service_type result = self.client.get_children_async( path='%s/services/%s' % (self.chroot, service_type), watch=functools.partial(self.on_service_type_watch, service), ) try: names = result.get(timeout=timeout) except NoNodeError: raise LookupFailure(None, "failed to resolve %s" % service.service_type) logger.info("lookup %s %r", service_type, names) identities = set(service.identities()) for name in names: kwargs = self._get_service_znode(service, service_type, name) identity = kwargs.pop('identity') service.update(identity, **kwargs) try: identities.remove(identity) except KeyError: pass for identity in identities: service.remove(identity) return service def _get_zk_path(self, service_type, identity): return '%s/services/%s/%s' % (self.chroot, service_type, identity) def register(self, container, service_type, timeout=1): path = self._get_zk_path(service_type, container.identity) value = json.dumps({ 'endpoint': container.endpoint, 'identity': container.identity, 'log_endpoint': container.log_endpoint, }) result = self.client.create_async( path, value.encode('utf-8'), ephemeral=True, makepath=True) # FIXME: result.set_exception(RegistrationFailure()) result.get(timeout=timeout) def unregister(self, container, service_type, timeout=1): path = self._get_zk_path(service_type, container.identity) result = self.client.delete_async(path) result.set_exception(RegistrationFailure()) result.get(timeout=timeout)
class Zookeeper: def __init__(self, hosts): self.zk = KazooClient(hosts=hosts, handler=SequentialGeventHandler(), logger=logger) # returns immediately event = self.zk.start_async() # Wait for 30 seconds and see if we're connected event.wait(timeout=30) try: if not self.zk.connected: # Not connected, stop trying to connect self.zk.stop() except (ConnectionLossException, NoAuthException) as error: raise error except Exception as error: raise error @coroutine def get_children(self, node): try: children = self.zk.get_children_async(node) raise Return(children.get()) except Exception as error: raise error @coroutine def get_node(self, node): try: data = self.zk.get_async(node) raise Return(data.get()) except Exception as error: raise error @coroutine def check_path_exist(self, path): try: result = self.zk.exists(path) if result: raise Return(True) else: raise Return(False) except Exception as error: raise error @coroutine def create_path(self, path): try: result = self.zk.ensure_path_async(path) raise Return(result.get()) except Exception as error: raise error @coroutine def create_node(self, path, value): try: result = self.zk.create_async(path=path, value=value, acl=None, ephemeral=True) raise Return(result.get()) except Exception as error: raise error @coroutine def update_node(self, path, value, version=-1): try: result = self.zk.set_async(path, value, version) raise Return(result.get()) except Exception as error: raise error @coroutine def update_node(self, path, value, version=-1): try: result = self.zk.set_async(path, value, version) raise Return(result.get()) except Exception as error: raise error @coroutine def del_node(self, node): try: node_info = self.zk.delete_async(node) raise Return(node_info.get()) except Exception as error: raise error def close(self): self.zk.stop()
if MACHINE_STRING not in children: create_service_async_job = zk.create_async( path=ROOT_PATH + SERVICE_PATH + '/' + MACHINE_STRING, value=MACHINE_AVAIL_SERVICE, ephemeral=True) create_service_async_job.rawlink(create_path_async_callback) else: print "PATH existed." except (ConnectionLossException, NoAuthException): logger.error('Can not connect.') sys.exit(1) if __name__ == '__main__': zk.start() zk.add_listener(my_listener) get_children_async_obj = zk.get_children_async(ROOT_PATH + SERVICE_PATH) get_children_async_obj.rawlink(get_service_path_children_callback) while True: input_data = raw_input() if input_data != foredata: set_value_async = zk.set_async(path=ROOT_PATH + SERVICE_PATH + '/' + MACHINE_STRING, value=input_data) set_value_async.rawlink( partial(set_service_child_value_callback, input_data)) if input_data == 'exit': print 'Exit' sys.exit(1) time.sleep(5)
def handle_new_conn(con): while True: data = con.recv(1024) if not data: break if data == 'services': print 'get services' con.send(str(G_SERVICE_DICT)) con.close() if __name__ == '__main__': zk.start() zk.add_listener(my_listener) get_children_async_obj = zk.get_children_async(ROOT_PATH) get_children_async_obj.rawlink(get_root_children_async_callback) s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: s.bind((SOCKET_HOST, SOCKET_PORT)) except socket.error, msg: print 'Bind failed. Error Code : ' + str(msg[0]) + ' Message ' + msg[1] sys.exit() s.listen(10) while True: conn, addr = s.accept() # conn.settimeout(30) print 'Connected with ' + addr[0] + ':' + str(addr[1]) thread.start_new_thread(handle_new_conn, (conn, ))
timeout=1, handler=SequentialGeventHandler()) event = zk.start_async() event.wait(timeout=1) # wait()方法等待start_async()返回的事件对象 if not zk.connected: # 由于可能永远连接失败,因此判断连接状态,做异常情况处理 zk.stop() raise Exception("Unable to connect") def my_callback(async_obj): try: print '-------------------------' children = async_obj.get() do_something(children) except (ConnectionLossException, NoAuthException): sys.exit(1) zk.create_async("/lws/test/1", b"test") data = zk.get_async("/lws/test/1") print data async_obj = zk.get_children_async("/lws/test/1") # print async_obj async_obj.rawlink(my_callback) # data = zk.exists_async("/lws/test/1") # print data
if __name__ == '__main__': # トランザクションの開始 tx = zk.transaction() ## 基本的な使い方を確認 # パスの生成 zk.ensure_path(root) # znodeが未作成であれば作成 znode = root + '/sample_znode' if zk.exists(znode) is None: zk.create(znode, b'sample_data') print_status(znode) # データの更新 zk.set(znode, b'updated_data') print_status(znode) # 子ノードの追加 znode2 = root + '/sample_znode2' if zk.exists(znode2) is None: zk.create(znode2, b'sample_data2') print_status(znode2) ## 非同期処理はこうやって使う async_obj = zk.get_children_async(root) async_obj.rawlink(doAsync) # ノードの削除 zk.delete(root, recursive=True) # コミット results = tx.commit() print('#####[Result]#####') print(results)
from kazoo.handlers.gevent import SequentialGeventHandler logging.basicConfig() zk = KazooClient(hosts='zoo1:2181', handler=SequentialGeventHandler()) event = zk.start_async() event.wait(timeout=30) if not zk.connected: # Not connected, stop trying to connect print("ummm") zk.stop() raise Exception("Unable to connect.") def my_callback(async_obj): try: children = async_obj.get() print("hi") except (ConnectionLossException, NoAuthException): sys.exit(1) # Both these statements return immediately, the second sets a callback # that will be run when get_children_async has its return value zk.create_async("/master", b'mas') zk.create_async("/master/node_1", b'c1') print("gdgd") async_obj = zk.get_children_async("/master") async_obj.rawlink(my_callback) zk.create_async("/master/node_2", b'c2')