class ZookeeperServiceRegistry(BaseServiceRegistry):
def __init__(self, hosts=DEFAULT_HOSTS, chroot=DEFAULT_CHROOT):
super(ZookeeperServiceRegistry, self).__init__()
self.chroot = chroot
self.client = KazooClient(
hosts=hosts,
handler=SequentialGeventHandler(),
)
self.client.add_listener(self.on_kazoo_state_change)
self.start_count = 0
@classmethod
def from_config(cls, config, **kwargs):
return cls(hosts=config.get('hosts', DEFAULT_HOSTS),
chroot=config.get('chroot', DEFAULT_CHROOT),
**kwargs)
def on_start(self, timeout=10):
self.start_count += 1
if self.start_count > 1:
return
started = self.client.start_async()
started.wait(timeout=timeout)
if not self.client.connected:
raise RuntimeError('could not connect to zookeeper')
logger.debug('connected to zookeeper (version=%s)',
'.'.join(map(str, self.client.server_version())))
def on_stop(self):
self.start_count -= 1
if self.start_count != 0:
return
self.client.stop()
def on_kazoo_state_change(self, state):
logger.info('kazoo connection state changed to %s', state)
def on_service_type_watch(self, service, event):
try:
if event.type == EventType.CHILD:
# FIXME: figure out proper retry strategy
self.client.retry(self.lookup, service.container, service)
except Exception:
logger.exception('error in service type watcher')
def on_service_watch(self, service, event):
try:
prefix, service_type, identity = event.path.rsplit('/', 2)
if event.type == EventType.DELETED:
service.remove(identity)
except Exception:
logger.exception('error in service watcher')
def _get_service_znode(self, service, service_type, identity):
path = self._get_zk_path(service_type, identity)
result = self.client.get_async(path,
watch=functools.partial(
self.on_service_watch, service))
value, znode = result.get()
items = six.iteritems(json.loads(value.decode('utf-8')))
return {str(k): str(v) for k, v in items}
def discover(self, container):
result = self.client.get_children_async(path='%s/services' %
self.chroot, )
return list(result.get())
def lookup(self, container, service, watch=True, timeout=1):
def child_watch(event):
print(event)
service_type = service.service_type
result = self.client.get_children_async(
path='%s/services/%s' % (self.chroot, service_type),
watch=functools.partial(self.on_service_type_watch, service),
)
try:
names = result.get(timeout=timeout)
except NoNodeError:
raise LookupFailure(None,
"failed to resolve %s" % service.service_type)
logger.info("lookup %s %r", service_type, names)
identities = set(service.identities())
for name in names:
kwargs = self._get_service_znode(service, service_type, name)
identity = kwargs.pop('identity')
service.update(identity, **kwargs)
try:
identities.remove(identity)
except KeyError:
pass
for identity in identities:
service.remove(identity)
return service
def _get_zk_path(self, service_type, identity):
return '%s/services/%s/%s' % (self.chroot, service_type, identity)
def register(self, container, service_type, timeout=1):
path = self._get_zk_path(service_type, container.identity)
value = json.dumps({
'endpoint': container.endpoint,
'identity': container.identity,
'log_endpoint': container.log_endpoint,
})
result = self.client.create_async(path,
value.encode('utf-8'),
ephemeral=True,
makepath=True)
# FIXME: result.set_exception(RegistrationFailure())
result.get(timeout=timeout)
def unregister(self, container, service_type, timeout=1):
path = self._get_zk_path(service_type, container.identity)
result = self.client.delete_async(path)
result.set_exception(RegistrationFailure())
result.get(timeout=timeout)
import sys
from kazoo.client import KazooClient
from kazoo.client import KazooState
import logging
logging.basicConfig()
zk = KazooClient(hosts='127.0.0.1:2181')
zk.start()
from kazoo.exceptions import ConnectionLossException
from kazoo.exceptions import NoAuthException
def my_callback(async_obj):
try:
children = async_obj.get()
print(children)
except (ConnectionLossException, NoAuthException):
sys.exit(1)
# Both these statements return immediately, the second sets a callback
# that will be run when get_children_async has its return value
async_obj = zk.get_children_async("/zk-demo")
async_obj.rawlink(my_callback)
class ZookeeperServiceRegistry(BaseServiceRegistry):
def __init__(self, hosts=DEFAULT_HOSTS, chroot=DEFAULT_CHROOT):
super(ZookeeperServiceRegistry, self).__init__()
self.chroot = chroot
self.client = KazooClient(
hosts=hosts,
handler=SequentialGeventHandler(),
)
self.client.add_listener(self.on_kazoo_state_change)
self.start_count = 0
@classmethod
def from_config(cls, config, **kwargs):
return cls(
hosts=config.get('hosts', DEFAULT_HOSTS),
chroot=config.get('chroot', DEFAULT_CHROOT),
**kwargs
)
def on_start(self, timeout=10):
self.start_count += 1
if self.start_count > 1:
return
started = self.client.start_async()
started.wait(timeout=timeout)
if not self.client.connected:
raise RuntimeError('could not connect to zookeeper')
logger.debug('connected to zookeeper (version=%s)', '.'.join(map(str, self.client.server_version())))
def on_stop(self):
self.start_count -= 1
if self.start_count != 0:
return
self.client.stop()
def on_kazoo_state_change(self, state):
logger.info('kazoo connection state changed to %s', state)
def on_service_type_watch(self, service, event):
try:
if event.type == EventType.CHILD:
# FIXME: figure out proper retry strategy
self.client.retry(self.lookup, service.container, service)
except Exception:
logger.exception('error in service type watcher')
def on_service_watch(self, service, event):
try:
prefix, service_type, identity = event.path.rsplit('/', 2)
if event.type == EventType.DELETED:
service.remove(identity)
except Exception:
logger.exception('error in service watcher')
def _get_service_znode(self, service, service_type, identity):
path = self._get_zk_path(service_type, identity)
result = self.client.get_async(
path, watch=functools.partial(self.on_service_watch, service))
value, znode = result.get()
items = six.iteritems(json.loads(value.decode('utf-8')))
return {str(k): str(v) for k, v in items}
def discover(self, container):
result = self.client.get_children_async(
path='%s/services' % self.chroot,
)
return list(result.get())
def lookup(self, container, service, watch=True, timeout=1):
def child_watch(event):
print(event)
service_type = service.service_type
result = self.client.get_children_async(
path='%s/services/%s' % (self.chroot, service_type),
watch=functools.partial(self.on_service_type_watch, service),
)
try:
names = result.get(timeout=timeout)
except NoNodeError:
raise LookupFailure(None, "failed to resolve %s" % service.service_type)
logger.info("lookup %s %r", service_type, names)
identities = set(service.identities())
for name in names:
kwargs = self._get_service_znode(service, service_type, name)
identity = kwargs.pop('identity')
service.update(identity, **kwargs)
try:
identities.remove(identity)
except KeyError:
pass
for identity in identities:
service.remove(identity)
return service
def _get_zk_path(self, service_type, identity):
return '%s/services/%s/%s' % (self.chroot, service_type, identity)
def register(self, container, service_type, timeout=1):
path = self._get_zk_path(service_type, container.identity)
value = json.dumps({
'endpoint': container.endpoint,
'identity': container.identity,
'log_endpoint': container.log_endpoint,
})
result = self.client.create_async(
path,
value.encode('utf-8'),
ephemeral=True, makepath=True)
# FIXME: result.set_exception(RegistrationFailure())
result.get(timeout=timeout)
def unregister(self, container, service_type, timeout=1):
path = self._get_zk_path(service_type, container.identity)
result = self.client.delete_async(path)
result.set_exception(RegistrationFailure())
result.get(timeout=timeout)
class Zookeeper:
def __init__(self, hosts):
self.zk = KazooClient(hosts=hosts,
handler=SequentialGeventHandler(),
logger=logger)
# returns immediately
event = self.zk.start_async()
# Wait for 30 seconds and see if we're connected
event.wait(timeout=30)
try:
if not self.zk.connected:
# Not connected, stop trying to connect
self.zk.stop()
except (ConnectionLossException, NoAuthException) as error:
raise error
except Exception as error:
raise error
@coroutine
def get_children(self, node):
try:
children = self.zk.get_children_async(node)
raise Return(children.get())
except Exception as error:
raise error
@coroutine
def get_node(self, node):
try:
data = self.zk.get_async(node)
raise Return(data.get())
except Exception as error:
raise error
@coroutine
def check_path_exist(self, path):
try:
result = self.zk.exists(path)
if result:
raise Return(True)
else:
raise Return(False)
except Exception as error:
raise error
@coroutine
def create_path(self, path):
try:
result = self.zk.ensure_path_async(path)
raise Return(result.get())
except Exception as error:
raise error
@coroutine
def create_node(self, path, value):
try:
result = self.zk.create_async(path=path,
value=value,
acl=None,
ephemeral=True)
raise Return(result.get())
except Exception as error:
raise error
@coroutine
def update_node(self, path, value, version=-1):
try:
result = self.zk.set_async(path, value, version)
raise Return(result.get())
except Exception as error:
raise error
@coroutine
def update_node(self, path, value, version=-1):
try:
result = self.zk.set_async(path, value, version)
raise Return(result.get())
except Exception as error:
raise error
@coroutine
def del_node(self, node):
try:
node_info = self.zk.delete_async(node)
raise Return(node_info.get())
except Exception as error:
raise error
def close(self):
self.zk.stop()
if MACHINE_STRING not in children:
create_service_async_job = zk.create_async(
path=ROOT_PATH + SERVICE_PATH + '/' + MACHINE_STRING,
value=MACHINE_AVAIL_SERVICE,
ephemeral=True)
create_service_async_job.rawlink(create_path_async_callback)
else:
print "PATH existed."
except (ConnectionLossException, NoAuthException):
logger.error('Can not connect.')
sys.exit(1)
if __name__ == '__main__':
zk.start()
zk.add_listener(my_listener)
get_children_async_obj = zk.get_children_async(ROOT_PATH + SERVICE_PATH)
get_children_async_obj.rawlink(get_service_path_children_callback)
while True:
input_data = raw_input()
if input_data != foredata:
set_value_async = zk.set_async(path=ROOT_PATH + SERVICE_PATH +
'/' + MACHINE_STRING,
value=input_data)
set_value_async.rawlink(
partial(set_service_child_value_callback, input_data))
if input_data == 'exit':
print 'Exit'
sys.exit(1)
time.sleep(5)
def handle_new_conn(con):
while True:
data = con.recv(1024)
if not data:
break
if data == 'services':
print 'get services'
con.send(str(G_SERVICE_DICT))
con.close()
if __name__ == '__main__':
zk.start()
zk.add_listener(my_listener)
get_children_async_obj = zk.get_children_async(ROOT_PATH)
get_children_async_obj.rawlink(get_root_children_async_callback)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
s.bind((SOCKET_HOST, SOCKET_PORT))
except socket.error, msg:
print 'Bind failed. Error Code : ' + str(msg[0]) + ' Message ' + msg[1]
sys.exit()
s.listen(10)
while True:
conn, addr = s.accept()
# conn.settimeout(30)
print 'Connected with ' + addr[0] + ':' + str(addr[1])
thread.start_new_thread(handle_new_conn, (conn, ))
timeout=1,
handler=SequentialGeventHandler())
event = zk.start_async()
event.wait(timeout=1) # wait()方法等待start_async()返回的事件对象
if not zk.connected: # 由于可能永远连接失败，因此判断连接状态，做异常情况处理
zk.stop()
raise Exception("Unable to connect")
def my_callback(async_obj):
try:
print '-------------------------'
children = async_obj.get()
do_something(children)
except (ConnectionLossException, NoAuthException):
sys.exit(1)
zk.create_async("/lws/test/1", b"test")
data = zk.get_async("/lws/test/1")
print data
async_obj = zk.get_children_async("/lws/test/1")
# print async_obj
async_obj.rawlink(my_callback)
# data = zk.exists_async("/lws/test/1")
# print data
if __name__ == '__main__':
# トランザクションの開始
tx = zk.transaction()
## 基本的な使い方を確認
# パスの生成
zk.ensure_path(root)
# znodeが未作成であれば作成
znode = root + '/sample_znode'
if zk.exists(znode) is None:
zk.create(znode, b'sample_data')
print_status(znode)
# データの更新
zk.set(znode, b'updated_data')
print_status(znode)
# 子ノードの追加
znode2 = root + '/sample_znode2'
if zk.exists(znode2) is None:
zk.create(znode2, b'sample_data2')
print_status(znode2)
## 非同期処理はこうやって使う
async_obj = zk.get_children_async(root)
async_obj.rawlink(doAsync)
# ノードの削除
zk.delete(root, recursive=True)
# コミット
results = tx.commit()
print('#####[Result]#####')
print(results)
import sys
from kazoo.client import KazooClient
from kazoo.client import KazooState
import logging
logging.basicConfig()
zk = KazooClient(hosts='127.0.0.1:2181')
zk.start()
from kazoo.exceptions import ConnectionLossException
from kazoo.exceptions import NoAuthException
def my_callback(async_obj):
try:
children = async_obj.get()
print(children)
except (ConnectionLossException, NoAuthException):
sys.exit(1)
# Both these statements return immediately, the second sets a callback
# that will be run when get_children_async has its return value
async_obj = zk.get_children_async("/zk-demo")
async_obj.rawlink(my_callback)
from kazoo.handlers.gevent import SequentialGeventHandler
logging.basicConfig()
zk = KazooClient(hosts='zoo1:2181', handler=SequentialGeventHandler())
event = zk.start_async()
event.wait(timeout=30)
if not zk.connected:
# Not connected, stop trying to connect
print("ummm")
zk.stop()
raise Exception("Unable to connect.")
def my_callback(async_obj):
try:
children = async_obj.get()
print("hi")
except (ConnectionLossException, NoAuthException):
sys.exit(1)
# Both these statements return immediately, the second sets a callback
# that will be run when get_children_async has its return value
zk.create_async("/master", b'mas')
zk.create_async("/master/node_1", b'c1')
print("gdgd")
async_obj = zk.get_children_async("/master")
async_obj.rawlink(my_callback)
zk.create_async("/master/node_2", b'c2')
