def test_async_getchildren(self):
self.ensureCreated("/zk-python-getchildrentest", flags=0)
self.ensureCreated("/zk-python-getchildrentest/child")
def gc_callback(handle, rc, children):
self.cv.acquire()
self.rc = rc
self.children = children
self.callback_flag = True
self.cv.notify()
self.cv.release()
self.cv.acquire()
self.callback_flag = False
zookeeper.aget_children(self.handle, "/zk-python-getchildrentest",
None, gc_callback)
self.cv.wait(15)
self.assertEqual(self.callback_flag, True, "aget_children timed out")
self.assertEqual(
self.rc, zookeeper.OK,
"Return code for aget_children was not OK - %s" %
zookeeper.zerror(self.rc))
self.assertEqual(
len(self.children), 1,
"Expected to find 1 child, got " + str(len(self.children)))
def async_get_children(self, path, watcher=None, callback=None):
"""Async call to get zookeeper node's children.
Args:
path: zookeeper node path
watcher: watcher method to be invoked upon node creation
or removal with Zookeeper.Event as its sole argument.
watcher will be invoked in the context of the underlying
zookeeper API threads.
callback: callback method to be invoked upon operation completion with
zookeeper api handle, return_code, and list of children nodes.
callback will be invoked in the context of the underlying
zookeeper API thread.
Returns:
Zookeeper.AsyncResult if callback is None, otherwise None.
"""
async_result = None
if callback is None:
async_result = self.AsyncResult()
def async_callback(handle, return_code, children):
if return_code == zookeeper.OK:
async_result.set(children)
else:
async_result.set_exception(
self.error_to_exception(return_code))
callback = async_callback
zookeeper.aget_children(self.handle, path,
self._watcher_proxy(watcher), callback)
return async_result
def async_get_children(self, path, watcher=None, callback=None):
"""Async call to get zookeeper node's children.
Args:
path: zookeeper node path
watcher: watcher method to be invoked upon node creation
or removal with Zookeeper.Event as its sole argument.
watcher will be invoked in the context of the underlying
zookeeper API threads.
callback: callback method to be invoked upon operation completion with
zookeeper api handle, return_code, and list of children nodes.
callback will be invoked in the context of the underlying
zookeeper API thread.
Returns:
Zookeeper.AsyncResult if callback is None, otherwise None.
"""
async_result = None
if callback is None:
async_result = self.AsyncResult()
def async_callback(handle, return_code, children):
if return_code == zookeeper.OK:
async_result.set(children)
else:
async_result.set_exception(self.error_to_exception(return_code))
callback = async_callback
zookeeper.aget_children(self.handle, path, self._watcher_proxy(watcher), callback)
return async_result
def async_get_children(self, path, watcher=None):
"""Async call to get zookeeper node's children.
Args:
path: zookeeper node path
watcher: watcher method to be invoked upon node creation
watcher will be invoked in the context of a newly
spawned greenlet.
or removal with Zookeeper.Event as its sole argument.
Returns:
Zookeeper.AsyncResult if callback is None, otherwise None.
"""
async_result = self._async_result()
def callback(handle, return_code, children):
if return_code == zookeeper.OK:
async_result.set(children)
else:
async_result.set_exception(self.error_to_exception(return_code))
watcher_callback, greenlet = self._spawn_watcher(watcher)
zookeeper.aget_children(self.handle, path, watcher_callback, callback)
return async_result
def test_async_getchildren_with_watcher(self):
self.ensureCreated("/zk-python-getchildrentest", flags=0)
self.ensureCreated("/zk-python-getchildrentest/child")
watched = []
def watcher(*args):
self.cv.acquire()
watched.append(args)
self.cv.notify()
self.cv.release()
def children_callback(*args):
self.cv.acquire()
self.cv.notify()
self.cv.release()
zookeeper.aget_children(
self.handle, "/zk-python-getchildrentest", watcher, children_callback)
self.cv.acquire()
self.cv.wait()
self.cv.release()
self.cv.acquire()
self.ensureCreated("/zk-python-getchildrentest/child2")
self.cv.wait(15)
self.assertTrue(watched)
def get_children_async(self, path, watcher=None):
async_result = self._new_async_result()
watcher_callback, watcher_greenlet = self._setup_watcher(watcher)
def callback(handle, code, children):
self._queue_result(async_result, code != zookeeper.OK,
err_to_exception(code) if code != zookeeper.OK
else children, watcher_greenlet)
zookeeper.aget_children(self._handle, path, watcher_callback, callback)
return async_result
def get_children_async(self, path, watch=None):
"""Asynchronously get a list of child nodes of a path
@param path: path of node to list
@param watch: optional watch callback to set for future changes to this path
@return: AsyncResult set with list of child node names on success
@rtype: AsyncResult
"""
async_result = self._sync.async_result()
callback = partial(_generic_callback, async_result)
watch_callback = self._wrap_watch_callback(watch) if watch else None
zookeeper.aget_children(self._handle, path, watch_callback, callback)
return async_result
def get_children_async(self, path, watch=None):
"""Asynchronously get a list of child nodes of a path
@param path: path of node to list
@param watch: optional watch callback to set for future changes to this path
@return: AsyncResult set with list of child node names on success
@rtype: AsyncResult
"""
async_result = self._sync.async_result()
callback = partial(_generic_callback, async_result)
watch_callback = self._wrap_watch_callback(watch) if watch else None
zookeeper.aget_children(self._handle, path, watch_callback, callback)
return async_result
def get_children(self, path, watcher=None):
"""
lists the children of a node synchronously.
PARAMETERS:
path: the name of the node. Expressed as a file name with slashes
separating ancestors of the node.
(subsequent parameters are optional)
watcher: if non-null, a watch will be set at the server to notify
the client if the node changes.
RETURNS:
A list of znode names
EXCEPTIONS:
NONODE the node does not exist.
NOAUTH the client does not have permission.
BADARGUMENTS - invalid input parameters
INVALIDSTATE - zhandle state is either SESSION_EXPIRED_STATE
or AUTH_FAILED_STATE
MARSHALLINGERROR - failed to marshall a request; possibly, out
of memory
"""
results = []
pc = utils.PipeCondition()
ok = zookeeper.aget_children(self._zhandle, path, watcher, functools.partial(generic_completion, pc, results))
assert ok == zookeeper.OK
pc.wait()
# unpack result as strings_completion
handle, rc, children = results
assert handle == self._zhandle
if rc == zookeeper.OK:
return children
self._raise_exception(rc)
def get_children_async(self, path, watcher=None):
async_result = self._new_async_result()
def callback(handle, code, children):
if code != zookeeper.OK:
exc = err_to_exception(code)
async_result.set_exception(exc)
else:
async_result.set(children)
watcher_callback, watcher_greenlet = self._setup_watcher(watcher)
#TODO cleanup the watcher greenlet on error
zookeeper.aget_children(self._handle, path, watcher_callback, callback)
return async_result
def node_info(self, path_parts):
self.node_data = None
self.node_data_flag = threading.Event()
self.queue_nodes = None
self.queue_nodes_flag = threading.Event()
self.server.log.debug('trying to get node: %s' % '/queues/' + '/'.join(path_parts))
if zookeeper.OK != zookeeper.aget(self.server.zk,
'/queues/' + '/'.join(path_parts),
None,
self.get_handler):
self.server.log.warning('Unable to get znode')
if zookeeper.OK != zookeeper.aget_children(self.server.zk,
'/queues/' + path_parts[0],
None,
self.get_children_handler):
self.server.log.warning('Unable to get znode children')
self.node_data_flag.wait()
self.queue_nodes_flag.wait()
queue_position = 1
for child in sorted(self.queue_nodes):
if path_parts[-1] in child:
break
queue_position += 1
self.send_response(200)
self.send_header("Content-type", "text/xml")
self.end_headers()
self.wfile.write("<message>\n\t<name>%s</name>\n\t<queue>%s</queue>\n\t<data>%s</data>\n\t<position>%d</position>\n\t<queue_size>%d</queue_size>\n</message>" % (path_parts[1], path_parts[0], self.node_data, queue_position, len(self.queue_nodes)))
def aget_children(self, path, watcher=None, handler=None):
"""A simple wrapper for zookeeper async get_children function.
This function wraps the zookeeper aget_children call which allows the caller
to register a function to handle the child node list once it is received as well
as a function that will be called once any child nodes are added/removed. If
neither is given then this function will do nothing.
Args:
path: The znode to watch.
watcher: Called when child nodes are added or removed. the basic
footprint of this function is:
func(zh, path)
zh will be this object, and path will be the znode path.
handler: Called when the child nodes has been fetched from zookeper. The basic
footprint of this function is:
func(zh, rc, children, path)
zh will be this object and path will be the znode path.
rc is the return code from zookeeper.
children is the list of child nodes after the create/delete.
Returns:
Nothing.
"""
register = False
get = False
self._children_lock.acquire()
if watcher:
register = path not in self._children_watches
self._children_watches.setdefault(path, []).append(watcher)
if handler:
get = path not in self._children_handlers
self._children_handlers.setdefault(path, []).append(handler)
self._children_lock.release()
if register or get:
if register:
w = self._children_watcher
else:
w = None
# We use a lambda here so we can make sure that the path gets appended
# to the args. This allows us to multiplex the call.
h = (lambda zh, rc, data: self._children_handler(zh, rc, data, path))
# FIXME(error handling)
logging.debug('Performing a get_children against %s', path)
zookeeper.aget_children(self._zookeeper, path, w, h)
def aget_children(self, path, watcher=None, handler=None):
"""A simple wrapper for zookeeper async get_children function.
This function wraps the zookeeper aget_children call which allows the caller
to register a function to handle the child node list once it is received as well
as a function that will be called once any child nodes are added/removed. If
neither is given then this function will do nothing.
Args:
path: The znode to watch.
watcher: Called when child nodes are added or removed. the basic
footprint of this function is:
func(zh, path)
zh will be this object, and path will be the znode path.
handler: Called when the child nodes has been fetched from zookeper. The basic
footprint of this function is:
func(zh, rc, children, path)
zh will be this object and path will be the znode path.
rc is the return code from zookeeper.
children is the list of child nodes after the create/delete.
Returns:
Nothing.
"""
register = False
get = False
self._children_lock.acquire()
if watcher:
register = path not in self._children_watches
self._children_watches.setdefault(path, []).append(watcher)
if handler:
get = path not in self._children_handlers
self._children_handlers.setdefault(path, []).append(handler)
self._children_lock.release()
if register or get:
if register:
w = self._children_watcher
else:
w = None
# We use a lambda here so we can make sure that the path gets appended
# to the args. This allows us to multiplex the call.
h = (lambda zh, rc, data: self._children_handler(zh, rc, data, path))
# FIXME(error handling)
logging.debug('Performing a get_children against %s', path)
zookeeper.aget_children(self._zookeeper, path, w, h)
def test_async_getchildren(self):
self.ensureCreated("/zk-python-getchildrentest", flags=0)
self.ensureCreated("/zk-python-getchildrentest/child")
def gc_callback(handle, rc, children):
self.cv.acquire()
self.rc = rc
self.children = children
self.callback_flag = True
self.cv.notify()
self.cv.release()
self.cv.acquire()
self.callback_flag = False
zookeeper.aget_children(self.handle, "/zk-python-getchildrentest", None, gc_callback)
self.cv.wait(15)
self.assertEqual(self.callback_flag, True, "aget_children timed out")
self.assertEqual(self.rc, zookeeper.OK, "Return code for aget_children was not OK - %s" % zookeeper.zerror(self.rc))
self.assertEqual(len(self.children), 1, "Expected to find 1 child, got " + str(len(self.children)))
def _get_children(self, path, watcher):
d = defer.Deferred()
if self._check_connected(d):
return d
def _cb_get_children(result_code, children):
if self._check_result(result_code, d, path=path):
return
d.callback(children)
callback = self._zk_thread_callback(_cb_get_children)
watcher = self._wrap_watcher(watcher, "child", path)
result = zookeeper.aget_children(self.handle, path, watcher, callback)
self._check_result(result, d, path=path)
return d
def queue_info(self, queue):
self.queue_nodes = None
self.queue_nodes_flag = threading.Event()
if zookeeper.OK != zookeeper.aget_children(self.server.zk,
'/queues/' + queue,
None,
self.get_children_handler):
self.server.log.warning('Unable to get znode children')
self.queue_nodes_flag.wait()
self.send_response(200)
self.send_header("Content-type", "text/xml")
self.end_headers()
self.wfile.write("<queue>\n\t<name>%s</name>\n\t<queue_size>%d</queue_size>\n</queue>" % (queue, len(self.queue_nodes)))
def test_dispatch_types(self):
"""
Test all the various dispatch mechanisms internal to the module.
"""
def dispatch_callback(*args, **kwargs):
self.callback_flag = True
self.ensureCreated("/zk-python-dispatch-test")
self.callback_harness(
lambda: zookeeper.adelete(
self.handle, "/zk-python-dispatch-test", -1,
self.create_callback(dispatch_callback)), lambda: self.
assertEqual(True, self.callback_flag, "Void dispatch not fired"))
self.ensureCreated("/zk-python-dispatch-test")
self.callback_harness(
lambda: zookeeper.aexists(
self.handle, "/zk-python-dispatch-test", None,
self.create_callback(dispatch_callback)), lambda: self.
assertEqual(True, self.callback_flag, "Stat dispatch not fired"))
self.callback_harness(
lambda: zookeeper.aget(
self.handle, "/zk-python-dispatch-test", None,
self.create_callback(dispatch_callback)), lambda: self.
assertEqual(True, self.callback_flag, "Data dispatch not fired"))
self.callback_harness(
lambda: zookeeper.aget_children(
self.handle, "/", None, self.
create_callback(dispatch_callback)), lambda: self.assertEqual(
True, self.callback_flag, "Strings dispatch not fired"))
self.callback_harness(
lambda: getattr(zookeeper, 'async')
(self.handle, "/", self.create_callback(dispatch_callback)),
lambda: self.assertEqual(True, self.callback_flag,
"String dispatch not fired"))
self.callback_harness(
lambda: zookeeper.aget_acl(
self.handle, "/", self.create_callback(dispatch_callback)),
lambda: self.assertEqual(True, self.callback_flag,
"ACL dispatch not fired"))
def run(self):
self.locate()
while True:
try:
self.lock.acquire()
if self.safeMode == True:
time.sleep(timeout)
zookeeper.close(self.zh)
conn_cv.acquire()
self.zh = zookeeper.init(self.zkservers, self.conn_callback, session_time)
conn_cv.wait()
conn_cv.release()
self.locate()
if self.safeMode == False:
if zookeeper.OK != zookeeper.aget_children(self.zh, self.actionNode, self.queue_watcher, None):
logger.error('Fail to monitor action queue for: '+self.actionNode+', activate safe mode.')
self.safeMode = True
except Exception, err:
logger.exception(err)
def run(self):
self.locate()
while True:
try:
self.lock.acquire()
if self.safeMode == True:
time.sleep(timeout)
zookeeper.close(self.zh)
conn_cv.acquire()
self.zh = zookeeper.init(self.zkservers, self.conn_callback, session_time)
conn_cv.wait()
conn_cv.release()
self.locate()
if self.safeMode == False:
if zookeeper.OK != zookeeper.aget_children(self.zh, self.actionNode, self.queue_watcher, None):
logger.error('Fail to monitor action queue for: '+self.actionNode+', activate safe mode.')
self.safeMode = True
except Exception, err:
logger.exception(err)
def get_children(self, path, watcher=None):
"""
lists the children of a node synchronously.
PARAMETERS:
path: the name of the node. Expressed as a file name with slashes
separating ancestors of the node.
(subsequent parameters are optional)
watcher: if non-null, a watch will be set at the server to notify
the client if the node changes.
RETURNS:
A list of znode names
EXCEPTIONS:
NONODE the node does not exist.
NOAUTH the client does not have permission.
BADARGUMENTS - invalid input parameters
INVALIDSTATE - zhandle state is either SESSION_EXPIRED_STATE
or AUTH_FAILED_STATE
MARSHALLINGERROR - failed to marshall a request; possibly, out
of memory
"""
results = []
pc = utils.PipeCondition()
ok = zookeeper.aget_children(
self._zhandle, path, watcher,
functools.partial(generic_completion, pc, results))
assert ok == zookeeper.OK
pc.wait()
#unpack result as strings_completion
handle, rc, children = results
assert handle == self._zhandle
if rc == zookeeper.OK:
return children
self._raise_exception(rc)
def aget(self): return zookeeper.aget_children(self.zh, self.actionNode, self.queue_watcher, self.queue_callback)
def _aget_children(self, path):
self._try_zoo(
lambda: self._use_socket(
lambda z: zookeeper.aget_children(z, path, self._events,
self._ls_cb(path))))
def aget(self): return zookeeper.aget_children(self.zh, self.actionNode, self.queue_watcher, self.queue_callback)
