def test_stat_deleted_node(self): """ Test for a bug that surfaced when trying to build a stat object from a non-existant node. """ self.ensureDeleted("/zk-python-test-deleteme") self.assertRaises(zookeeper.NoNodeException, zookeeper.get, self.handle, "/zk-python-test-deleteme") self.cv = threading.Condition() def callback(handle, rc, value, stat): self.cv.acquire() self.stat = stat self.rc = rc self.value = value self.callback_flag = True self.cv.notify() self.cv.release() self.cv.acquire() zookeeper.aget(self.handle, "/zk-python-test-deleteme", None, callback) self.cv.wait(15) self.assertEqual(self.callback_flag, True, "aget timed out!") self.assertEqual(self.stat, None, "Stat should be none!") self.assertEqual(self.value, None, "Value should be none!")
def async_get_data(self, path, watcher=None, callback=None): """Async call to get zookeeper node's data. Args: path: zookeeper node path watcher: watcher method to be invoked upon node creation or removal with Zookeeper.Event as its sole argument. watcher will be invoked in the context of the underlying zookeeper API threads. callback: callback method to be invoked upon operation completion with zookeeper api handle, return_code, data, and stat. callback will be invoked in the context of the underlying zookeeper API thread. Returns: Zookeeper.AsyncResult if callback is None, otherwise None. """ async_result = None if callback is None: async_result = self.AsyncResult() def async_callback(handle, return_code, data, stat): if return_code == zookeeper.OK: async_result.set((data, stat)) else: async_result.set_exception(self.error_to_exception(return_code)) callback = async_callback zookeeper.aget(self.handle, path, self._watcher_proxy(watcher), callback) return async_result
def async_get_data(self, path, watcher=None): """Async call to get zookeeper node's data. Args: path: zookeeper node path watcher: watcher method to be invoked upon node creation or removal with Zookeeper.Event as its sole argument. watcher will be invoked in the context of a newly spawned greenlet. Returns: Zookeeper.AsyncResult if callback is None, otherwise None. """ async_result = self._async_result() def callback(handle, return_code, data, stat): if return_code == zookeeper.OK: async_result.set((data, stat)) else: async_result.set_exception(self.error_to_exception(return_code)) watcher_callback, greenlet = self._spawn_watcher(watcher) zookeeper.aget(self.handle, path, watcher_callback, callback) return async_result
def async_get_data(self, path, watcher=None, callback=None): """Async call to get zookeeper node's data. Args: path: zookeeper node path watcher: watcher method to be invoked upon node creation or removal with Zookeeper.Event as its sole argument. watcher will be invoked in the context of the underlying zookeeper API threads. callback: callback method to be invoked upon operation completion with zookeeper api handle, return_code, data, and stat. callback will be invoked in the context of the underlying zookeeper API thread. Returns: Zookeeper.AsyncResult if callback is None, otherwise None. """ async_result = None if callback is None: async_result = self.AsyncResult() def async_callback(handle, return_code, data, stat): if return_code == zookeeper.OK: async_result.set((data, stat)) else: async_result.set_exception( self.error_to_exception(return_code)) callback = async_callback zookeeper.aget(self.handle, path, self._watcher_proxy(watcher), callback) return async_result
def _global_watch(self, zh, event, state, path): """Called when the connection to zookeeper has a state change.""" if state == zookeeper.EXPIRED_SESSION_STATE: self._connect() if state == zookeeper.CONNECTED_STATE: self._clientid = zookeeper.client_id(self._zookeeper) # Catch up all gets requested before we were able to connect. while self._pending_gets: path, w, h = self._pending_gets.pop() zookeeper.aget(self._zookeeper, path, w, h)
def get_async(self, path, watcher=None): async_result = self._new_async_result() watcher_callback, watcher_greenlet = self._setup_watcher(watcher) def callback(handle, code, value, stat): self._queue_result(async_result, code != zookeeper.OK, err_to_exception(code) if code != zookeeper.OK else (value, stat), watcher_greenlet) zookeeper.aget(self._handle, path, watcher_callback, callback) return async_result
def deadlock(): cv = threading.Condition() def callback(*args): cv.acquire() cv.notifyAll() cv.release() time.sleep(1) cv.acquire() zookeeper.aget(handle, "/", None, callback) cv.wait() zookeeper.close(handle)
def get_async(self, path, watch=None): """Asynchronously get the value of a node @param path: path of node @param watch: optional watch callback to set for future changes to this path @return AsyncResult set with tuple (value, stat) of node on success @rtype AsyncResult """ async_result = self._sync.async_result() callback = partial(_generic_callback, async_result) watch_callback = self._wrap_watch_callback(watch) if watch else None zookeeper.aget(self._handle, path, watch_callback, callback) return async_result
def aget(self, node, callback, rccallback=None): # callback is invoked when the watcher triggers # rccallback is invoked when the result of attaching # becomes available (OK, NONODE and so on) assert callable(callback), "callback must be callable" if rccallback is not None: assert callable(rccallback), "rccallback must be callable" def watcher(self, zh, event, state, path): self.logger.info("Node state has been changed") if event == zookeeper.CHANGED_EVENT: self.logger.debug("Node %s has been modified", path) elif event == zookeeper.CREATED_EVENT: self.logger.debug("Node %s has been created", path) elif event == zookeeper.DELETED_EVENT: self.logger.warning("Node %s has been deleted", path) if state == zookeeper.EXPIRED_SESSION_STATE: self.logger.error("Session has expired") callback(event, state, path) def rc_handler(self, zh, rc, data, stat): if zookeeper.OK == rc: self.logger.debug("Callback has been attached succesfully") elif zookeeper.NONODE == rc: self.logger.warning("Watched node doesn't exists") if rccallback is not None: rccallback(rc) res = zookeeper.aget(self.zkhandle, node, partial(watcher, self), partial(rc_handler, self)) return res == zookeeper.OK
def node_info(self, path_parts): self.node_data = None self.node_data_flag = threading.Event() self.queue_nodes = None self.queue_nodes_flag = threading.Event() self.server.log.debug('trying to get node: %s' % '/queues/' + '/'.join(path_parts)) if zookeeper.OK != zookeeper.aget(self.server.zk, '/queues/' + '/'.join(path_parts), None, self.get_handler): self.server.log.warning('Unable to get znode') if zookeeper.OK != zookeeper.aget_children(self.server.zk, '/queues/' + path_parts[0], None, self.get_children_handler): self.server.log.warning('Unable to get znode children') self.node_data_flag.wait() self.queue_nodes_flag.wait() queue_position = 1 for child in sorted(self.queue_nodes): if path_parts[-1] in child: break queue_position += 1 self.send_response(200) self.send_header("Content-type", "text/xml") self.end_headers() self.wfile.write("<message>\n\t<name>%s</name>\n\t<queue>%s</queue>\n\t<data>%s</data>\n\t<position>%d</position>\n\t<queue_size>%d</queue_size>\n</message>" % (path_parts[1], path_parts[0], self.node_data, queue_position, len(self.queue_nodes)))
def get(self, path, watcher=None): """ gets the data associated with a node synchronously. PARAMETERS: path: the name of the node. Expressed as a file name with slashes separating ancestors of the node. (subsequent parameters are optional) watcher: if not None, a watch will be set at the server to notify the client if the node changes. RETURNS: the (data, stat) tuple associated with the node """ results = [] pc = utils.PipeCondition() ok = zookeeper.aget(self._zhandle, path, watcher, functools.partial(generic_completion, pc, results)) assert ok == zookeeper.OK pc.wait() #unpack result as data_completion handle, rc, data, stat = results assert handle == self._zhandle if rc == zookeeper.OK: return (data, stat) self._raise_exception(rc)
def get(self, path, watcher=None): """ gets the data associated with a node synchronously. PARAMETERS: path: the name of the node. Expressed as a file name with slashes separating ancestors of the node. (subsequent parameters are optional) watcher: if not None, a watch will be set at the server to notify the client if the node changes. RETURNS: the (data, stat) tuple associated with the node """ results = [] pc = utils.PipeCondition() ok = zookeeper.aget(self._zhandle, path, watcher, functools.partial(generic_completion, pc, results)) assert ok == zookeeper.OK pc.wait() # unpack result as data_completion handle, rc, data, stat = results assert handle == self._zhandle if rc == zookeeper.OK: return (data, stat) self._raise_exception(rc)
def get_async(self, path, watcher=None): async_result = self._new_async_result() def callback(handle, code, value, stat): if code != zookeeper.OK: exc = err_to_exception(code) async_result.set_exception(exc) else: async_result.set((value, stat)) watcher_callback, watcher_greenlet = self._setup_watcher(watcher) #TODO cleanup the watcher greenlet on error zookeeper.aget(self._handle, path, watcher_callback, callback) return async_result
def aget(self, path, watcher=None, handler=None): """A simple wrapper for zookeeper async get function. This function wraps the zookeeper aget call which allows the caller to register a function to handle the data once it is received as well as a function that will be called once the data has been updated. If neither is given then this function will do nothing. Args: path: The znode to watch. watcher: Called when the given znode is updated or changed. the basic footprint of this function is: func(zh, path) zh will be this object, and path will be the znode path. handler: Called when the data has been fetched from zookeper. The basic footprint of this function is: func(zh, path, rc, data) zh will be this object and path will be the znode path. rc is the return code from zookeeper. data is the contents of the znode. Returns: Nothing. """ register = False get = False self._lock.acquire() if watcher: register = path not in self._watches self._watches.setdefault(path, []).append(watcher) if handler: get = path not in self._handlers self._handlers.setdefault(path, []).append(handler) self._lock.release() if register or get: if register: w = self._watcher else: w = None # We use a lambda here so we can make sure that the path gets appended # to the args. This allows us to multiplex the call. h = (lambda zh, rc, data, stat: self._handler(zh, rc, data, stat, path)) logging.debug('Performing a get against %s', path) zookeeper.aget(self._zookeeper, path, w, h)
def _global_watch(self, zh, event, state, path): """Called when the connection to zookeeper has a state change.""" logging.debug('Global watch fired: %s %s %s' % (event, state, path)) if state == zookeeper.EXPIRED_SESSION_STATE: self._clientid = None self._connect() elif state == zookeeper.CONNECTED_STATE: self._clientid = zookeeper.client_id(self._zookeeper) # Re-get all existing watched files. logging.debug('Registering watches on existing watch objects.') for path in self._watches.iterkeys(): logging.debug('Registering watch against: %s' % path) h = self._handler_wrapper(path) zookeeper.aget(self._zookeeper, path, self._watcher, h) # Catch up all gets requested before we were able to connect. while self._pending_gets: path, w, h = self._pending_gets.pop() zookeeper.aget(self._zookeeper, path, w, h)
def aget(self, path, watcher=None, handler=None): """A simple wrapper for zookeeper async get function. This function wraps the zookeeper aget call which allows the caller to register a function to handle the data once it is received as well as a function that will be called once the data has been updated. If neither is given then this function will do nothing. Args: path: The znode to watch. watcher: Called when the given znode is updated or changed. the basic footprint of this function is: func(zh, path) zh will be this object, and path will be the znode path. handler: Called when the data has been fetched from zookeper. The basic footprint of this function is: func(zh, path, rc, data) zh will be this object and path will be the znode path. rc is the return code from zookeeper. data is the contents of the znode. Returns: Nothing. """ register = False get = False self._lock.acquire() if watcher: register = path not in self._watches self._watches.setdefault(path, []).append(watcher) if handler: get = path not in self._handlers self._handlers.setdefault(path, []).append(handler) self._lock.release() if register or get: if register: w = self._watcher else: w = None h = self._handler_wrapper(path) logging.debug('Performing a get against %s', path) zookeeper.aget(self._zookeeper, path, w, h)
def __init__(self, options, args): threading.Thread.__init__(self) logger.debug("Initializing MyClass thread.") if options.verbose: zookeeper.set_debug_level(zookeeper.LOG_LEVEL_DEBUG) self.zh = zookeeper.init(options.servers) if zookeeper.OK != zookeeper.aget(self.zh, self.znode, self.watcher, self.handler): logger.critical("Unable to get znode! Exiting.") sys.exit(1)
def __init__(self, options, args): threading.Thread.__init__(self) logger.debug('Initializing MyClass thread.') if options.verbose: zookeeper.set_debug_level(zookeeper.LOG_LEVEL_DEBUG) self.zh = zookeeper.init(options.servers) if zookeeper.OK != zookeeper.aget(self.zh, self.znode, self.watcher, self.handler): logger.critical('Unable to get znode! Exiting.') sys.exit(1)
def _get(self, path, watcher): d = defer.Deferred() if self._check_connected(d): return d def _cb_get(result_code, value, stat): if self._check_result(result_code, d, path=path): return d.callback((value, stat)) callback = self._zk_thread_callback(_cb_get) watcher = self._wrap_watcher(watcher, "get", path) result = zookeeper.aget(self.handle, path, watcher, callback) self._check_result(result, d, path=path) return d
def aget(self, node, callback): if not callable(callback): return None def watcher(self, zh, event, state, path): self.logger.info("Node state has been changed") #print "event", event if event == zookeeper.CHANGED_EVENT: self.logger.debug("Node %s has been modified" % str(path)) elif event == zookeeper.CREATED_EVENT: self.logger.debug("Node %s has been created" % str(path)) elif event == zookeeper.DELETED_EVENT: self.logger.warning("Node %s has been deleted" % str(path)) if state == zookeeper.EXPIRED_SESSION_STATE: self.logger.error("Session expired") callback(event, state, path) return zookeeper.aget(self.zkhandle, node, partial(watcher, self), partial(self.handler, zookeeper))
def test_dispatch_types(self): """ Test all the various dispatch mechanisms internal to the module. """ def dispatch_callback(*args, **kwargs): self.callback_flag = True self.ensureCreated("/zk-python-dispatch-test") self.callback_harness( lambda: zookeeper.adelete( self.handle, "/zk-python-dispatch-test", -1, self.create_callback(dispatch_callback)), lambda: self. assertEqual(True, self.callback_flag, "Void dispatch not fired")) self.ensureCreated("/zk-python-dispatch-test") self.callback_harness( lambda: zookeeper.aexists( self.handle, "/zk-python-dispatch-test", None, self.create_callback(dispatch_callback)), lambda: self. assertEqual(True, self.callback_flag, "Stat dispatch not fired")) self.callback_harness( lambda: zookeeper.aget( self.handle, "/zk-python-dispatch-test", None, self.create_callback(dispatch_callback)), lambda: self. assertEqual(True, self.callback_flag, "Data dispatch not fired")) self.callback_harness( lambda: zookeeper.aget_children( self.handle, "/", None, self. create_callback(dispatch_callback)), lambda: self.assertEqual( True, self.callback_flag, "Strings dispatch not fired")) self.callback_harness( lambda: getattr(zookeeper, 'async') (self.handle, "/", self.create_callback(dispatch_callback)), lambda: self.assertEqual(True, self.callback_flag, "String dispatch not fired")) self.callback_harness( lambda: zookeeper.aget_acl( self.handle, "/", self.create_callback(dispatch_callback)), lambda: self.assertEqual(True, self.callback_flag, "ACL dispatch not fired"))
def test_async_getset(self): self.cv = threading.Condition() def get_callback(handle, rc, value, stat): self.cv.acquire() self.callback_flag = True self.rc = rc self.value = (value, stat) self.cv.notify() self.cv.release() def set_callback(handle, rc, stat): self.cv.acquire() self.callback_flag = True self.rc = rc self.value = stat self.cv.notify() self.cv.release() self.assertEqual(self.connected, True, "Not connected!") self.cv.acquire() self.callback_flag = False ret = zookeeper.aset(self.handle, "/zk-python-agetsettest", "off", -1, set_callback) self.assertEqual(ret, zookeeper.OK, "aset failed") while not self.callback_flag: self.cv.wait(15) self.cv.release() self.assertEqual(self.callback_flag, True, "aset timed out") self.cv.acquire() self.callback_flag = False ret = zookeeper.aget(self.handle, "/zk-python-agetsettest", None, get_callback) self.assertEqual(ret, zookeeper.OK, "aget failed") self.cv.wait(15) self.cv.release() self.assertEqual(self.callback_flag, True, "aget timed out") self.assertEqual(self.value[0], "off", "Data is not 'off' as expected: " + self.value[0])
def test_async_getset(self): self.cv = threading.Condition() def get_callback(handle, rc, value, stat): self.cv.acquire() self.callback_flag = True self.rc = rc self.value = (value,stat) self.cv.notify() self.cv.release() def set_callback(handle, rc, stat): self.cv.acquire() self.callback_flag = True self.rc = rc self.value = stat self.cv.notify() self.cv.release() self.assertEqual(self.connected, True, "Not connected!") self.cv.acquire() self.callback_flag = False ret = zookeeper.aset(self.handle, "/zk-python-agetsettest", "off", -1, set_callback) self.assertEqual(ret, zookeeper.OK, "aset failed") while not self.callback_flag: self.cv.wait(15) self.cv.release() self.assertEqual(self.callback_flag, True, "aset timed out") self.cv.acquire() self.callback_flag = False ret = zookeeper.aget(self.handle, "/zk-python-agetsettest", None, get_callback) self.assertEqual(ret, zookeeper.OK, "aget failed") while not self.callback_flag: self.cv.wait(15) self.cv.release() self.assertEqual(self.callback_flag, True, "aget timed out") self.assertEqual(self.value[0], "off", "Data is not 'off' as expected: " + self.value[0])
def aget(self, path, callback, watcher=None): return zookeeper.aget(self.handle, path, watcher, callback)
return "%s/session_%d" % (root, i) @timed def do_operation(op, s, root, async, count, ephemeral=True, data=None): async_results = [] for i in range(count): path = child_path(root, i) if async: if op == Operation.create: cb = CreateCallback() zookeeper.acreate(s, path, data, acl, zookeeper.EPHEMERAL if ephemeral else 0, cb) elif op == Operation.get: cb = GetCallback() zookeeper.aget(s, path, None, cb) elif op == Operation.set: cb = SetCallback() zookeeper.aset(s, path, data, -1, cb) elif op == Operation.delete: cb = DeleteCallback() zookeeper.adelete(s, path, -1, cb) async_results.append(cb) else: if op == Operation.create: zookeeper.create(s, path, data, acl, zookeeper.EPHEMERAL if ephemeral else 0) elif op == Operation.get: zookeeper.get(s, path) elif op == Operation.set: zookeeper.set(s, path, data)
def _aget(self, path): self._try_zoo( lambda: self._use_socket( lambda z: zookeeper.aget(z, path, self._events, self._get_cb(path))))
def aget(self): return zookeeper.aget(self.zh, self.znode, self.watcher, self.handler)