def async_delete(self, path, version=None, callback=None): """Async call to delete zookeeper node's data. Args: path: zookeeper node path version: expected node version callback: callback method to be invoked upon operation completion with zookeeper api handle, return_code. callback will be invoked in the context of the underlying zookeeper API thread. Returns: Zookeeper.AsyncResult if callback is None, otherwise None. """ version = version if version is not None else -1 async_result = None if callback is None: async_result = self.AsyncResult() def async_callback(handle, return_code): if return_code == zookeeper.OK: async_result.set(None) else: async_result.set_exception( self.error_to_exception(return_code)) callback = async_callback zookeeper.adelete(self.handle, path, version, callback) return async_result
def generateIDBlock(self, app_id, entity_key = GLOBAL_ID_KEY): """ Generate ID block for specific key. This generates ID block that is unique in specified key. If the key doesn't specify, this generates global ID. This method returns long start ID and block length tuple. """ self.__waitForConnect() idrootpath = self.__getIDRootPath(app_id, entity_key) path = PATH_SEPARATOR.join([idrootpath, APP_ID_PREFIX]) value = entity_key start = -1 retry = True while retry: retry = False try: idpath = zookeeper.create(self.handle, path, value, ZOO_ACL_OPEN, zookeeper.SEQUENCE) zookeeper.adelete(self.handle, idpath) idbase = long(idpath.split(PATH_SEPARATOR)[-1].lstrip(APP_ID_PREFIX)) start = idbase * ID_BLOCK # self.__updateNode(PATH_SEPARATOR.join([idrootpath, APP_ID_PREID_PATH]), str(start)) if start == 0: retry = True except zookeeper.NoNodeException: self.__forceCreatePath(idrootpath) retry = True except Exception, e: print e raise ZKTransactionException(ZKTransactionException.TYPE_UNKNOWN, "Fail to generate ID: %s" % e)
def getTransactionID(self, app_id): """ Get new transaction ID for transaction. This function only create transaction ID, and you must lock particular root entity using acquireLock(). The ID is long number. The transaction will expire in 30 seconds. """ self.__waitForConnect() rootpath = self.__getTransactionRootPath(app_id) value = str(time.time()) id = -1 retry = True while retry: retry = False path = None try: path = zookeeper.create(self.handle, PATH_SEPARATOR.join([rootpath, APP_TX_PREFIX]), value, ZOO_ACL_OPEN, zookeeper.SEQUENCE) if path: id = long(path.split(PATH_SEPARATOR)[-1].lstrip(APP_TX_PREFIX)) if id == 0: # avoid id 0 zookeeper.adelete(self.handle, path) retry = True except zookeeper.NoNodeException: self.__forceCreatePath(rootpath) retry = True return id
def async_delete(self, path, version=None, callback=None): """Async call to delete zookeeper node's data. Args: path: zookeeper node path version: expected node version callback: callback method to be invoked upon operation completion with zookeeper api handle, return_code. callback will be invoked in the context of the underlying zookeeper API thread. Returns: Zookeeper.AsyncResult if callback is None, otherwise None. """ version = version if version is not None else -1 async_result = None if callback is None: async_result = self.AsyncResult() def async_callback(handle, return_code): if return_code == zookeeper.OK: async_result.set(None) else: async_result.set_exception(self.error_to_exception(return_code)) callback = async_callback zookeeper.adelete(self.handle, path, version, callback) return async_result
def delete_async(self, path, version=-1): async_result = self._new_async_result() def callback(handle, code): self._queue_result(async_result, code != zookeeper.OK, err_to_exception(code) if code != zookeeper.OK else code) zookeeper.adelete(self._handle, path, version, callback) return async_result
def delete_async(self, path, version=-1): """Asynchronously delete a node @param path: path of node to delete @param version: version of node to delete, or -1 for any @return AyncResult set upon completion @rtype AsyncResult """ async_result = self._sync.async_result() callback = partial(_generic_callback, async_result) zookeeper.adelete(self._handle, path, version, callback) return async_result
def test_async_delete(self): ZOO_OPEN_ACL_UNSAFE = { "perms": 0x1f, "scheme": "world", "id": "anyone" } self.assertEqual(self.connected, True) ret = zookeeper.create(self.handle, "/zk-python-adeletetest", "nodecontents", [ZOO_OPEN_ACL_UNSAFE], zookeeper.EPHEMERAL) self.assertEqual(ret, "/zk-python-adeletetest") self.cv = threading.Condition() self.callback_flag = False self.rc = -1 def callback(handle, rc): self.cv.acquire() self.callback_flag = True self.cv.notify() self.rc = rc # don't assert this here, as if the assertion fails, the test will block self.cv.release() self.cv.acquire() ret = zookeeper.adelete(self.handle, "/zk-python-adeletetest", -1, callback) self.assertEqual(ret, zookeeper.OK, "adelete failed") while not self.callback_flag: self.cv.wait(15) self.cv.release() self.assertEqual(self.callback_flag, True, "adelete timed out") self.assertEqual(self.rc, zookeeper.OK)
def test_async_delete(self): ZOO_OPEN_ACL_UNSAFE = {"perms":0x1f, "scheme":"world", "id" :"anyone"} self.assertEqual(self.connected, True) ret = zookeeper.create(self.handle, "/zk-python-adeletetest", "nodecontents", [ZOO_OPEN_ACL_UNSAFE], zookeeper.EPHEMERAL) self.assertEqual(ret, "/zk-python-adeletetest") self.cv = threading.Condition() self.callback_flag = False self.rc = -1 def callback(handle, rc): self.cv.acquire() self.callback_flag = True self.cv.notify() self.rc = rc # don't assert this here, as if the assertion fails, the test will block self.cv.release() self.cv.acquire() ret = zookeeper.adelete(self.handle,"/zk-python-adeletetest",-1,callback) self.assertEqual(ret, zookeeper.OK, "adelete failed") while not self.callback_flag: self.cv.wait(15) self.cv.release() self.assertEqual(self.callback_flag, True, "adelete timed out") self.assertEqual(self.rc, zookeeper.OK)
def create_sequence_node(self, path, value): """ Creates a new sequence node in ZooKeeper, with a non-zero initial ID. We avoid using zero as the initial ID because Google App Engine apps can use a zero ID as a sentinel value, to indicate that an ID should be allocated for them. Args: path: The prefix to create the sequence node at. For example, a prefix of '/abc' would result in a sequence node of '/abc1' being created. value: The value that we should store in the sequence node. Returns: A long that represents the sequence ID. Raises: ZKTransactionException: If the sequence node couldn't be created. """ retries_left = self.DEFAULT_NUM_RETRIES while retries_left > 0: try: txn_id_path = self.run_with_timeout(self.DEFAULT_ZK_TIMEOUT, self.DEFAULT_NUM_RETRIES, zookeeper.create, self.handle, path, value, ZOO_ACL_OPEN, zookeeper.SEQUENCE) if txn_id_path: txn_id = long(txn_id_path.split(PATH_SEPARATOR)[-1].lstrip( APP_TX_PREFIX)) if txn_id == 0: logging.warning("Created sequence ID 0 - deleting it.") zookeeper.adelete(self.handle, txn_id_path) else: logging.debug("Created sequence ID {0} at path {1}".format(txn_id, txn_id_path)) return txn_id except zookeeper.NoNodeException: self.force_create_path(path) finally: retries_left -= 1 logging.error("Unable to create sequence node with path {0}, value {1}" \ .format(path, value)) raise ZKTransactionException("Unable to create sequence node with path" \ " {0}, value {1}".format(path, value))
def releaseLock(self, app_id, txid, key = None): """ Release acquired lock. You must call acquireLock() first. if the transaction is not valid or it is expired, this raises Exception. After the release lock, you could not use transaction ID again. If there is no lock, this method returns False. """ self.__waitForConnect() self.checkTransaction(app_id, txid) txpath = self.__getTransactionPath(app_id, txid) has_lock = False try: lockpath = zookeeper.get(self.handle, PATH_SEPARATOR.join([txpath, TX_LOCK_PATH]), None)[0] if key: lockroot = self.__getLockRootPath(app_id, key) if not lockroot == lockpath: raise ZKTransactionException(ZKTransactionException.TYPE_DIFFERENT_ROOTKEY, "You can not specify different root entity for release.") zookeeper.adelete(self.handle, lockpath) has_lock = True except zookeeper.NoNodeException: # there is no lock. pass # If the transaction doesn't have active lock or not, we should delete it. # delete transaction node for child in zookeeper.get_children(self.handle, txpath): zookeeper.adelete(self.handle, PATH_SEPARATOR.join([txpath, child])) zookeeper.adelete(self.handle, txpath) return has_lock
def async_delete(self, path, version=None): """Async call to delete zookeeper node's data. Args: path: zookeeper node path version: expected node version Returns: Zookeeper.AsyncResult if callback is None, otherwise None. """ version = version if version is not None else -1 async_result = self._async_result() def callback(handle, return_code): if return_code == zookeeper.OK: async_result.set(None) else: async_result.set_exception(self.error_to_exception(return_code)) zookeeper.adelete(self.handle, path, version, callback) return async_result
def notifyFailedTransaction(self, app_id, txid): """ Notify failed transaction id. This method will add the transaction id into black list. After this call, the transaction becomes invalid. """ self.__waitForConnect() self.checkTransaction(app_id, txid) print "notify failed transaction app:%s, txid:%d" % (app_id, txid) txpath = self.__getTransactionPath(app_id, txid) lockpath = None try: lockpath = zookeeper.get(self.handle, PATH_SEPARATOR.join([txpath, TX_LOCK_PATH]), None)[0] except zookeeper.NoNodeException: # there is no lock. it means there is no need to rollback. pass if lockpath: # add transacion id to black list. now = str(time.time()) broot = self.__getBlacklistRootPath(app_id) if not zookeeper.exists(self.handle, broot): self.__forceCreatePath(broot) zookeeper.acreate(self.handle, PATH_SEPARATOR.join([broot, str(txid)]), now, ZOO_ACL_OPEN) # update local cache before notification if app_id in self.blacklistcache: with self.blacklistcv: self.blacklistcache[app_id].add(str(txid)) # copy valid transaction id for each updated key into valid list. for child in zookeeper.get_children(self.handle, txpath): if re.match("^" + TX_UPDATEDKEY_PREFIX, child): value = zookeeper.get(self.handle, PATH_SEPARATOR.join([txpath, child]), None)[0] valuelist = value.split(PATH_SEPARATOR) key = urllib.unquote_plus(valuelist[0]) vid = valuelist[1] vtxroot = self.__getValidTransactionRootPath(app_id) if not zookeeper.exists(self.handle, vtxroot): self.__forceCreatePath(vtxroot) vtxpath = self.__getValidTransactionPath(app_id, key) zookeeper.acreate(self.handle, vtxpath, vid, ZOO_ACL_OPEN) # release the lock try: zookeeper.adelete(self.handle, lockpath) except zookeeper.NoNodeException: # this should be retry. pass # just remove transaction node try: for item in zookeeper.get_children(self.handle, txpath): zookeeper.adelete(self.handle, PATH_SEPARATOR.join([txpath, item])) zookeeper.adelete(self.handle, txpath) except zookeeper.NoNodeException: # something wrong. next GC will take care of it. return False return True
def delete(self, path, version=-1): """ Delete the node at the given path. If the current node version on the server is more recent than that supplied by the client, a bad version exception wil be thrown. A version of -1 (default) specifies any version. @param path: the path of the node to be deleted. @param version: the integer version of the node. """ d = defer.Deferred() callback = self._zk_thread_callback(self._cb_deleted, d, path) result = zookeeper.adelete(self.handle, path, version, callback) self._check_result(result, d, path=path) return d
def test_dispatch_types(self): """ Test all the various dispatch mechanisms internal to the module. """ def dispatch_callback(*args, **kwargs): self.callback_flag = True self.ensureCreated("/zk-python-dispatch-test") self.callback_harness( lambda: zookeeper.adelete( self.handle, "/zk-python-dispatch-test", -1, self.create_callback(dispatch_callback)), lambda: self. assertEqual(True, self.callback_flag, "Void dispatch not fired")) self.ensureCreated("/zk-python-dispatch-test") self.callback_harness( lambda: zookeeper.aexists( self.handle, "/zk-python-dispatch-test", None, self.create_callback(dispatch_callback)), lambda: self. assertEqual(True, self.callback_flag, "Stat dispatch not fired")) self.callback_harness( lambda: zookeeper.aget( self.handle, "/zk-python-dispatch-test", None, self.create_callback(dispatch_callback)), lambda: self. assertEqual(True, self.callback_flag, "Data dispatch not fired")) self.callback_harness( lambda: zookeeper.aget_children( self.handle, "/", None, self. create_callback(dispatch_callback)), lambda: self.assertEqual( True, self.callback_flag, "Strings dispatch not fired")) self.callback_harness( lambda: getattr(zookeeper, 'async') (self.handle, "/", self.create_callback(dispatch_callback)), lambda: self.assertEqual(True, self.callback_flag, "String dispatch not fired")) self.callback_harness( lambda: zookeeper.aget_acl( self.handle, "/", self.create_callback(dispatch_callback)), lambda: self.assertEqual(True, self.callback_flag, "ACL dispatch not fired"))
def delete(self, path, version=-1): """ delete a node in zookeeper synchronously. PARAMETERS: path: the name of the node. Expressed as a file name with slashes separating ancestors of the node. (Subsequent parameters are optional) version: the expected version of the node. The function will fail if the actual version of the node does not match the expected version. If -1 (the default) is used the version check will not take place. RETURNS: OK operation completed successfully One of the following exceptions is returned when an error occurs. NONODE the node does not exist. NOAUTH the client does not have permission. BADVERSION expected version does not match actual version. NOTEMPTY children are present; node cannot be deleted. BADARGUMENTS - invalid input parameters INVALIDSTATE - zhandle state is either SESSION_EXPIRED_STATE or AUTH_FAILED_STATE MARSHALLINGERROR - failed to marshal a request; possibly, out of memory """ pc = utils.StatePipeCondition() ok = zookeeper.adelete(self._zhandle, path, version, functools.partial(_generic_completion, pc)) assert ok == zookeeper.OK results = pc.wait_and_get() pc.close() #unpack result as void_completion handle, rc = results assert handle == self._zhandle if rc == zookeeper.OK: return rc self._raise_exception(rc)
def delete(self, path, version=-1): """ delete a node in zookeeper synchronously. PARAMETERS: path: the name of the node. Expressed as a file name with slashes separating ancestors of the node. (Subsequent parameters are optional) version: the expected version of the node. The function will fail if the actual version of the node does not match the expected version. If -1 (the default) is used the version check will not take place. RETURNS: OK operation completed successfully One of the following exceptions is returned when an error occurs. NONODE the node does not exist. NOAUTH the client does not have permission. BADVERSION expected version does not match actual version. NOTEMPTY children are present; node cannot be deleted. BADARGUMENTS - invalid input parameters INVALIDSTATE - zhandle state is either SESSION_EXPIRED_STATE or AUTH_FAILED_STATE MARSHALLINGERROR - failed to marshal a request; possibly, out of memory """ results = [] pc = utils.PipeCondition() ok = zookeeper.adelete( self._zhandle, path, version, functools.partial(generic_completion, pc, results)) assert ok == zookeeper.OK pc.wait() #unpack result as void_completion handle, rc = results assert handle == self._zhandle if rc == zookeeper.OK: return rc self._raise_exception(rc)
def releaseLock(self, app_id, txid, key = None): """ Release acquired lock. You must call acquireLock() first. if the transaction is not valid or it is expired, this raises Exception. After the release lock, you could not use transaction ID again. If there is no lock, this method returns False. Args: app_id: The application ID we are releasing a lock for. txid: The transaction ID we are releasing a lock for. key: The entity key we use to build the path. Returns: True on success. Raises: ZKTransactionException: When a lock can not be released. """ self.__waitForConnect() self.checkTransaction(app_id, txid) txpath = self.__getTransactionPath(app_id, txid) has_lock = False try: lockpath = zookeeper.get(self.handle, PATH_SEPARATOR.join([txpath, TX_LOCK_PATH]), None)[0] if key: lockroot = self.__getLockRootPath(app_id, key) if not lockroot == lockpath: raise ZKTransactionException(ZKTransactionException.TYPE_DIFFERENT_ROOTKEY, "zktransaction.releaseLock: You can not specify different root entity for release.") zookeeper.adelete(self.handle, lockpath) has_lock = True except zookeeper.NoNodeException: # there is no lock. pass # If the transaction doesn't have active lock or not, we should delete it. # delete transaction node for child in zookeeper.get_children(self.handle, txpath): zookeeper.adelete(self.handle, PATH_SEPARATOR.join([txpath, child])) zookeeper.adelete(self.handle, txpath) return has_lock
def adelete(self, path, callback, version=-1): return zookeeper.adelete(self.handle, path, version, callback)
for i in range(count): path = child_path(root, i) if async: if op == Operation.create: cb = CreateCallback() zookeeper.acreate(s, path, data, acl, zookeeper.EPHEMERAL if ephemeral else 0, cb) elif op == Operation.get: cb = GetCallback() zookeeper.aget(s, path, None, cb) elif op == Operation.set: cb = SetCallback() zookeeper.aset(s, path, data, -1, cb) elif op == Operation.delete: cb = DeleteCallback() zookeeper.adelete(s, path, -1, cb) async_results.append(cb) else: if op == Operation.create: zookeeper.create(s, path, data, acl, zookeeper.EPHEMERAL if ephemeral else 0) elif op == Operation.get: zookeeper.get(s, path) elif op == Operation.set: zookeeper.set(s, path, data) elif op == Operation.delete: zookeeper.delete(s, path) for cb in async_results: cb.wait()
if re.match("^" + TX_UPDATEDKEY_PREFIX, child): value = zookeeper.get(self.handle, PATH_SEPARATOR.join([txpath, child]), None)[0] valuelist = value.split(PATH_SEPARATOR) key = urllib.unquote_plus(valuelist[0]) vid = valuelist[1] vtxroot = self.get_valid_transaction_root_path(app_id) if not zookeeper.exists(self.handle, vtxroot): self.force_create_path(vtxroot) vtxpath = self.get_valid_transaction_path(app_id, key) zookeeper.acreate(self.handle, vtxpath, vid, ZOO_ACL_OPEN) # Release the locks. for lock in lock_list: zookeeper.adelete(self.handle, lock) if self.is_xg(app_id, txid): zookeeper.adelete(self.handle, self.get_xg_path(app_id, txid)) # Remove the transaction paths. for item in zookeeper.get_children(self.handle, txpath): zookeeper.adelete(self.handle, PATH_SEPARATOR.join([txpath, item])) zookeeper.adelete(self.handle, txpath) except zookeeper.ZooKeeperException, zk_exception: logging.error("There was a ZooKeeper exception {0}".format(str( zk_exception))) return True def reestablish_connection(self): """ Checks the connection and resets it as needed. """