def create_async(self, path, value, acl=None, ephemeral=False, sequence=False): """Asynchronously create a ZNode @param path: path of node @param value: initial value of node @param acl: permissions for node @param ephemeral: boolean indicating whether node is ephemeral (tied to this session) @param sequence: boolean indicating whether path is suffixed with a unique index @return: AsyncResult object set on completion with the real path of the new node @rtype AsyncResult """ flags = 0 if ephemeral: flags |= zookeeper.EPHEMERAL if sequence: flags |= zookeeper.SEQUENCE if acl is None: acl = (ZK_OPEN_ACL_UNSAFE, ) async_result = self._sync.async_result() callback = partial(_generic_callback, async_result) zookeeper.acreate(self._handle, path, value, list(acl), flags, callback) return async_result
def __init__(self, zkhosts, root=NODE_HQ_ROOT, alivenode='alive'): """zkhosts: a string or a list. list will be ','.join-ed into a string. root: root node path (any parents must exist, if any) """ if not isinstance(zkhosts, basestring): zkhosts = ','.join(zkhosts) self.zkhosts = zkhosts self.ROOT = root self.alivenode = alivenode self.nodename = os.uname()[1] self.NODE_SERVERS = self.ROOT + '/servers' self.NODE_ME = self.NODE_SERVERS+'/'+self.nodename self.zh = zk.init(self.zkhosts, self.__watcher) self.jobs = {} self.initialize_node_structure() if not zk.exists(self.zh, self.NODE_ME): zk.create(self.zh, self.NODE_ME, '', PERM_WORLD) # setup notification zk.get_children(self.zh, self.NODE_SERVERS, self.__servers_watcher) #zk.get_children(self.zh, self.NODE_ME, self.__watcher) self.NODE_JOBS = self.NODE_ME+'/jobs' zk.acreate(self.zh, self.NODE_JOBS, '', PERM_WORLD)
def async_create(self, path, data=None, acl=None, sequence=False, ephemeral=False): """Async call to create Zookeeper node. Args: path: zookeeper node path, i.e. /my/zookeeper/node/path data: optional zookeeper node data (string) acl: optional zookeeper access control list (default is insecure) sequence: if True node will be created by adding a unique number the supplied path. ephemeral: if True, node will automatically be deleted when client exists. Returns: Zookeeper.AsyncResult if callback is None, otherwise None. """ data = data or "" acl = acl or self.acl flags = (zookeeper.SEQUENCE if sequence else 0) | (zookeeper.EPHEMERAL if ephemeral else 0) async_result = self._async_result() def callback(handle, return_code, path): if return_code == zookeeper.OK: async_result.set(path) else: async_result.set_exception(self.error_to_exception(return_code)) zookeeper.acreate(self.handle, path, data, acl, flags, callback) return async_result
def create_ignore_errors(zh, path, data): def completion(*args, **kwargs): rc, zrc, ignored = args if rc != 0: print("zookeeper.acreate() error") elif zrc != 0 and zrc != zookeeper.NODEEXISTS: print('create/set('+path+') : FAILED') sem.release() zookeeper.acreate(zh, path, data, acl_openbar, 0, completion)
def notifyFailedTransaction(self, app_id, txid): """ Notify failed transaction id. This method will add the transaction id into black list. After this call, the transaction becomes invalid. """ self.__waitForConnect() self.checkTransaction(app_id, txid) print "notify failed transaction app:%s, txid:%d" % (app_id, txid) txpath = self.__getTransactionPath(app_id, txid) lockpath = None try: lockpath = zookeeper.get(self.handle, PATH_SEPARATOR.join([txpath, TX_LOCK_PATH]), None)[0] except zookeeper.NoNodeException: # there is no lock. it means there is no need to rollback. pass if lockpath: # add transacion id to black list. now = str(time.time()) broot = self.__getBlacklistRootPath(app_id) if not zookeeper.exists(self.handle, broot): self.__forceCreatePath(broot) zookeeper.acreate(self.handle, PATH_SEPARATOR.join([broot, str(txid)]), now, ZOO_ACL_OPEN) # update local cache before notification if app_id in self.blacklistcache: with self.blacklistcv: self.blacklistcache[app_id].add(str(txid)) # copy valid transaction id for each updated key into valid list. for child in zookeeper.get_children(self.handle, txpath): if re.match("^" + TX_UPDATEDKEY_PREFIX, child): value = zookeeper.get(self.handle, PATH_SEPARATOR.join([txpath, child]), None)[0] valuelist = value.split(PATH_SEPARATOR) key = urllib.unquote_plus(valuelist[0]) vid = valuelist[1] vtxroot = self.__getValidTransactionRootPath(app_id) if not zookeeper.exists(self.handle, vtxroot): self.__forceCreatePath(vtxroot) vtxpath = self.__getValidTransactionPath(app_id, key) zookeeper.acreate(self.handle, vtxpath, vid, ZOO_ACL_OPEN) # release the lock try: zookeeper.adelete(self.handle, lockpath) except zookeeper.NoNodeException: # this should be retry. pass # just remove transaction node try: for item in zookeeper.get_children(self.handle, txpath): zookeeper.adelete(self.handle, PATH_SEPARATOR.join([txpath, item])) zookeeper.adelete(self.handle, txpath) except zookeeper.NoNodeException: # something wrong. next GC will take care of it. return False return True
def create_async(self, path, value, acl, flags): async_result = self._new_async_result() def callback(handle, code, path): self._queue_result(async_result, code != zookeeper.OK, err_to_exception(code) if code != zookeeper.OK else path) zookeeper.acreate(self._handle, path, value, acl, flags, callback) return async_result
def publish_job(self, job): '''job: hq.CrawlJob''' node = self.NODE_JOBS+'/'+job.jobname zk.acreate(self.zh, node, '', PERM_WORLD) node = self.NODE_GJOBS+'/'+job.jobname self.acreate(node) self.acreate('/'.join((self.NODE_GJOBS, job.jobname, self.nodename)), flags='e')
def create_async(self, path, value, acl, flags): async_result = self._new_async_result() def callback(handle, code, path): if code != zookeeper.OK: exc = err_to_exception(code) async_result.set_exception(exc) else: async_result.set(path) zookeeper.acreate(self._handle, path, value, acl, flags, callback) return async_result
def _create_ignore_errors(zh, path, data, ctx): def _completion(ctx, *args, **kwargs): rc, zrc, ignored = args if rc != 0: logging.warn("zookeeper.acreate(%s) error rc=%d", path, rc) ctx['failed'] += 1 elif zrc != 0 and zrc != zookeeper.NODEEXISTS: logging.warn('create/set(%s) : FAILED', path) ctx['failed'] += 1 ctx['sem'].release() ctx['started'] += 1 zookeeper.acreate(zh, path, data, _acl_openbar, 0, lambda *a, **ka: _completion(ctx, *a, **ka))
def acquireLock(self, app_id, txid, entity_key = GLOBAL_LOCK_KEY): """ Acquire lock for transaction. You must call getTransactionID() first to obtain transaction ID. You could call this method anytime if the root entity key is same. Args: app_id: The application ID to acquire a lock for. txid: The transaction ID you are acquiring a lock for. Built into the path. entity_key: Used to get the root path. Raises: ZKTransactionException: If it could not get the lock. """ self.__waitForConnect() txpath = self.__getTransactionPath(app_id, txid) lockrootpath = self.__getLockRootPath(app_id, entity_key) lockpath = None if zookeeper.exists(self.handle, PATH_SEPARATOR.join([txpath, TX_LOCK_PATH])): # use current lock prelockpath = zookeeper.get(self.handle, PATH_SEPARATOR.join([txpath, TX_LOCK_PATH]), None)[0] if not lockrootpath == prelockpath: raise ZKTransactionException(ZKTransactionException.TYPE_DIFFERENT_ROOTKEY, "zktransaction.acquireLock: You can not lock different root entity in same transaction.") print "Already has lock: %s" % lockrootpath return True self.checkTransaction(app_id, txid) # create new lock retry = True while retry: retry = False try: lockpath = zookeeper.create(self.handle, lockrootpath, txpath, ZOO_ACL_OPEN) except zookeeper.NoNodeException: self.__forceCreatePath(PATH_SEPARATOR.join(lockrootpath.split(PATH_SEPARATOR)[:-1])) retry = True except zookeeper.NodeExistsException: # fail to get lock raise ZKTransactionException(ZKTransactionException.TYPE_CONCURRENT, "zktransaction.acquireLock: There is already another transaction using this lock") # set lockpath for transaction node # TODO: we should think about atomic operation or recovery of # inconsistent lockpath node. zookeeper.acreate(self.handle, PATH_SEPARATOR.join([txpath, TX_LOCK_PATH]), lockpath, ZOO_ACL_OPEN) return True
def acreate(self, path, data='', perm=PERM_WORLD, flags=''): try: return zk.acreate(self.zh, path, data, perm, create_flags(flags)) except SystemError: # acreate C code often returns error without setting exception, # which cuases SystemError. pass
def test_async_create(self): self.cv = threading.Condition() def callback(handle, rc, value): self.cv.acquire() self.callback_flag = True self.rc = rc self.cv.notify() self.cv.release() ZOO_OPEN_ACL_UNSAFE = { "perms": 0x1f, "scheme": "world", "id": "anyone" } self.assertEqual(self.connected, True, "Not connected!") self.cv.acquire() ret = zookeeper.acreate(self.handle, "/zk-python-acreatetest", "nodecontents", [ZOO_OPEN_ACL_UNSAFE], zookeeper.EPHEMERAL, callback) self.assertEqual(ret, zookeeper.OK, "acreate failed") while not self.callback_flag: self.cv.wait(15) self.cv.release() self.assertEqual(self.callback_flag, True, "acreate timed out") self.assertEqual(self.rc, zookeeper.OK)
def create_ignore_errors (zh, path, data): def completion (*args, **kwargs): rc, zrc, ignored = args if rc != 0: print "zookeeper.acreate() error" else: if zrc == 0: #print 'create/set('+path+') : OK' pass elif zrc == zookeeper.NODEEXISTS: #print 'create/set('+path+') : ALREADY' pass else: print 'create/set('+path+') : FAILED' sem.release() zookeeper.acreate(zh, path, data, acl_openbar, 0, completion)
def post_to_queue(self, form): self.results = None self.node = None self.results_flag = threading.Event() if zookeeper.OK != zookeeper.acreate(self.server.zk, '/queues/%s/%s-' % (form['queue'].value, form['queue'].value), form['message'].value, [{'perms': 31, 'scheme': 'world', 'id': 'anyone'}], zookeeper.SEQUENCE, self.post_handler): self.server.log.warning('Could not post job to queue %s' % form['queue'].value) self.results_flag.wait() if self.results == zookeeper.OK: self.server.log.info('New message posted to queue %s: %s' % (form['queue'].value, form['message'].value)) self.send_response(200) self.end_headers() self.wfile.write(self.node) elif self.results == zookeeper.NONODE: self.server.log.info('Creating queue %s' % form['queue'].value) if self.new_queue(form): self.post_to_queue(form) else: self.server.log.critical('Something strange happened: RC = %s' % self.results)
def create_ignore_errors(zh, path, data): def completion(*args, **kwargs): rc, zrc, ignored = args if rc != 0: print "zookeeper.acreate() error" else: if zrc == 0: #print 'create/set('+path+') : OK' pass elif zrc == zookeeper.NODEEXISTS: #print 'create/set('+path+') : ALREADY' pass else: print 'create/set(' + path + ') : FAILED' sem.release() zookeeper.acreate(zh, path, data, acl_openbar, 0, completion)
def registUpdatedKey(self, app_id, current_txid, target_txid, entity_key): """ Regist valid transaction id for entity. target_txid must be the latest valid transaction id for the entity. """ self.__waitForConnect() vtxpath = self.__getValidTransactionPath(app_id, entity_key) if zookeeper.exists(self.handle, vtxpath): # just update transaction id for entity if there is valid transaction node. zookeeper.aset(self.handle, vtxpath, str(target_txid)) else: # store the updated key info into current transaction node. value = PATH_SEPARATOR.join([urllib.quote_plus(entity_key), str(target_txid)]) txpath = self.__getTransactionPath(app_id, current_txid) if zookeeper.exists(self.handle, txpath): zookeeper.acreate(self.handle, PATH_SEPARATOR.join([txpath, TX_UPDATEDKEY_PREFIX]), value, ZOO_ACL_OPEN, zookeeper.SEQUENCE) else: raise ZKTransactionException(ZKTransactionException.TYPE_INVALID, "Transaction %d is not valid." % current_txid)
def acreate(self, path, callback, data="", flags=0, acl=[ZOO_OPEN_ACL_UNSAFE]): result = zookeeper.acreate(self.handle, path, data, acl, flags, callback) return result
def create(self, path, value, acl=None, flags=0): """ create a node synchronously. This method will create a node in ZooKeeper. A node can only be created if it does not already exists. The Create Flags affect the creation of nodes. If the EPHEMERAL flag is set, the node will automatically get removed if the client session goes away. If the SEQUENCE flag is set, a unique monotonically increasing sequence number is appended to the path name. PARAMETERS: path: The name of the node. Expressed as a file name with slashes separating ancestors of the node. value: The data to be stored in the node. acl: The initial ACL of the node. If None, the ACL of the parent will be used. flags: this parameter can be set to 0 for normal create or an OR of the Create Flags The real path that is created (this might be different than the path to create because of the SEQUENCE flag. the maximum length of real path you would want. RETURNS: The actual znode path that was created (may be different from path due to use of SEQUENTIAL flag). EXCEPTIONS: NONODE the parent node does not exist. NODEEXISTS the node already exists NOAUTH the client does not have permission. NOCHILDRENFOREPHEMERALS cannot create children of ephemeral nodes. BADARGUMENTS - invalid input parameters INVALIDSTATE - zhandle state is either SESSION_EXPIRED_STATE or AUTH_FAILED_STATE MARSHALLINGERROR - failed to marshall a request; possibly, out of memory """ pc = utils.StatePipeCondition() ok = zookeeper.acreate(self._zhandle, path, value, acl, flags, functools.partial(_generic_completion, pc)) assert ok == zookeeper.OK results = pc.wait_and_get() pc.close() #unpack result as string_completion handle, rc, real_path = results assert handle == self._zhandle if rc == zookeeper.OK: return real_path self._raise_exception(rc)
def create(self, path, value, acl=None, flags=0): """ create a node synchronously. This method will create a node in ZooKeeper. A node can only be created if it does not already exists. The Create Flags affect the creation of nodes. If the EPHEMERAL flag is set, the node will automatically get removed if the client session goes away. If the SEQUENCE flag is set, a unique monotonically increasing sequence number is appended to the path name. PARAMETERS: path: The name of the node. Expressed as a file name with slashes separating ancestors of the node. value: The data to be stored in the node. acl: The initial ACL of the node. If None, the ACL of the parent will be used. flags: this parameter can be set to 0 for normal create or an OR of the Create Flags The real path that is created (this might be different than the path to create because of the SEQUENCE flag. the maximum length of real path you would want. RETURNS: The actual znode path that was created (may be different from path due to use of SEQUENTIAL flag). EXCEPTIONS: NONODE the parent node does not exist. NODEEXISTS the node already exists NOAUTH the client does not have permission. NOCHILDRENFOREPHEMERALS cannot create children of ephemeral nodes. BADARGUMENTS - invalid input parameters INVALIDSTATE - zhandle state is either SESSION_EXPIRED_STATE or AUTH_FAILED_STATE MARSHALLINGERROR - failed to marshall a request; possibly, out of memory """ results = [] pc = utils.PipeCondition() ok = zookeeper.acreate( self._zhandle, path, value, acl, flags, functools.partial(generic_completion, pc, results)) assert ok == zookeeper.OK pc.wait() #unpack result as string_completion handle, rc, real_path = results assert handle == self._zhandle if rc == zookeeper.OK: return real_path self._raise_exception(rc)
def acquireLock(self, app_id, txid, entity_key = GLOBAL_LOCK_KEY): """ Acquire lock for transaction. You must call getTransactionID() first to obtain transaction ID. You could call this method anytime if the root entity key is same. If you could not get lock, this returns False. """ self.__waitForConnect() txpath = self.__getTransactionPath(app_id, txid) lockrootpath = self.__getLockRootPath(app_id, entity_key) lockpath = None if zookeeper.exists(self.handle, PATH_SEPARATOR.join([txpath, TX_LOCK_PATH])): # use current lock prelockpath = zookeeper.get(self.handle, PATH_SEPARATOR.join([txpath, TX_LOCK_PATH]), None)[0] if not lockrootpath == prelockpath: raise ZKTransactionException(ZKTransactionException.TYPE_DIFFERENT_ROOTKEY, "You can not lock different root entity in same transaction.") print "already has lock: %s" % lockrootpath return True self.checkTransaction(app_id, txid) # create new lock retry = True while retry: retry = False try: lockpath = zookeeper.create(self.handle, lockrootpath, txpath, ZOO_ACL_OPEN) except zookeeper.NoNodeException: self.__forceCreatePath(PATH_SEPARATOR.join(lockrootpath.split(PATH_SEPARATOR)[:-1])) retry = True except zookeeper.NodeExistsException: # fail to get lock raise ZKTransactionException(ZKTransactionException.TYPE_CONCURRENT, "There is already another transaction using this lock") # set lockpath for transaction node # TODO: we should think about atomic operation or recovery of # inconsistent lockpath node. zookeeper.acreate(self.handle, PATH_SEPARATOR.join([txpath, TX_LOCK_PATH]), lockpath, ZOO_ACL_OPEN) return True
def create_async(self, path, value, acl=None, ephemeral=False, sequence=False): """Asynchronously create a ZNode @param path: path of node @param value: initial value of node @param acl: permissions for node @param ephemeral: boolean indicating whether node is ephemeral (tied to this session) @param sequence: boolean indicating whether path is suffixed with a unique index @return: AsyncResult object set on completion with the real path of the new node @rtype AsyncResult """ flags = 0 if ephemeral: flags |= zookeeper.EPHEMERAL if sequence: flags |= zookeeper.SEQUENCE if acl is None: acl = (ZK_OPEN_ACL_UNSAFE,) async_result = self._sync.async_result() callback = partial(_generic_callback, async_result) zookeeper.acreate(self._handle, path, value, list(acl), flags, callback) return async_result
def new_queue(self, form): self.results = None self.results_flag = threading.Event() if zookeeper.OK != zookeeper.acreate(self.server.zk, '/queues/%s' % form['queue'].value, '', [{'perms': 31, 'scheme': 'world', 'id': 'anyone'}], 0, self.post_handler): self.server.log.warning('Could not create new queue %s' % form['queue'].value) self.results_flag.wait() if self.results != zookeeper.OK: self.server.log.critical('Something strange happened: RC = %s' % self.results) return False return True
def __create_node(self, node_name, data, acl, flags): self.create_flag = threading.Event() self.create_results = None self.node_name = None if zookeeper.OK != zookeeper.acreate(self.zk, node_name, data, acl, flags, self.__create_handler): raise zookeeper.ZooKeeperException('Could not create node %s' % node_name) self.create_flag.wait() if self.create_results != zookeeper.OK: raise zookeeper.ZooKeeperException('Could not create node %s, rc = %d' % (node_name, self.create_results)) return self.node_name
def __init__(self, config): HTTPServer.__init__(self, (config.get('HTTP Server','host'), config.getint('HTTP Server','port')), ArkHTTPHandler) self.log = logging.getLogger('arkmq.http') handler = logging.FileHandler(config.get('ArkMQ', 'log') + 'http.log') handler.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')) self.log.addHandler(handler) self.log.setLevel(logging.DEBUG) self.connection_flag = threading.Event() self.connection = None self.zk = zookeeper.init(zkservers, self.connection_handler) self.connection_flag.wait() if self.connection: self.log.info('Connected to zookeeper with handle %s' % self.zk) else: sys.exit(0) for queue in top_level_queues: if zookeeper.exists(self.zk, '/%s' % queue): self.log.debug('Node %s exists' % queue) continue self.create_flag = threading.Event() self.results = None if zookeeper.OK != zookeeper.acreate(self.zk, '/%s' % queue, '', [{'perms': 31, 'scheme': 'world', 'id': 'anyone'}], 0, self.create_handler): self.log.critical('Could not create node %s' % queue) self.create_flag.wait() if self.results != zookeeper.OK: self.log.critical('Could not create node %s' % queue) else: self.log.info('Created node %s' % queue)
def test_async_create(self): self.cv = threading.Condition() def callback(handle, rc, value): self.cv.acquire() self.callback_flag = True self.rc = rc self.cv.notify() self.cv.release() self.assertEqual(self.connected, True, "Not connected!") self.cv.acquire() ret = zookeeper.acreate(self.handle, "/zk-python-acreatetest", "nodecontents", [ZOO_OPEN_ACL_UNSAFE], zookeeper.EPHEMERAL, callback ) self.assertEqual(ret, zookeeper.OK, "acreate failed") while not self.callback_flag: self.cv.wait(15) self.cv.release() self.assertEqual(self.callback_flag, True, "acreate timed out") self.assertEqual(self.rc, zookeeper.OK)
def create(self, path, data="", acls=[ZOO_OPEN_ACL_UNSAFE], flags=0): """ Create a node with the given data and access control. @params path: The path to the node @params data: The node's content @params acls: A list of dictionaries specifying permissions. @params flags: Node creation flags (ephemeral, sequence, persistent) """ if acls == SKIP_ACLS: acls = [ZOO_OPEN_ACL_UNSAFE] d = defer.Deferred() if self._check_connected(d): return d callback = self._zk_thread_callback( self._cb_created, d, data, acls, flags) result = zookeeper.acreate( self.handle, path, data, acls, flags, callback) self._check_result(result, d, path=path) return d
return wrapper def child_path(root, i): return "%s/session_%d" % (root, i) @timed def do_operation(op, s, root, async, count, ephemeral=True, data=None): async_results = [] for i in range(count): path = child_path(root, i) if async: if op == Operation.create: cb = CreateCallback() zookeeper.acreate(s, path, data, acl, zookeeper.EPHEMERAL if ephemeral else 0, cb) elif op == Operation.get: cb = GetCallback() zookeeper.aget(s, path, None, cb) elif op == Operation.set: cb = SetCallback() zookeeper.aset(s, path, data, -1, cb) elif op == Operation.delete: cb = DeleteCallback() zookeeper.adelete(s, path, -1, cb) async_results.append(cb) else: if op == Operation.create: zookeeper.create(s, path, data, acl, zookeeper.EPHEMERAL if ephemeral else 0) elif op == Operation.get:
def set_complete(zh, rc, pv): #print >>sys.stderr, "aset completed: %s" % str(args) if rc == zk.NONODE: # does not exist yet - create anew zk.acreate(zh, NODE_JOB, '', PERM_WORLD)
pass except zookeeper.ZooKeeperException, zoo_exception: logging.error("Exception seen when notifying a failed transaction {0}"\ .format(str(zoo_exception))) return try: if lock_list: # Add the transaction ID to the blacklist. now = str(time.time()) blacklist_root = self.get_blacklist_root_path(app_id) if not zookeeper.exists(self.handle, blacklist_root): self.force_create_path(blacklist_root) zookeeper.acreate(self.handle, PATH_SEPARATOR.join([blacklist_root, str(txid)]), now, ZOO_ACL_OPEN) # Update local cache before notification. if app_id in self.blacklist_cache: with self.blacklist_cv: self.blacklist_cache[app_id].add(str(txid)) # Copy valid transaction ID for each updated key into valid list. for child in zookeeper.get_children(self.handle, txpath): if re.match("^" + TX_UPDATEDKEY_PREFIX, child): value = zookeeper.get(self.handle, PATH_SEPARATOR.join([txpath, child]), None)[0] valuelist = value.split(PATH_SEPARATOR) key = urllib.unquote_plus(valuelist[0]) vid = valuelist[1] vtxroot = self.get_valid_transaction_root_path(app_id)
def acreate(self, path, data='', perm=PERM_WORLD, flags=''): return zk.acreate(self.zh, path, data, perm, create_flags(flags))