def async_set_data(self, path, data, version=None, callback=None):
"""Async call to set zookeeper node's data.
Args:
path: zookeeper node path
data: zookeeper node data (string)
version: expected node version
callback: callback method to be invoked upon operation completion with
zookeeper api handle, return_code, data, and stat.
callback will be invoked in the context of the underlying
zookeeper API thread.
Returns:
Zookeeper.AsyncResult if callback is None, otherwise None.
"""
async_result = None
version = version if version is not None else -1
if callback is None:
async_result = self.AsyncResult()
def async_callback(handle, return_code, stat):
if return_code == zookeeper.OK:
async_result.set(stat)
else:
async_result.set_exception(
self.error_to_exception(return_code))
callback = async_callback
zookeeper.aset(self.handle, path, data, version, callback)
return async_result
def async_set_data(self, path, data, version=None, callback=None):
"""Async call to set zookeeper node's data.
Args:
path: zookeeper node path
data: zookeeper node data (string)
version: expected node version
callback: callback method to be invoked upon operation completion with
zookeeper api handle, return_code, data, and stat.
callback will be invoked in the context of the underlying
zookeeper API thread.
Returns:
Zookeeper.AsyncResult if callback is None, otherwise None.
"""
async_result = None
version = version if version is not None else -1
if callback is None:
async_result = self.AsyncResult()
def async_callback(handle, return_code, stat):
if return_code == zookeeper.OK:
async_result.set(stat)
else:
async_result.set_exception(self.error_to_exception(return_code))
callback = async_callback
zookeeper.aset(self.handle, path, data, version, callback)
return async_result
def set_async(self, path, data, version=-1):
async_result = self._new_async_result()
def callback(handle, code, stat):
self._queue_result(async_result, code != zookeeper.OK,
err_to_exception(code) if code != zookeeper.OK else stat)
zookeeper.aset(self._handle, path, data, version, callback)
return async_result
def set_async(self, path, data, version=-1):
async_result = self._new_async_result()
def callback(handle, code, stat):
if code != zookeeper.OK:
exc = err_to_exception(code)
async_result.set_exception(exc)
else:
async_result.set(stat)
zookeeper.aset(self._handle, path, data, version, callback)
return async_result
def set_async(self, path, data, version=-1):
"""Set the value of a node
If the version of the node being updated is newer than the supplied
version (and the supplied version is not -1), a BadVersionException
will be raised.
@param path: path of node to set
@param data: new data value
@param version: version of node being updated, or -1
@return: AsyncResult set with new node stat on success
@rtype AsyncResult
"""
async_result = self._sync.async_result()
callback = partial(_generic_callback, async_result)
zookeeper.aset(self._handle, path, data, version, callback)
return async_result
def registUpdatedKey(self, app_id, current_txid, target_txid, entity_key):
""" Regist valid transaction id for entity.
target_txid must be the latest valid transaction id for the entity.
"""
self.__waitForConnect()
vtxpath = self.__getValidTransactionPath(app_id, entity_key)
if zookeeper.exists(self.handle, vtxpath):
# just update transaction id for entity if there is valid transaction node.
zookeeper.aset(self.handle, vtxpath, str(target_txid))
else:
# store the updated key info into current transaction node.
value = PATH_SEPARATOR.join([urllib.quote_plus(entity_key), str(target_txid)])
txpath = self.__getTransactionPath(app_id, current_txid)
if zookeeper.exists(self.handle, txpath):
zookeeper.acreate(self.handle, PATH_SEPARATOR.join([txpath, TX_UPDATEDKEY_PREFIX]), value, ZOO_ACL_OPEN, zookeeper.SEQUENCE)
else:
raise ZKTransactionException(ZKTransactionException.TYPE_INVALID, "Transaction %d is not valid." % current_txid)
def registUpdatedKey(self, app_id, current_txid, target_txid, entity_key):
""" Regist valid transaction id for entity.
target_txid must be the latest valid transaction id for the entity.
"""
self.__waitForConnect()
vtxpath = self.__getValidTransactionPath(app_id, entity_key)
if zookeeper.exists(self.handle, vtxpath):
# just update transaction id for entity if there is valid transaction node.
zookeeper.aset(self.handle, vtxpath, str(target_txid))
else:
# store the updated key info into current transaction node.
value = PATH_SEPARATOR.join([urllib.quote_plus(entity_key), str(target_txid)])
txpath = self.__getTransactionPath(app_id, current_txid)
if zookeeper.exists(self.handle, txpath):
zookeeper.acreate(self.handle, PATH_SEPARATOR.join([txpath, TX_UPDATEDKEY_PREFIX]), value, ZOO_ACL_OPEN, zookeeper.SEQUENCE)
else:
raise ZKTransactionException(ZKTransactionException.TYPE_INVALID, "Transaction %d is not valid." % current_txid)
def set_async(self, path, data, version=-1):
"""Set the value of a node
If the version of the node being updated is newer than the supplied
version (and the supplied version is not -1), a BadVersionException
will be raised.
@param path: path of node to set
@param data: new data value
@param version: version of node being updated, or -1
@return: AsyncResult set with new node stat on success
@rtype AsyncResult
"""
async_result = self._sync.async_result()
callback = partial(_generic_callback, async_result)
zookeeper.aset(self._handle, path, data, version, callback)
return async_result
def __forceCreatePath(self, path, value = "default"):
""" Try to create path first.
If the parent doesn't exists, this method will try to create it.
"""
self.__waitForConnect()
retry = True
while retry:
retry = False
try:
zookeeper.create(self.handle, path, str(value), ZOO_ACL_OPEN)
except zookeeper.NoNodeException:
retry = self.__forceCreatePath(PATH_SEPARATOR.join(path.split(PATH_SEPARATOR)[:-1]), value)
except zookeeper.NodeExistsException:
# just update value
zookeeper.aset(self.handle, path, str(value))
except Exception as e:
print "warning: create error %s" % e
return False
return True
def __forceCreatePath(self, path, value = "default"):
""" Try to create path first.
If the parent doesn't exists, this method will try to create it.
"""
self.__waitForConnect()
retry = True
while retry:
retry = False
try:
zookeeper.create(self.handle, path, str(value), ZOO_ACL_OPEN)
except zookeeper.NoNodeException:
retry = self.__forceCreatePath(PATH_SEPARATOR.join(path.split(PATH_SEPARATOR)[:-1]), value)
except zookeeper.NodeExistsException:
# just update value
zookeeper.aset(self.handle, path, str(value))
except Exception as e:
print "warning: create error %s" % e
return False
return True
def publish_job(self, job):
'''job: hq.CrawlJob'''
ju = self.jobs.get(job)
# update 10 minutes interval
if ju is None or ju < time.time() - 10*60:
NODE_JOB = self.NODE_JOBS+'/'+job.jobname
def set_complete(zh, rc, pv):
#print >>sys.stderr, "aset completed: %s" % str(args)
if rc == zk.NONODE:
# does not exist yet - create anew
zk.acreate(zh, NODE_JOB, '', PERM_WORLD)
try:
zk.aset(self.zh, NODE_JOB, '', -1, set_complete)
except:
logging.warn('aset failed', exc_info=1)
pass
node2 = self.NODE_GJOBS+'/'+job.jobname
self.acreate(node2)
self.acreate('/'.join((self.NODE_GJOBS, job.jobname, self.nodename)),
flags='e')
self.jobs[job] = time.time()
def force_create_path(self, path, value="default"):
""" Creates a new ZooKeeper node at the given path, recursively creating its
parent nodes if necessary.
Args:
path: A PATH_SEPARATOR-separated str that represents the path to create.
value: A str representing the value that should be associated with the
new node, created at path.
Returns:
True if the new ZooKeeper node at path was created successfully, and False
otherwise.
"""
logging.debug("Force creating path {0}, with value {1}".format(path, value))
self.wait_for_connect()
while True:
try:
zookeeper.create(self.handle, path, str(value), ZOO_ACL_OPEN)
return True
except zookeeper.NoNodeException:
# Recursively create this node's parents, but don't return - the next
# iteration of this loop will create this node once the parents are
# created.
logging.debug("Couldn't create path {0}, so creating its parent " \
"first.".format(path))
self.force_create_path(PATH_SEPARATOR.join(
path.split(PATH_SEPARATOR)[:-1]), value)
except zookeeper.NodeExistsException: # just update value
logging.debug("Path {0} already exists, so setting its value.".format(
path))
zookeeper.aset(self.handle, path, str(value))
return True
except zookeeper.ZooKeeperException, zoo_exception:
logging.error("Couldn't create path {0} because of ZK exception {1}".\
format(path, str(zoo_exception)))
return False
except Exception as exception:
logging.error("Couldn't create path {0} because of {1}".format(path,
str(exception)))
return False
def async_set_data(self, path, data, version=None):
"""Async call to set zookeeper node's data.
Args:
path: zookeeper node path
data: zookeeper node data (string)
version: expected node version
Returns:
Zookeeper.AsyncResult if callback is None, otherwise None.
"""
version = version if version is not None else -1
async_result = self._async_result()
def callback(handle, return_code, stat):
if return_code == zookeeper.OK:
async_result.set(stat)
else:
async_result.set_exception(self.error_to_exception(return_code))
zookeeper.aset(self.handle, path, data, version, callback)
return async_result
def set(self, path, data="", version=-1):
"""
Sets the data of a node at the given path. If the current node version
on the server is more recent than that supplied by the client, a bad
version exception wil be thrown. A version of -1 (default) specifies
any version.
@param path: The path of the node whose data we will set.
@param data: The data to store on the node.
@param version: Integer version value
"""
d = defer.Deferred()
if self._check_connected(d):
return d
callback = self._zk_thread_callback(self._cb_set, d, path, data)
result = zookeeper.aset(self.handle, path, data, version, callback)
self._check_result(result, d, path=path)
return d
def test_async_getset(self):
self.cv = threading.Condition()
def get_callback(handle, rc, value, stat):
self.cv.acquire()
self.callback_flag = True
self.rc = rc
self.value = (value, stat)
self.cv.notify()
self.cv.release()
def set_callback(handle, rc, stat):
self.cv.acquire()
self.callback_flag = True
self.rc = rc
self.value = stat
self.cv.notify()
self.cv.release()
self.assertEqual(self.connected, True, "Not connected!")
self.cv.acquire()
self.callback_flag = False
ret = zookeeper.aset(self.handle, "/zk-python-agetsettest", "off", -1,
set_callback)
self.assertEqual(ret, zookeeper.OK, "aset failed")
while not self.callback_flag:
self.cv.wait(15)
self.cv.release()
self.assertEqual(self.callback_flag, True, "aset timed out")
self.cv.acquire()
self.callback_flag = False
ret = zookeeper.aget(self.handle, "/zk-python-agetsettest", None,
get_callback)
self.assertEqual(ret, zookeeper.OK, "aget failed")
self.cv.wait(15)
self.cv.release()
self.assertEqual(self.callback_flag, True, "aget timed out")
self.assertEqual(self.value[0], "off",
"Data is not 'off' as expected: " + self.value[0])
def set2(self, path, data, version=-1):
"""
sets the data associated with a node.
PARAMETERS:
path: the name of the node. Expressed as a file name with slashes
separating ancestors of the node.
data: the buffer holding data to be written to the node.
(subsequent parameters are optional)
version: the expected version of the node. The function will fail if
the actual version of the node does not match the expected version.
If -1 is used the version check will not take place.
RETURNS:
the new stat of the node.
EXCEPTIONS:
NONODE the node does not exist.
NOAUTH the client does not have permission.
BADVERSION expected version does not match actual version.
BADARGUMENTS - invalid input parameters
INVALIDSTATE - zhandle state is either SESSION_EXPIRED_STATE or
AUTH_FAILED_STATE
MARSHALLINGERROR - failed to marshall a request; possibly, out of
memory
"""
pc = utils.StatePipeCondition()
ok = zookeeper.aset(self._zhandle, path, data, version,
functools.partial(_generic_completion,
pc))
assert ok == zookeeper.OK
results = pc.wait_and_get()
pc.close()
#unpack result as stat_completion
handle, rc, stat = results
assert handle == self._zhandle
if rc == zookeeper.OK:
return stat
self._raise_exception(rc)
def set2(self, path, data, version=-1):
"""
sets the data associated with a node.
PARAMETERS:
path: the name of the node. Expressed as a file name with slashes
separating ancestors of the node.
data: the buffer holding data to be written to the node.
(subsequent parameters are optional)
version: the expected version of the node. The function will fail if
the actual version of the node does not match the expected version.
If -1 is used the version check will not take place.
RETURNS:
the new stat of the node.
EXCEPTIONS:
NONODE the node does not exist.
NOAUTH the client does not have permission.
BADVERSION expected version does not match actual version.
BADARGUMENTS - invalid input parameters
INVALIDSTATE - zhandle state is either SESSION_EXPIRED_STATE or
AUTH_FAILED_STATE
MARSHALLINGERROR - failed to marshall a request; possibly, out of
memory
"""
results = []
pc = utils.PipeCondition()
ok = zookeeper.aset(self._zhandle, path, data, version,
functools.partial(generic_completion, pc, results))
assert ok == zookeeper.OK
pc.wait()
#unpack result as stat_completion
handle, rc, stat = results
assert handle == self._zhandle
if rc == zookeeper.OK:
return stat
self._raise_exception(rc)
def test_async_getset(self):
self.cv = threading.Condition()
def get_callback(handle, rc, value, stat):
self.cv.acquire()
self.callback_flag = True
self.rc = rc
self.value = (value,stat)
self.cv.notify()
self.cv.release()
def set_callback(handle, rc, stat):
self.cv.acquire()
self.callback_flag = True
self.rc = rc
self.value = stat
self.cv.notify()
self.cv.release()
self.assertEqual(self.connected, True, "Not connected!")
self.cv.acquire()
self.callback_flag = False
ret = zookeeper.aset(self.handle, "/zk-python-agetsettest", "off", -1, set_callback)
self.assertEqual(ret, zookeeper.OK, "aset failed")
while not self.callback_flag:
self.cv.wait(15)
self.cv.release()
self.assertEqual(self.callback_flag, True, "aset timed out")
self.cv.acquire()
self.callback_flag = False
ret = zookeeper.aget(self.handle, "/zk-python-agetsettest", None, get_callback)
self.assertEqual(ret, zookeeper.OK, "aget failed")
while not self.callback_flag:
self.cv.wait(15)
self.cv.release()
self.assertEqual(self.callback_flag, True, "aget timed out")
self.assertEqual(self.value[0], "off", "Data is not 'off' as expected: " + self.value[0])
def aset(self, path, callback, data="", version=-1):
return zookeeper.aset(self.handle, path, data, version, callback)
def aset(self, path, callback, data="", version=-1):
return zookeeper.aset(self.handle, path, data, version, callback)
@timed
def do_operation(op, s, root, async, count, ephemeral=True, data=None):
async_results = []
for i in range(count):
path = child_path(root, i)
if async:
if op == Operation.create:
cb = CreateCallback()
zookeeper.acreate(s, path, data, acl,
zookeeper.EPHEMERAL if ephemeral else 0, cb)
elif op == Operation.get:
cb = GetCallback()
zookeeper.aget(s, path, None, cb)
elif op == Operation.set:
cb = SetCallback()
zookeeper.aset(s, path, data, -1, cb)
elif op == Operation.delete:
cb = DeleteCallback()
zookeeper.adelete(s, path, -1, cb)
async_results.append(cb)
else:
if op == Operation.create:
zookeeper.create(s, path, data, acl,
zookeeper.EPHEMERAL if ephemeral else 0)
elif op == Operation.get:
zookeeper.get(s, path)
elif op == Operation.set:
zookeeper.set(s, path, data)
elif op == Operation.delete:
zookeeper.delete(s, path)
@timed
def do_operation(op, s, root, async, count, ephemeral=True, data=None):
async_results = []
for i in range(count):
path = child_path(root, i)
if async:
if op == Operation.create:
cb = CreateCallback()
zookeeper.acreate(s, path, data, acl,
zookeeper.EPHEMERAL if ephemeral else 0, cb)
elif op == Operation.get:
cb = GetCallback()
zookeeper.aget(s, path, None, cb)
elif op == Operation.set:
cb = SetCallback()
zookeeper.aset(s, path, data, -1, cb)
elif op == Operation.delete:
cb = DeleteCallback()
zookeeper.adelete(s, path, -1, cb)
async_results.append(cb)
else:
if op == Operation.create:
zookeeper.create(s, path, data, acl,
zookeeper.EPHEMERAL if ephemeral else 0)
elif op == Operation.get:
zookeeper.get(s, path)
elif op == Operation.set:
zookeeper.set(s, path, data)
elif op == Operation.delete:
zookeeper.delete(s, path)