def create_async(self,
path,
value,
acl=None,
ephemeral=False,
sequence=False):
"""Asynchronously create a ZNode
@param path: path of node
@param value: initial value of node
@param acl: permissions for node
@param ephemeral: boolean indicating whether node is ephemeral (tied to this session)
@param sequence: boolean indicating whether path is suffixed with a unique index
@return: AsyncResult object set on completion with the real path of the new node
@rtype AsyncResult
"""
flags = 0
if ephemeral:
flags |= zookeeper.EPHEMERAL
if sequence:
flags |= zookeeper.SEQUENCE
if acl is None:
acl = (ZK_OPEN_ACL_UNSAFE, )
async_result = self._sync.async_result()
callback = partial(_generic_callback, async_result)
zookeeper.acreate(self._handle, path, value, list(acl), flags,
callback)
return async_result
def __init__(self, zkhosts, root=NODE_HQ_ROOT, alivenode='alive'):
"""zkhosts: a string or a list. list will be ','.join-ed into a string.
root: root node path (any parents must exist, if any)
"""
if not isinstance(zkhosts, basestring):
zkhosts = ','.join(zkhosts)
self.zkhosts = zkhosts
self.ROOT = root
self.alivenode = alivenode
self.nodename = os.uname()[1]
self.NODE_SERVERS = self.ROOT + '/servers'
self.NODE_ME = self.NODE_SERVERS+'/'+self.nodename
self.zh = zk.init(self.zkhosts, self.__watcher)
self.jobs = {}
self.initialize_node_structure()
if not zk.exists(self.zh, self.NODE_ME):
zk.create(self.zh, self.NODE_ME, '', PERM_WORLD)
# setup notification
zk.get_children(self.zh, self.NODE_SERVERS, self.__servers_watcher)
#zk.get_children(self.zh, self.NODE_ME, self.__watcher)
self.NODE_JOBS = self.NODE_ME+'/jobs'
zk.acreate(self.zh, self.NODE_JOBS, '', PERM_WORLD)
def async_create(self, path, data=None, acl=None, sequence=False, ephemeral=False):
"""Async call to create Zookeeper node.
Args:
path: zookeeper node path, i.e. /my/zookeeper/node/path
data: optional zookeeper node data (string)
acl: optional zookeeper access control list (default is insecure)
sequence: if True node will be created by adding a unique number
the supplied path.
ephemeral: if True, node will automatically be deleted when client exists.
Returns:
Zookeeper.AsyncResult if callback is None, otherwise None.
"""
data = data or ""
acl = acl or self.acl
flags = (zookeeper.SEQUENCE if sequence else 0) | (zookeeper.EPHEMERAL if ephemeral else 0)
async_result = self._async_result()
def callback(handle, return_code, path):
if return_code == zookeeper.OK:
async_result.set(path)
else:
async_result.set_exception(self.error_to_exception(return_code))
zookeeper.acreate(self.handle, path, data, acl, flags, callback)
return async_result
def create_ignore_errors(zh, path, data):
def completion(*args, **kwargs):
rc, zrc, ignored = args
if rc != 0:
print("zookeeper.acreate() error")
elif zrc != 0 and zrc != zookeeper.NODEEXISTS:
print('create/set('+path+') : FAILED')
sem.release()
zookeeper.acreate(zh, path, data, acl_openbar, 0, completion)
def notifyFailedTransaction(self, app_id, txid):
""" Notify failed transaction id.
This method will add the transaction id into black list.
After this call, the transaction becomes invalid.
"""
self.__waitForConnect()
self.checkTransaction(app_id, txid)
print "notify failed transaction app:%s, txid:%d" % (app_id, txid)
txpath = self.__getTransactionPath(app_id, txid)
lockpath = None
try:
lockpath = zookeeper.get(self.handle, PATH_SEPARATOR.join([txpath, TX_LOCK_PATH]), None)[0]
except zookeeper.NoNodeException:
# there is no lock. it means there is no need to rollback.
pass
if lockpath:
# add transacion id to black list.
now = str(time.time())
broot = self.__getBlacklistRootPath(app_id)
if not zookeeper.exists(self.handle, broot):
self.__forceCreatePath(broot)
zookeeper.acreate(self.handle, PATH_SEPARATOR.join([broot, str(txid)]), now, ZOO_ACL_OPEN)
# update local cache before notification
if app_id in self.blacklistcache:
with self.blacklistcv:
self.blacklistcache[app_id].add(str(txid))
# copy valid transaction id for each updated key into valid list.
for child in zookeeper.get_children(self.handle, txpath):
if re.match("^" + TX_UPDATEDKEY_PREFIX, child):
value = zookeeper.get(self.handle, PATH_SEPARATOR.join([txpath, child]), None)[0]
valuelist = value.split(PATH_SEPARATOR)
key = urllib.unquote_plus(valuelist[0])
vid = valuelist[1]
vtxroot = self.__getValidTransactionRootPath(app_id)
if not zookeeper.exists(self.handle, vtxroot):
self.__forceCreatePath(vtxroot)
vtxpath = self.__getValidTransactionPath(app_id, key)
zookeeper.acreate(self.handle, vtxpath, vid, ZOO_ACL_OPEN)
# release the lock
try:
zookeeper.adelete(self.handle, lockpath)
except zookeeper.NoNodeException:
# this should be retry.
pass
# just remove transaction node
try:
for item in zookeeper.get_children(self.handle, txpath):
zookeeper.adelete(self.handle, PATH_SEPARATOR.join([txpath, item]))
zookeeper.adelete(self.handle, txpath)
except zookeeper.NoNodeException:
# something wrong. next GC will take care of it.
return False
return True
def create_async(self, path, value, acl, flags):
async_result = self._new_async_result()
def callback(handle, code, path):
self._queue_result(async_result, code != zookeeper.OK,
err_to_exception(code) if code != zookeeper.OK else path)
zookeeper.acreate(self._handle, path, value, acl, flags, callback)
return async_result
def notifyFailedTransaction(self, app_id, txid):
""" Notify failed transaction id.
This method will add the transaction id into black list.
After this call, the transaction becomes invalid.
"""
self.__waitForConnect()
self.checkTransaction(app_id, txid)
print "notify failed transaction app:%s, txid:%d" % (app_id, txid)
txpath = self.__getTransactionPath(app_id, txid)
lockpath = None
try:
lockpath = zookeeper.get(self.handle, PATH_SEPARATOR.join([txpath, TX_LOCK_PATH]), None)[0]
except zookeeper.NoNodeException:
# there is no lock. it means there is no need to rollback.
pass
if lockpath:
# add transacion id to black list.
now = str(time.time())
broot = self.__getBlacklistRootPath(app_id)
if not zookeeper.exists(self.handle, broot):
self.__forceCreatePath(broot)
zookeeper.acreate(self.handle, PATH_SEPARATOR.join([broot, str(txid)]), now, ZOO_ACL_OPEN)
# update local cache before notification
if app_id in self.blacklistcache:
with self.blacklistcv:
self.blacklistcache[app_id].add(str(txid))
# copy valid transaction id for each updated key into valid list.
for child in zookeeper.get_children(self.handle, txpath):
if re.match("^" + TX_UPDATEDKEY_PREFIX, child):
value = zookeeper.get(self.handle, PATH_SEPARATOR.join([txpath, child]), None)[0]
valuelist = value.split(PATH_SEPARATOR)
key = urllib.unquote_plus(valuelist[0])
vid = valuelist[1]
vtxroot = self.__getValidTransactionRootPath(app_id)
if not zookeeper.exists(self.handle, vtxroot):
self.__forceCreatePath(vtxroot)
vtxpath = self.__getValidTransactionPath(app_id, key)
zookeeper.acreate(self.handle, vtxpath, vid, ZOO_ACL_OPEN)
# release the lock
try:
zookeeper.adelete(self.handle, lockpath)
except zookeeper.NoNodeException:
# this should be retry.
pass
# just remove transaction node
try:
for item in zookeeper.get_children(self.handle, txpath):
zookeeper.adelete(self.handle, PATH_SEPARATOR.join([txpath, item]))
zookeeper.adelete(self.handle, txpath)
except zookeeper.NoNodeException:
# something wrong. next GC will take care of it.
return False
return True
def publish_job(self, job):
'''job: hq.CrawlJob'''
node = self.NODE_JOBS+'/'+job.jobname
zk.acreate(self.zh, node, '', PERM_WORLD)
node = self.NODE_GJOBS+'/'+job.jobname
self.acreate(node)
self.acreate('/'.join((self.NODE_GJOBS, job.jobname, self.nodename)),
flags='e')
def create_async(self, path, value, acl, flags):
async_result = self._new_async_result()
def callback(handle, code, path):
if code != zookeeper.OK:
exc = err_to_exception(code)
async_result.set_exception(exc)
else:
async_result.set(path)
zookeeper.acreate(self._handle, path, value, acl, flags, callback)
return async_result
def _create_ignore_errors(zh, path, data, ctx):
def _completion(ctx, *args, **kwargs):
rc, zrc, ignored = args
if rc != 0:
logging.warn("zookeeper.acreate(%s) error rc=%d", path, rc)
ctx['failed'] += 1
elif zrc != 0 and zrc != zookeeper.NODEEXISTS:
logging.warn('create/set(%s) : FAILED', path)
ctx['failed'] += 1
ctx['sem'].release()
ctx['started'] += 1
zookeeper.acreate(zh, path, data, _acl_openbar, 0,
lambda *a, **ka: _completion(ctx, *a, **ka))
def acquireLock(self, app_id, txid, entity_key = GLOBAL_LOCK_KEY):
""" Acquire lock for transaction.
You must call getTransactionID() first to obtain transaction ID.
You could call this method anytime if the root entity key is same.
Args:
app_id: The application ID to acquire a lock for.
txid: The transaction ID you are acquiring a lock for. Built into
the path.
entity_key: Used to get the root path.
Raises:
ZKTransactionException: If it could not get the lock.
"""
self.__waitForConnect()
txpath = self.__getTransactionPath(app_id, txid)
lockrootpath = self.__getLockRootPath(app_id, entity_key)
lockpath = None
if zookeeper.exists(self.handle, PATH_SEPARATOR.join([txpath, TX_LOCK_PATH])):
# use current lock
prelockpath = zookeeper.get(self.handle, PATH_SEPARATOR.join([txpath, TX_LOCK_PATH]), None)[0]
if not lockrootpath == prelockpath:
raise ZKTransactionException(ZKTransactionException.TYPE_DIFFERENT_ROOTKEY,
"zktransaction.acquireLock: You can not lock different root entity in same transaction.")
print "Already has lock: %s" % lockrootpath
return True
self.checkTransaction(app_id, txid)
# create new lock
retry = True
while retry:
retry = False
try:
lockpath = zookeeper.create(self.handle, lockrootpath, txpath, ZOO_ACL_OPEN)
except zookeeper.NoNodeException:
self.__forceCreatePath(PATH_SEPARATOR.join(lockrootpath.split(PATH_SEPARATOR)[:-1]))
retry = True
except zookeeper.NodeExistsException:
# fail to get lock
raise ZKTransactionException(ZKTransactionException.TYPE_CONCURRENT,
"zktransaction.acquireLock: There is already another transaction using this lock")
# set lockpath for transaction node
# TODO: we should think about atomic operation or recovery of
# inconsistent lockpath node.
zookeeper.acreate(self.handle, PATH_SEPARATOR.join([txpath, TX_LOCK_PATH]), lockpath, ZOO_ACL_OPEN)
return True
def acreate(self, path, data='', perm=PERM_WORLD, flags=''):
try:
return zk.acreate(self.zh, path, data, perm, create_flags(flags))
except SystemError:
# acreate C code often returns error without setting exception,
# which cuases SystemError.
pass
def test_async_create(self):
self.cv = threading.Condition()
def callback(handle, rc, value):
self.cv.acquire()
self.callback_flag = True
self.rc = rc
self.cv.notify()
self.cv.release()
ZOO_OPEN_ACL_UNSAFE = {
"perms": 0x1f,
"scheme": "world",
"id": "anyone"
}
self.assertEqual(self.connected, True, "Not connected!")
self.cv.acquire()
ret = zookeeper.acreate(self.handle, "/zk-python-acreatetest",
"nodecontents", [ZOO_OPEN_ACL_UNSAFE],
zookeeper.EPHEMERAL, callback)
self.assertEqual(ret, zookeeper.OK, "acreate failed")
while not self.callback_flag:
self.cv.wait(15)
self.cv.release()
self.assertEqual(self.callback_flag, True, "acreate timed out")
self.assertEqual(self.rc, zookeeper.OK)
def create_ignore_errors (zh, path, data):
def completion (*args, **kwargs):
rc, zrc, ignored = args
if rc != 0:
print "zookeeper.acreate() error"
else:
if zrc == 0:
#print 'create/set('+path+') : OK'
pass
elif zrc == zookeeper.NODEEXISTS:
#print 'create/set('+path+') : ALREADY'
pass
else:
print 'create/set('+path+') : FAILED'
sem.release()
zookeeper.acreate(zh, path, data, acl_openbar, 0, completion)
def post_to_queue(self, form):
self.results = None
self.node = None
self.results_flag = threading.Event()
if zookeeper.OK != zookeeper.acreate(self.server.zk,
'/queues/%s/%s-' % (form['queue'].value,
form['queue'].value),
form['message'].value,
[{'perms': 31, 'scheme': 'world', 'id': 'anyone'}],
zookeeper.SEQUENCE,
self.post_handler):
self.server.log.warning('Could not post job to queue %s' % form['queue'].value)
self.results_flag.wait()
if self.results == zookeeper.OK:
self.server.log.info('New message posted to queue %s: %s'
% (form['queue'].value, form['message'].value))
self.send_response(200)
self.end_headers()
self.wfile.write(self.node)
elif self.results == zookeeper.NONODE:
self.server.log.info('Creating queue %s' % form['queue'].value)
if self.new_queue(form):
self.post_to_queue(form)
else:
self.server.log.critical('Something strange happened: RC = %s' % self.results)
def create_ignore_errors(zh, path, data):
def completion(*args, **kwargs):
rc, zrc, ignored = args
if rc != 0:
print "zookeeper.acreate() error"
else:
if zrc == 0:
#print 'create/set('+path+') : OK'
pass
elif zrc == zookeeper.NODEEXISTS:
#print 'create/set('+path+') : ALREADY'
pass
else:
print 'create/set(' + path + ') : FAILED'
sem.release()
zookeeper.acreate(zh, path, data, acl_openbar, 0, completion)
def registUpdatedKey(self, app_id, current_txid, target_txid, entity_key):
""" Regist valid transaction id for entity.
target_txid must be the latest valid transaction id for the entity.
"""
self.__waitForConnect()
vtxpath = self.__getValidTransactionPath(app_id, entity_key)
if zookeeper.exists(self.handle, vtxpath):
# just update transaction id for entity if there is valid transaction node.
zookeeper.aset(self.handle, vtxpath, str(target_txid))
else:
# store the updated key info into current transaction node.
value = PATH_SEPARATOR.join([urllib.quote_plus(entity_key), str(target_txid)])
txpath = self.__getTransactionPath(app_id, current_txid)
if zookeeper.exists(self.handle, txpath):
zookeeper.acreate(self.handle, PATH_SEPARATOR.join([txpath, TX_UPDATEDKEY_PREFIX]), value, ZOO_ACL_OPEN, zookeeper.SEQUENCE)
else:
raise ZKTransactionException(ZKTransactionException.TYPE_INVALID, "Transaction %d is not valid." % current_txid)
def registUpdatedKey(self, app_id, current_txid, target_txid, entity_key):
""" Regist valid transaction id for entity.
target_txid must be the latest valid transaction id for the entity.
"""
self.__waitForConnect()
vtxpath = self.__getValidTransactionPath(app_id, entity_key)
if zookeeper.exists(self.handle, vtxpath):
# just update transaction id for entity if there is valid transaction node.
zookeeper.aset(self.handle, vtxpath, str(target_txid))
else:
# store the updated key info into current transaction node.
value = PATH_SEPARATOR.join([urllib.quote_plus(entity_key), str(target_txid)])
txpath = self.__getTransactionPath(app_id, current_txid)
if zookeeper.exists(self.handle, txpath):
zookeeper.acreate(self.handle, PATH_SEPARATOR.join([txpath, TX_UPDATEDKEY_PREFIX]), value, ZOO_ACL_OPEN, zookeeper.SEQUENCE)
else:
raise ZKTransactionException(ZKTransactionException.TYPE_INVALID, "Transaction %d is not valid." % current_txid)
def acreate(self,
path,
callback,
data="",
flags=0,
acl=[ZOO_OPEN_ACL_UNSAFE]):
result = zookeeper.acreate(self.handle, path, data, acl, flags,
callback)
return result
def create(self, path, value, acl=None, flags=0):
"""
create a node synchronously.
This method will create a node in ZooKeeper. A node can only be created
if it does not already exists. The Create Flags affect the creation of
nodes.
If the EPHEMERAL flag is set, the node will automatically get removed
if the client session goes away. If the SEQUENCE flag is set, a unique
monotonically increasing sequence number is appended to the path name.
PARAMETERS:
path: The name of the node. Expressed as a file name with slashes
separating ancestors of the node.
value: The data to be stored in the node.
acl: The initial ACL of the node. If None, the ACL of the parent will
be used.
flags: this parameter can be set to 0 for normal create or an OR
of the Create Flags
The real path that is created (this might be different than the
path to create because of the SEQUENCE flag.
the maximum length of real path you would want.
RETURNS:
The actual znode path that was created (may be different from path due
to use of SEQUENTIAL flag).
EXCEPTIONS:
NONODE the parent node does not exist.
NODEEXISTS the node already exists
NOAUTH the client does not have permission.
NOCHILDRENFOREPHEMERALS cannot create children of ephemeral nodes.
BADARGUMENTS - invalid input parameters
INVALIDSTATE - zhandle state is either SESSION_EXPIRED_STATE or
AUTH_FAILED_STATE
MARSHALLINGERROR - failed to marshall a request; possibly, out of
memory
"""
pc = utils.StatePipeCondition()
ok = zookeeper.acreate(self._zhandle, path, value, acl, flags,
functools.partial(_generic_completion,
pc))
assert ok == zookeeper.OK
results = pc.wait_and_get()
pc.close()
#unpack result as string_completion
handle, rc, real_path = results
assert handle == self._zhandle
if rc == zookeeper.OK:
return real_path
self._raise_exception(rc)
def create(self, path, value, acl=None, flags=0):
"""
create a node synchronously.
This method will create a node in ZooKeeper. A node can only be created if
it does not already exists. The Create Flags affect the creation of nodes.
If the EPHEMERAL flag is set, the node will automatically get removed if the
client session goes away. If the SEQUENCE flag is set, a unique
monotonically increasing sequence number is appended to the path name.
PARAMETERS:
path: The name of the node. Expressed as a file name with slashes
separating ancestors of the node.
value: The data to be stored in the node.
acl: The initial ACL of the node. If None, the ACL of the parent will be
used.
flags: this parameter can be set to 0 for normal create or an OR
of the Create Flags
The real path that is created (this might be different than the
path to create because of the SEQUENCE flag.
the maximum length of real path you would want.
RETURNS:
The actual znode path that was created (may be different from path due
to use of SEQUENTIAL flag).
EXCEPTIONS:
NONODE the parent node does not exist.
NODEEXISTS the node already exists
NOAUTH the client does not have permission.
NOCHILDRENFOREPHEMERALS cannot create children of ephemeral nodes.
BADARGUMENTS - invalid input parameters
INVALIDSTATE - zhandle state is either SESSION_EXPIRED_STATE or
AUTH_FAILED_STATE
MARSHALLINGERROR - failed to marshall a request; possibly, out of
memory
"""
results = []
pc = utils.PipeCondition()
ok = zookeeper.acreate(
self._zhandle, path, value, acl, flags,
functools.partial(generic_completion, pc, results))
assert ok == zookeeper.OK
pc.wait()
#unpack result as string_completion
handle, rc, real_path = results
assert handle == self._zhandle
if rc == zookeeper.OK:
return real_path
self._raise_exception(rc)
def acquireLock(self, app_id, txid, entity_key = GLOBAL_LOCK_KEY):
""" Acquire lock for transaction.
You must call getTransactionID() first to obtain transaction ID.
You could call this method anytime if the root entity key is same.
If you could not get lock, this returns False.
"""
self.__waitForConnect()
txpath = self.__getTransactionPath(app_id, txid)
lockrootpath = self.__getLockRootPath(app_id, entity_key)
lockpath = None
if zookeeper.exists(self.handle, PATH_SEPARATOR.join([txpath, TX_LOCK_PATH])):
# use current lock
prelockpath = zookeeper.get(self.handle, PATH_SEPARATOR.join([txpath, TX_LOCK_PATH]), None)[0]
if not lockrootpath == prelockpath:
raise ZKTransactionException(ZKTransactionException.TYPE_DIFFERENT_ROOTKEY, "You can not lock different root entity in same transaction.")
print "already has lock: %s" % lockrootpath
return True
self.checkTransaction(app_id, txid)
# create new lock
retry = True
while retry:
retry = False
try:
lockpath = zookeeper.create(self.handle, lockrootpath, txpath, ZOO_ACL_OPEN)
except zookeeper.NoNodeException:
self.__forceCreatePath(PATH_SEPARATOR.join(lockrootpath.split(PATH_SEPARATOR)[:-1]))
retry = True
except zookeeper.NodeExistsException:
# fail to get lock
raise ZKTransactionException(ZKTransactionException.TYPE_CONCURRENT, "There is already another transaction using this lock")
# set lockpath for transaction node
# TODO: we should think about atomic operation or recovery of
# inconsistent lockpath node.
zookeeper.acreate(self.handle, PATH_SEPARATOR.join([txpath, TX_LOCK_PATH]), lockpath, ZOO_ACL_OPEN)
return True
def create_async(self, path, value, acl=None, ephemeral=False, sequence=False):
"""Asynchronously create a ZNode
@param path: path of node
@param value: initial value of node
@param acl: permissions for node
@param ephemeral: boolean indicating whether node is ephemeral (tied to this session)
@param sequence: boolean indicating whether path is suffixed with a unique index
@return: AsyncResult object set on completion with the real path of the new node
@rtype AsyncResult
"""
flags = 0
if ephemeral:
flags |= zookeeper.EPHEMERAL
if sequence:
flags |= zookeeper.SEQUENCE
if acl is None:
acl = (ZK_OPEN_ACL_UNSAFE,)
async_result = self._sync.async_result()
callback = partial(_generic_callback, async_result)
zookeeper.acreate(self._handle, path, value, list(acl), flags, callback)
return async_result
def new_queue(self, form):
self.results = None
self.results_flag = threading.Event()
if zookeeper.OK != zookeeper.acreate(self.server.zk,
'/queues/%s' % form['queue'].value,
'',
[{'perms': 31, 'scheme': 'world', 'id': 'anyone'}],
0,
self.post_handler):
self.server.log.warning('Could not create new queue %s' % form['queue'].value)
self.results_flag.wait()
if self.results != zookeeper.OK:
self.server.log.critical('Something strange happened: RC = %s' % self.results)
return False
return True
def __create_node(self, node_name, data, acl, flags):
self.create_flag = threading.Event()
self.create_results = None
self.node_name = None
if zookeeper.OK != zookeeper.acreate(self.zk,
node_name,
data,
acl,
flags,
self.__create_handler):
raise zookeeper.ZooKeeperException('Could not create node %s' % node_name)
self.create_flag.wait()
if self.create_results != zookeeper.OK:
raise zookeeper.ZooKeeperException('Could not create node %s, rc = %d' % (node_name, self.create_results))
return self.node_name
def __init__(self, config):
HTTPServer.__init__(self,
(config.get('HTTP Server','host'), config.getint('HTTP Server','port')),
ArkHTTPHandler)
self.log = logging.getLogger('arkmq.http')
handler = logging.FileHandler(config.get('ArkMQ', 'log') + 'http.log')
handler.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(message)s'))
self.log.addHandler(handler)
self.log.setLevel(logging.DEBUG)
self.connection_flag = threading.Event()
self.connection = None
self.zk = zookeeper.init(zkservers, self.connection_handler)
self.connection_flag.wait()
if self.connection:
self.log.info('Connected to zookeeper with handle %s' % self.zk)
else:
sys.exit(0)
for queue in top_level_queues:
if zookeeper.exists(self.zk, '/%s' % queue):
self.log.debug('Node %s exists' % queue)
continue
self.create_flag = threading.Event()
self.results = None
if zookeeper.OK != zookeeper.acreate(self.zk,
'/%s' % queue,
'',
[{'perms': 31, 'scheme': 'world', 'id': 'anyone'}],
0,
self.create_handler):
self.log.critical('Could not create node %s' % queue)
self.create_flag.wait()
if self.results != zookeeper.OK:
self.log.critical('Could not create node %s' % queue)
else:
self.log.info('Created node %s' % queue)
def test_async_create(self):
self.cv = threading.Condition()
def callback(handle, rc, value):
self.cv.acquire()
self.callback_flag = True
self.rc = rc
self.cv.notify()
self.cv.release()
self.assertEqual(self.connected, True, "Not connected!")
self.cv.acquire()
ret = zookeeper.acreate(self.handle, "/zk-python-acreatetest", "nodecontents",
[ZOO_OPEN_ACL_UNSAFE], zookeeper.EPHEMERAL,
callback )
self.assertEqual(ret, zookeeper.OK, "acreate failed")
while not self.callback_flag:
self.cv.wait(15)
self.cv.release()
self.assertEqual(self.callback_flag, True, "acreate timed out")
self.assertEqual(self.rc, zookeeper.OK)
def create(self, path, data="", acls=[ZOO_OPEN_ACL_UNSAFE], flags=0):
"""
Create a node with the given data and access control.
@params path: The path to the node
@params data: The node's content
@params acls: A list of dictionaries specifying permissions.
@params flags: Node creation flags (ephemeral, sequence, persistent)
"""
if acls == SKIP_ACLS:
acls = [ZOO_OPEN_ACL_UNSAFE]
d = defer.Deferred()
if self._check_connected(d):
return d
callback = self._zk_thread_callback(
self._cb_created, d, data, acls, flags)
result = zookeeper.acreate(
self.handle, path, data, acls, flags, callback)
self._check_result(result, d, path=path)
return d
def acreate(self, path, callback, data="", flags=0, acl=[ZOO_OPEN_ACL_UNSAFE]):
result = zookeeper.acreate(self.handle, path, data, acl, flags, callback)
return result
return wrapper
def child_path(root, i):
return "%s/session_%d" % (root, i)
@timed
def do_operation(op, s, root, async, count, ephemeral=True, data=None):
async_results = []
for i in range(count):
path = child_path(root, i)
if async:
if op == Operation.create:
cb = CreateCallback()
zookeeper.acreate(s, path, data, acl,
zookeeper.EPHEMERAL if ephemeral else 0, cb)
elif op == Operation.get:
cb = GetCallback()
zookeeper.aget(s, path, None, cb)
elif op == Operation.set:
cb = SetCallback()
zookeeper.aset(s, path, data, -1, cb)
elif op == Operation.delete:
cb = DeleteCallback()
zookeeper.adelete(s, path, -1, cb)
async_results.append(cb)
else:
if op == Operation.create:
zookeeper.create(s, path, data, acl,
zookeeper.EPHEMERAL if ephemeral else 0)
elif op == Operation.get:
def set_complete(zh, rc, pv):
#print >>sys.stderr, "aset completed: %s" % str(args)
if rc == zk.NONODE:
# does not exist yet - create anew
zk.acreate(zh, NODE_JOB, '', PERM_WORLD)
pass
except zookeeper.ZooKeeperException, zoo_exception:
logging.error("Exception seen when notifying a failed transaction {0}"\
.format(str(zoo_exception)))
return
try:
if lock_list:
# Add the transaction ID to the blacklist.
now = str(time.time())
blacklist_root = self.get_blacklist_root_path(app_id)
if not zookeeper.exists(self.handle, blacklist_root):
self.force_create_path(blacklist_root)
zookeeper.acreate(self.handle, PATH_SEPARATOR.join([blacklist_root,
str(txid)]), now, ZOO_ACL_OPEN)
# Update local cache before notification.
if app_id in self.blacklist_cache:
with self.blacklist_cv:
self.blacklist_cache[app_id].add(str(txid))
# Copy valid transaction ID for each updated key into valid list.
for child in zookeeper.get_children(self.handle, txpath):
if re.match("^" + TX_UPDATEDKEY_PREFIX, child):
value = zookeeper.get(self.handle, PATH_SEPARATOR.join([txpath,
child]), None)[0]
valuelist = value.split(PATH_SEPARATOR)
key = urllib.unquote_plus(valuelist[0])
vid = valuelist[1]
vtxroot = self.get_valid_transaction_root_path(app_id)
def acreate(self, path, data='', perm=PERM_WORLD, flags=''):
return zk.acreate(self.zh, path, data, perm, create_flags(flags))
return wrapper
def child_path(root, i):
return "%s/session_%d" % (root, i)
@timed
def do_operation(op, s, root, async, count, ephemeral=True, data=None):
async_results = []
for i in range(count):
path = child_path(root, i)
if async:
if op == Operation.create:
cb = CreateCallback()
zookeeper.acreate(s, path, data, acl,
zookeeper.EPHEMERAL if ephemeral else 0, cb)
elif op == Operation.get:
cb = GetCallback()
zookeeper.aget(s, path, None, cb)
elif op == Operation.set:
cb = SetCallback()
zookeeper.aset(s, path, data, -1, cb)
elif op == Operation.delete:
cb = DeleteCallback()
zookeeper.adelete(s, path, -1, cb)
async_results.append(cb)
else:
if op == Operation.create:
zookeeper.create(s, path, data, acl,
zookeeper.EPHEMERAL if ephemeral else 0)
elif op == Operation.get:
