def __tryGC(self, app_id, app_path):
try:
# print "try to obtrain app %s lock" % app_id
val = zookeeper.get(self.handle, PATH_SEPARATOR.join([app_path, GC_TIME_PATH]), None)[0]
lasttime = float(val)
except zookeeper.NoNodeException:
lasttime = 0
if lasttime + GC_INTERVAL < time.time():
# try to gc this app.
gc_path = PATH_SEPARATOR.join([app_path, GC_LOCK_PATH])
try:
now = str(time.time())
zookeeper.create(self.handle, gc_path, now, ZOO_ACL_OPEN, zookeeper.EPHEMERAL)
# succeed to obtain lock.
# TODO: should we handle the timeout of gc also?
try:
self.__executeGC(app_id, app_path)
# update lasttime when gc was succeeded.
now = str(time.time())
self.__updateNode(PATH_SEPARATOR.join([app_path, GC_TIME_PATH]), now)
except Exception, e:
print "warning: gc error %s" % e
traceback.print_exc()
zookeeper.delete(self.handle, gc_path, -1)
except zookeeper.NodeExistsException:
# fail to obtain lock. try next time.
pass
def setUp(self):
zktestbase.TestBase.setUp(self)
try:
zookeeper.delete(self.handle, "/zk-python-acltest")
zookeeper.delete(self.handle, "/zk-python-aacltest")
except:
pass
def __init__(self,queuename, port, is_producer=False):
self.connected = False
self.queuename = "/" + queuename
self.cv = threading.Condition()
zookeeper.set_log_stream(open("/dev/null"))
def watcher(handle,type,state,path):
print "Connected"
self.cv.acquire()
self.connected = True
self.cv.notify()
self.cv.release()
self.cv.acquire()
self.handle = zookeeper.init("localhost:%d" % port, watcher, 10000)
self.cv.wait(10.0)
if not self.connected:
print "Connection to ZooKeeper cluster timed out - is a server running on localhost:%d?" % port
sys.exit()
self.cv.release()
if is_producer:
while True:
try:
zookeeper.create(self.handle,self.queuename,"queue top level", [ZOO_OPEN_ACL_UNSAFE],0)
print "Fila criada, OK"
return
except zookeeper.NodeExistsException:
print "Tratorando filas existentes"
while True:
children = sorted(zookeeper.get_children(self.handle, self.queuename,None))
if len(children) == 0:
(data,stat) = zookeeper.get(self.handle, self.queuename, None)
zookeeper.delete(self.handle, self.queuename, stat["version"])
break
for child in children:
data = self.get_and_delete(self.queuename + "/" + child)
def remove_machine(self, machine):
try:
zookeeper.delete(self._z,
machine)
return True
except zookeeper.NoNodeException:
return False
def post(self):
request_dict = self.request.arguments
node_key = (request_dict['node_key'])[0]
cluster_name = (request_dict['cluster_name'])[0]
zk=zookeeper.init(self.zk_connect(cluster_name))
data = []
def get_node_tree(node_key):
if node_key == "/":
for node in zookeeper.get_children(zk,node_key):
key = "/" + node
if (zookeeper.get(zk,key)[1])['numChildren'] > 0:
get_node_tree(key)
print key
else:
print key
else:
for node in zookeeper.get_children(zk,node_key):
key = node_key + "/" + node
if (zookeeper.get(zk,key)[1])['numChildren'] > 0:
get_node_tree(key)
data.append(key)
else:
data.append(key)
return data
get_node_tree(node_key)
data.append(node_key)
for items in data:
zookeeper.delete(zk,items)
zookeeper.close(zk)
self.write("删除成功")
def unlock(self):
try:
(data, _) = zookeeper.get(self.handle, self.lockname, None)
if data == self.uuid:
zookeeper.delete(self.handle, self.lockname)
except:
self.log.exception('unlock')
def delete_tree (zh, path):
path = path.replace('//', '/')
try:
for n in tuple(zookeeper.get_children(zh, path)):
delete_tree(zh, path + '/' + n)
zookeeper.delete(zh, path)
except:
pass
def delete_recursive(handle, path):
try:
children = zookeeper.get_children(handle, path)
for child in children:
delete_recursive(handle, PATH_SEPARATOR.join([path, child]))
zookeeper.delete(handle, path, -1)
except zookeeper.NoNodeException:
pass
def deleteRecursive(self, path):
self.__waitForConnect()
try:
children = zookeeper.get_children(self.handle, path)
for child in children:
self.deleteRecursive(PATH_SEPARATOR.join([path, child]))
zookeeper.delete(self.handle, path, -1)
except zookeeper.NoNodeException:
pass
def remove_node(self, node_id):
"""Remove a node from the cluster."""
path = self.MEMBERSHIP_NODE + "/" + str(node_id)
try:
zookeeper.delete(self.handle, path)
self.logger.info("Node %d is removed" % node_id)
except zookeeper.NoNodeException:
self.logger.warn("%s does not exist" % path)
def get_and_delete(self,node):
try:
(data,stat) = zookeeper.get(self.handle, node, None)
zookeeper.delete(self.handle, node, stat["version"])
return data
except IOError, e:
if e.message == zookeeper.zerror(zookeeper.NONODE):
return None
raise e
def write(self, path, contents, ephemeral=False, exclusive=False):
"""
Writes the contents to the path in zookeeper. It will create the path in
zookeeper if it does not already exist.
This method will return True if the value is written, False otherwise.
(The value will not be written if the exclusive is True and the node
already exists.)
"""
partial_path = ''
# We start from the second element because we do not want to inclued
# the initial empty string before the first "/" because all paths begin
# with "/". We also don't want to include the final node because that
# is dealt with later.
for path_part in path.split("/")[1:-1]:
partial_path = partial_path + "/" + path_part
if not(zookeeper.exists(self.handle, partial_path)):
try:
zookeeper.create(self.handle, partial_path, '', [self.acl], 0)
except zookeeper.NodeExistsException:
pass
exists = zookeeper.exists(self.handle, path)
# Don't create it if we're exclusive.
if exists and exclusive:
return False
# We make sure that we have the creation flags for ephemeral nodes,
# otherwise they could be associated with previous connections that
# have not yet timed out.
if ephemeral and exists:
try:
zookeeper.delete(self.handle, path)
except zookeeper.NoNodeException:
pass
exists = False
if exists:
zookeeper.set(self.handle, path, contents)
return True
else:
flags = (ephemeral and zookeeper.EPHEMERAL or 0)
try:
zookeeper.create(self.handle, path, contents, [self.acl], flags)
return True
except zookeeper.NodeExistsException:
if not(exclusive):
# Woah, something happened between the top and here.
# We just restart and retry the whole routine.
self.write(path, contents, ephemeral=ephemeral)
return True
else:
return False
def mark_node_unavailable(self, node_id):
"""Mark a node unavailable."""
path = self.AVAILABILITY_NODE + "/" + str(node_id)
try:
data = zookeeper.get(self.handle, path)[0]
zookeeper.delete(self.handle, path)
self.logger.info("Node %d: %s is now unavailable" % (node_id, data))
except zookeeper.NoNodeException:
self.logger.warn("Tried to mark node %s unavailable, but it did not exist" % path)
def deleteNode(self, zpath):
#Delete a node recursively until root node
while zpath != self.root[:-1]:
try:
zookeeper.delete(self.zh, zpath)
except zookeeper.NoNodeException:
pass
except zookeeper.NotEmptyException:
break
finally:
zpath = zpath[:-1].rsplit('/', 1)[0]
def reset(self):
"""Reset both MEMBERSHIP_NODE and AVAILABILITY_NODE to empty nodes."""
nodes = zookeeper.get_children(self.handle, self.MEMBERSHIP_NODE)
for node_id in nodes:
path = self.MEMBERSHIP_NODE + "/" + node_id
try:
zookeeper.delete(self.handle, path)
self.logger.info("Node %s is removed (because of reset)" % node_id)
except zookeeper.NoNodeException:
self.logger.warn("%s does not exist" % path)
def delete(self, path, version=-1, recursive=False):
"""
删除节点，可以制定特定版本的节点，默认是-1，即删除所有版本节点
"""
if recursive:
for child in self.get_children(path):
self.delete(path=os.path.join(path, child), recursive=True)
zookeeper.delete(self.handle, path, version)
if not self.exists(path):
return 1
else:
return 0
def clean_tree(zh, root):
"""Recursively removes the zookeeper subtree underneath :root given zookeeper handle :zh."""
try:
if not zookeeper.exists(zh, root):
return True
for child in zookeeper.get_children(zh, root):
if not ZookeeperUtil.clean_tree(zh, posixpath.join(root, child)):
return False
zookeeper.delete(zh, root)
except zookeeper.ZooKeeperException as e:
return False
return True
def deleteTree(self, path="/", handle=1):
"""
Destroy all the nodes in zookeeper path
"""
for child in zookeeper.get_children(handle, path):
if child == "zookeeper": # skip the metadata node
continue
child_path = "/" + ("%s/%s" % (path, child)).strip("/")
try:
self.deleteTree(child_path, handle)
zookeeper.delete(handle, child_path, -1)
except zookeeper.ZooKeeperException, e:
print "Error on path", child_path, e
def post(self):
request_dict = self.request.arguments
node_key = (request_dict['node_key'])[0]
cluster_name = (request_dict['cluster_name'])[0]
zk=zookeeper.init(self.zk_connect(cluster_name))
try:
zookeeper.delete(zk,node_key)
msg = '删除成功'
except:
msg = '无法删除节点'
finally:
zookeeper.close(zk)
self.write(msg)
def release(self):
# All release has to do, if you follow the logic in acquire, is delete the unique ID that this lock created. That will wake
# up all the other waiters and whoever is first in line can then have the lock.
global handle
released = False
while not released:
try:
zookeeper.delete(handle, self.znode)
released = True
except zookeeper.ConnectionLossException:
reconnect()
self.znode = None
self.keyname = None
def _write(self, path, contents, ephemeral, exclusive, sequential, mustexist):
# We start from the second element because we do not want to inclued
# the initial empty string before the first "/" because all paths begin
# with "/". We also don't want to include the final node because that
# is dealt with later.
partial_path = ''
for path_part in path.split("/")[1:-1]:
partial_path = partial_path + "/" + path_part
if not(zookeeper.exists(self.handle, partial_path)):
try:
zookeeper.create(self.handle, partial_path, '', [self.acl], 0)
except zookeeper.NodeExistsException:
pass
if sequential:
exists = False
else:
exists = zookeeper.exists(self.handle, path)
# Don't create it if we're exclusive.
if exists and exclusive:
return False
# Check if we require the node to exist.
if not exists and mustexist:
return False
# We make sure that we have the creation flags for ephemeral nodes,
# otherwise they could be associated with previous connections that
# have not yet timed out.
if ephemeral and exists:
try:
zookeeper.delete(self.handle, path)
except zookeeper.NoNodeException:
pass
exists = False
if exists:
zookeeper.set(self.handle, path, contents)
return path
else:
flags = 0
if ephemeral:
flags = flags | zookeeper.EPHEMERAL
if sequential:
flags = flags | zookeeper.SEQUENCE
# NOTE: We return the final path created.
return zookeeper.create(self.handle, path, contents, [self.acl], flags)
def delete(self, path):
"""
Delete the ZooKeeper Node at `path`
Arguments:
- `path`: string
Return: None
Exceptions: NoNodeError
"""
try:
zookeeper.delete(self._zk, path)
except zookeeper.NoNodeException:
errmsg = "The Node {0} does not exist".format(path)
raise exceptions.NoNodeError(errmsg)
def delete(self, path):
"""
Delete the path.
"""
if zookeeper.exists(self.handle, path):
path_children = zookeeper.get_children(self.handle, path)
for child in path_children:
try:
self.delete(path + "/" + child)
except zookeeper.NoNodeException:
pass
try:
zookeeper.delete(self.handle, path)
except zookeeper.NoNodeException:
pass
def delete(self, path, version=-1):
start = time.time()
result = zookeeper.delete(self.handle, path, version)
if VERBOSE:
print("Node %s deleted in %d ms"
% (path, int((time.time() - start) * 1000)))
return result
def set_node(delurl,addurl):
handler = zookeeper.init("127.0.0.1:2181")
delnode = zookeeper.exists(handler,delurl)
addnode = zookeeper.exists(handler,addurl)
#找到旧的URL就删除
if delnode:
zookeeper.delete(handler,delurl)#删除旧的URL
else:
write_log(' zookeeper not found old url '+delurl)
#如果新的URL不存在就创建
if addnode == None:
try:
zookeeper.create(handler,addurl,'',[{"perms":0x1f,"scheme":"world","id":"anyone"}],0)#zookeeper重设URL
write_log(' zookeeper url from '+delurl+' change to '+addurl+' success')
except Exception, e:
write_log(' zookeeper url from '+delurl+' change to '+addurl+' failed')
def get_and_delete(self,node):
"""
Atomic get-and-delete operation. Returns None on failure.
"""
try:
(data,stat) = zookeeper.get(self.handle, node, None)
zookeeper.delete(self.handle, node, stat["version"])
return data
except zookeeper.NoNodeException:
# Someone deleted the node in between our get and delete
return None
except zookeeper.BadVersionException, e:
# Someone is modifying the queue in place. You can reasonably
# either retry to re-read the item, or abort.
print "Queue item %d modified in place, aborting..." % node
raise e
def delete(self, path):
"""
Delete the path.
"""
if not path:
raise BadArgumentsException("Invalid path: %s" % (path))
path_children = self.list_children(path)
for child in path_children:
try:
self.delete(path + "/" + child)
except zookeeper.NoNodeException:
pass
try:
zookeeper.delete(self.handle, path)
except zookeeper.NoNodeException:
pass
def delete(self, version, await_update=1): """Delete the node at this path. This can fail for all sorts of reasons: not empty, bad version, doesn't exist, various server problems. If the node should be deleted regardless of its current version, version can be given as -1. """ self.__zk._use_socket(lambda z: zookeeper.delete(z, self.path, version)) self._wait_version(await_update, -1)
def test_sync_delete(self):
ZOO_OPEN_ACL_UNSAFE = {"perms":0x1f, "scheme":"world", "id" :"anyone"}
self.assertEqual(self.connected, True)
ret = zookeeper.create(self.handle, "/zk-python-deletetest", "nodecontents", [ZOO_OPEN_ACL_UNSAFE], zookeeper.EPHEMERAL)
self.assertEqual(ret, "/zk-python-deletetest")
ret = zookeeper.delete(self.handle,"/zk-python-deletetest")
self.assertEqual(ret, zookeeper.OK)
children = zookeeper.get_children(self.handle, "/")
self.assertEqual(False, "zk-python-deletetest" in children)
def main():
data = options.znode_data_size * "D"
zookeeper.set_debug_level(zookeeper.LOG_LEVEL_WARN)
s = zookeeper.init(options.server)
if zookeeper.exists(s, options.root_znode, None):
children = zookeeper.get_children(s, options.root_znode, None)
print "delete old entries: %d" % len(children)
for child in children:
zookeeper.delete(s, "%s/%s" % (options.root_znode, child))
else:
zookeeper.create(s, options.root_znode, "zkpy root znode", acl, 0)
evaluation(s, options.root_znode, data, options)
zookeeper.close(s)
print("Performance test complete")