def queue_callback(self, zh, rc, data):
if zookeeper.OK == rc:
try:
for child in sorted(data):
action = self.actionNode + '/' + child
workLog = self.actionNode + '/' + child + '/worklog'
statusLog = self.statusNode + '/status-'
""" Launch the task if the task has not been executed """
if zookeeper.exists(zh, workLog, None) == None:
self.launch(zh, workLog, action, statusLog)
else:
""" If task has been previous launched, check for partial execution """
buffer = zookeeper.get(zh, workLog, 0)
state = simplejson.loads(buffer[0])
""" If task is incompleted in execution, launch again """
if 'status' in state and state['status'] == 'STARTING':
logger.info('Relaunch '+child)
self.launch(zh, workLog, action, statusLog)
else:
""" If the task has been launched, and completed, update status queue """
if zookeeper.exists(zh, statusLog, None) == None:
logger.info('Update status.')
self.update(zh, statusLog, state)
except NoNodeException, err:
""" Skip no node exception """
except Exception, err:
logger.exception(err)
def queue_callback(self, zh, rc, data):
if zookeeper.OK == rc:
try:
for child in sorted(data):
action = self.actionNode + '/' + child
workLog = self.actionNode + '/' + child + '/worklog'
statusLog = self.statusNode + '/status-'
""" Launch the task if the task has not been executed """
if zookeeper.exists(zh, workLog, None) == None:
self.launch(zh, workLog, action, statusLog)
else:
""" If task has been previous launched, check for partial execution """
buffer = zookeeper.get(zh, workLog, 0)
state = simplejson.loads(buffer[0])
""" If task is incompleted in execution, launch again """
if 'status' in state and state['status'] == 'STARTING':
logger.info('Relaunch '+child)
self.launch(zh, workLog, action, statusLog)
else:
""" If the task has been launched, and completed, update status queue """
if zookeeper.exists(zh, statusLog, None) == None:
logger.info('Update status.')
self.update(zh, statusLog, state)
except NoNodeException, err:
""" Skip no node exception """
except Exception, err:
logger.exception(err)
def set_watcher():
def fn():
self.callback_flag = True
self.callback_flag = False
zookeeper.exists(
self.handle, "/zk-python-lose-scope-test",
self.create_callback(lambda handle, type, state, path: fn()))
def notifyFailedTransaction(self, app_id, txid):
""" Notify failed transaction id.
This method will add the transaction id into black list.
After this call, the transaction becomes invalid.
"""
self.__waitForConnect()
self.checkTransaction(app_id, txid)
print "notify failed transaction app:%s, txid:%d" % (app_id, txid)
txpath = self.__getTransactionPath(app_id, txid)
lockpath = None
try:
lockpath = zookeeper.get(self.handle, PATH_SEPARATOR.join([txpath, TX_LOCK_PATH]), None)[0]
except zookeeper.NoNodeException:
# there is no lock. it means there is no need to rollback.
pass
if lockpath:
# add transacion id to black list.
now = str(time.time())
broot = self.__getBlacklistRootPath(app_id)
if not zookeeper.exists(self.handle, broot):
self.__forceCreatePath(broot)
zookeeper.acreate(self.handle, PATH_SEPARATOR.join([broot, str(txid)]), now, ZOO_ACL_OPEN)
# update local cache before notification
if app_id in self.blacklistcache:
with self.blacklistcv:
self.blacklistcache[app_id].add(str(txid))
# copy valid transaction id for each updated key into valid list.
for child in zookeeper.get_children(self.handle, txpath):
if re.match("^" + TX_UPDATEDKEY_PREFIX, child):
value = zookeeper.get(self.handle, PATH_SEPARATOR.join([txpath, child]), None)[0]
valuelist = value.split(PATH_SEPARATOR)
key = urllib.unquote_plus(valuelist[0])
vid = valuelist[1]
vtxroot = self.__getValidTransactionRootPath(app_id)
if not zookeeper.exists(self.handle, vtxroot):
self.__forceCreatePath(vtxroot)
vtxpath = self.__getValidTransactionPath(app_id, key)
zookeeper.acreate(self.handle, vtxpath, vid, ZOO_ACL_OPEN)
# release the lock
try:
zookeeper.adelete(self.handle, lockpath)
except zookeeper.NoNodeException:
# this should be retry.
pass
# just remove transaction node
try:
for item in zookeeper.get_children(self.handle, txpath):
zookeeper.adelete(self.handle, PATH_SEPARATOR.join([txpath, item]))
zookeeper.adelete(self.handle, txpath)
except zookeeper.NoNodeException:
# something wrong. next GC will take care of it.
return False
return True
def notifyFailedTransaction(self, app_id, txid):
""" Notify failed transaction id.
This method will add the transaction id into black list.
After this call, the transaction becomes invalid.
"""
self.__waitForConnect()
self.checkTransaction(app_id, txid)
print "notify failed transaction app:%s, txid:%d" % (app_id, txid)
txpath = self.__getTransactionPath(app_id, txid)
lockpath = None
try:
lockpath = zookeeper.get(self.handle, PATH_SEPARATOR.join([txpath, TX_LOCK_PATH]), None)[0]
except zookeeper.NoNodeException:
# there is no lock. it means there is no need to rollback.
pass
if lockpath:
# add transacion id to black list.
now = str(time.time())
broot = self.__getBlacklistRootPath(app_id)
if not zookeeper.exists(self.handle, broot):
self.__forceCreatePath(broot)
zookeeper.acreate(self.handle, PATH_SEPARATOR.join([broot, str(txid)]), now, ZOO_ACL_OPEN)
# update local cache before notification
if app_id in self.blacklistcache:
with self.blacklistcv:
self.blacklistcache[app_id].add(str(txid))
# copy valid transaction id for each updated key into valid list.
for child in zookeeper.get_children(self.handle, txpath):
if re.match("^" + TX_UPDATEDKEY_PREFIX, child):
value = zookeeper.get(self.handle, PATH_SEPARATOR.join([txpath, child]), None)[0]
valuelist = value.split(PATH_SEPARATOR)
key = urllib.unquote_plus(valuelist[0])
vid = valuelist[1]
vtxroot = self.__getValidTransactionRootPath(app_id)
if not zookeeper.exists(self.handle, vtxroot):
self.__forceCreatePath(vtxroot)
vtxpath = self.__getValidTransactionPath(app_id, key)
zookeeper.acreate(self.handle, vtxpath, vid, ZOO_ACL_OPEN)
# release the lock
try:
zookeeper.adelete(self.handle, lockpath)
except zookeeper.NoNodeException:
# this should be retry.
pass
# just remove transaction node
try:
for item in zookeeper.get_children(self.handle, txpath):
zookeeper.adelete(self.handle, PATH_SEPARATOR.join([txpath, item]))
zookeeper.adelete(self.handle, txpath)
except zookeeper.NoNodeException:
# something wrong. next GC will take care of it.
return False
return True
def write(self, path, contents, ephemeral=False, exclusive=False):
"""
Writes the contents to the path in zookeeper. It will create the path in
zookeeper if it does not already exist.
This method will return True if the value is written, False otherwise.
(The value will not be written if the exclusive is True and the node
already exists.)
"""
partial_path = ''
# We start from the second element because we do not want to inclued
# the initial empty string before the first "/" because all paths begin
# with "/". We also don't want to include the final node because that
# is dealt with later.
for path_part in path.split("/")[1:-1]:
partial_path = partial_path + "/" + path_part
if not(zookeeper.exists(self.handle, partial_path)):
try:
zookeeper.create(self.handle, partial_path, '', [self.acl], 0)
except zookeeper.NodeExistsException:
pass
exists = zookeeper.exists(self.handle, path)
# Don't create it if we're exclusive.
if exists and exclusive:
return False
# We make sure that we have the creation flags for ephemeral nodes,
# otherwise they could be associated with previous connections that
# have not yet timed out.
if ephemeral and exists:
try:
zookeeper.delete(self.handle, path)
except zookeeper.NoNodeException:
pass
exists = False
if exists:
zookeeper.set(self.handle, path, contents)
return True
else:
flags = (ephemeral and zookeeper.EPHEMERAL or 0)
try:
zookeeper.create(self.handle, path, contents, [self.acl], flags)
return True
except zookeeper.NodeExistsException:
if not(exclusive):
# Woah, something happened between the top and here.
# We just restart and retry the whole routine.
self.write(path, contents, ephemeral=ephemeral)
return True
else:
return False
def initialize_node_structure(self):
if not zk.exists(self.zh, self.ROOT):
self.create(self.ROOT)
if not zk.exists(self.zh, self.NODE_SERVERS):
self.create(self.NODE_SERVERS)
self.NODE_GJOBS = self.ROOT + '/jobs'
if not zk.exists(self.zh, self.NODE_GJOBS):
self.create(self.NODE_GJOBS)
def _write(self, path, contents, ephemeral, exclusive, sequential,
mustexist):
# We start from the second element because we do not want to inclued
# the initial empty string before the first "/" because all paths begin
# with "/". We also don't want to include the final node because that
# is dealt with later.
partial_path = ''
for path_part in path.split("/")[1:-1]:
partial_path = partial_path + "/" + path_part
if not (zookeeper.exists(self.handle, partial_path)):
try:
zookeeper.create(self.handle, partial_path, '', [self.acl],
0)
except zookeeper.NodeExistsException:
pass
if sequential:
exists = False
else:
exists = zookeeper.exists(self.handle, path)
# Don't create it if we're exclusive.
if exists and exclusive:
return False
# Check if we require the node to exist.
if not exists and mustexist:
return False
# We make sure that we have the creation flags for ephemeral nodes,
# otherwise they could be associated with previous connections that
# have not yet timed out.
if ephemeral and exists:
try:
zookeeper.delete(self.handle, path)
except zookeeper.NoNodeException:
pass
exists = False
if exists:
zookeeper.set(self.handle, path, contents)
return path
else:
flags = 0
if ephemeral:
flags = flags | zookeeper.EPHEMERAL
if sequential:
flags = flags | zookeeper.SEQUENCE
# NOTE: We return the final path created.
return zookeeper.create(self.handle, path, contents, [self.acl],
flags)
def node_add():
basepath = request.args.get('path', '')
if request.method == 'POST':
path = request.form.get('path')
data = request.form.get('data')
print request.form
print request.values
verify = True
if len(path) == 0:
verify = False
flash('"Node Path" not allow empty')
elif path[0] == '/':
verify = False
flash('"Node Path" can not start with "/"')
else:
path = CONFIG_NODE / basepath / path
if zookeeper.exists(zk, path):
verify = False
flash('Path already exists:"%s"' % (path))
if verify:
nodepath = FilePath('/')
try:
for node in path.split('/'):
nodepath = nodepath / node
if not zookeeper.exists(zk, nodepath):
if nodepath != path:
zookeeper.create(zk, nodepath, '',
[{
"perms": zookeeper.PERM_ALL,
"scheme": 'world',
"id": 'anyone'
}], 0)
else:
zookeeper.create(zk, nodepath, data,
[{
"perms": zookeeper.PERM_ALL,
"scheme": 'world',
"id": 'anyone'
}], 0)
return redirect('/node')
except BaseException, e:
flash(e.message)
formdata = {
'path': request.form.get('path') if path != '' else '',
'data': data
}
def add_twemproxy(host, port, zookeeper_str, product_prefix):
try:
zk = zookeeper.init(zookeeper_str)
if not zookeeper.exists(zk, "/twemproxy"):
create_zookeeper(zk, "/twemproxy", "")
zk_twproxy_path = "/twemproxy/%s/%s:%d" % (product_prefix, host, port)
if not zookeeper.exists(zk, zk_twproxy_path):
create_zookeeper(zk, zk_twproxy_path, "")
return True
else:
return False
except zookeeper.NodeExistsException, e:
handle_output(False, "path %s exist\n" % pathstr)
def before_request():
if not zookeeper.exists(zk, ROOT_NODE):
zookeeper.create(zk, ROOT_NODE, '', [{
"perms": zookeeper.PERM_ALL,
"scheme": "world",
"id": "anyone"
}], 0)
elif not zookeeper.exists(zk, CONFIG_NODE):
zookeeper.create(zk, CONFIG_NODE, '', [{
"perms": zookeeper.PERM_ALL,
"scheme": "world",
"id": "anyone"
}], 0)
def server_delete():
name = request.args.get('name', '')
if name == '':
return redirect(request.referrer)
if zookeeper.exists(zk, ONLINE_NODE / name):
flash('"%s" is already online and can not be deleted' % (name))
return redirect(request.referrer)
nodepath = SERVIE_NODE / name
if zookeeper.exists(zk, nodepath):
zookeeper_delete_node(zk, nodepath)
return redirect(request.referrer)
def _write(self, path, contents, ephemeral, exclusive, sequential, mustexist):
# We start from the second element because we do not want to inclued
# the initial empty string before the first "/" because all paths begin
# with "/". We also don't want to include the final node because that
# is dealt with later.
partial_path = ''
for path_part in path.split("/")[1:-1]:
partial_path = partial_path + "/" + path_part
if not(zookeeper.exists(self.handle, partial_path)):
try:
zookeeper.create(self.handle, partial_path, '', [self.acl], 0)
except zookeeper.NodeExistsException:
pass
if sequential:
exists = False
else:
exists = zookeeper.exists(self.handle, path)
# Don't create it if we're exclusive.
if exists and exclusive:
return False
# Check if we require the node to exist.
if not exists and mustexist:
return False
# We make sure that we have the creation flags for ephemeral nodes,
# otherwise they could be associated with previous connections that
# have not yet timed out.
if ephemeral and exists:
try:
zookeeper.delete(self.handle, path)
except zookeeper.NoNodeException:
pass
exists = False
if exists:
zookeeper.set(self.handle, path, contents)
return path
else:
flags = 0
if ephemeral:
flags = flags | zookeeper.EPHEMERAL
if sequential:
flags = flags | zookeeper.SEQUENCE
# NOTE: We return the final path created.
return zookeeper.create(self.handle, path, contents, [self.acl], flags)
def zookeeper_node_set(nodepath, nodevalue, perms, **option):
ephemeral = 0
if option.has_key('ephemeral') and option['ephemeral']:
ephemeral = zookeeper.EPHEMERAL
parent_perms = perms
if option.has_key('parent_perms'):
parent_perms = option['parent_perms']
p = FilePath('/')
for v in nodepath.split('/'):
p = p / v
if not zookeeper.exists(G.zookeeper, p):
if not option.has_key('nocreate') or not option['nocreate']:
if p == nodepath:
print zookeeper.create(G.zookeeper, p, nodevalue,
[{
"perms": perms,
"scheme": "world",
"id": "anyone"
}], ephemeral)
return True
else:
zookeeper.create(G.zookeeper, p, '', [{
"perms": parent_perms,
"scheme": "world",
"id": "anyone"
}], 0)
elif p == nodepath:
print zookeeper.set(G.zookeeper, p, nodevalue)
return True
return False
def exists(path):
init_client()
global handle
try:
return zookeeper.exists(handle,path) is not None
except zookeeper.NoNodeException:
return False
def set_node(delurl,addurl):
handler = zookeeper.init("127.0.0.1:2181")
delnode = zookeeper.exists(handler,delurl)
addnode = zookeeper.exists(handler,addurl)
#找到旧的URL就删除
if delnode:
zookeeper.delete(handler,delurl)#删除旧的URL
else:
write_log(' zookeeper not found old url '+delurl)
#如果新的URL不存在就创建
if addnode == None:
try:
zookeeper.create(handler,addurl,'',[{"perms":0x1f,"scheme":"world","id":"anyone"}],0)#zookeeper重设URL
write_log(' zookeeper url from '+delurl+' change to '+addurl+' success')
except Exception, e:
write_log(' zookeeper url from '+delurl+' change to '+addurl+' failed')
def getReadLock(self, metadata):
#Acquire read lock on file through ZooKeeper
fn = metadata.filename
print "Trying to acquire read lock for " + self.root + fn
self.zpath = self.createLockNode(fn, 'r')
myLock = self.zpath.rsplit('/',1)[1]
while True:
children = sorted(zookeeper.get_children(self.zh, self.root + fn))
lastLock = ''
for child in children:
try:
if child == myLock:
break
elif zookeeper.get(self.zh, self.root+fn+'/'+child)[0] == 'w':
lastLock = child
except:
pass
if lastLock != '':
def watcher (handle, type, state, path):
self.cv2.acquire()
self.cv2.notify()
self.cv2.release()
self.cv2.acquire()
if zookeeper.exists(self.zh, self.root+fn+'/'+lastLock, watcher):
self.cv2.wait()
self.cv2.release()
else:
break
print "Acquired read lock for " + self.root + fn
def test_sync_exists(self):
self.assertEqual(self.connected, True)
ret = zookeeper.exists(self.handle, "/zk-python-existstest", None)
self.assertNotEqual(
ret, None,
"/zk-python-existstest does not exist (possibly means creation failure)"
)
def __init__(self, zkhosts, root=NODE_HQ_ROOT, alivenode='alive'):
"""zkhosts: a string or a list. list will be ','.join-ed into a string.
root: root node path (any parents must exist, if any)
"""
if not isinstance(zkhosts, basestring):
zkhosts = ','.join(zkhosts)
self.zkhosts = zkhosts
self.ROOT = root
self.alivenode = alivenode
self.nodename = os.uname()[1]
self.NODE_SERVERS = self.ROOT + '/servers'
self.NODE_ME = self.NODE_SERVERS+'/'+self.nodename
self.zh = zk.init(self.zkhosts, self.__watcher)
self.jobs = {}
self.initialize_node_structure()
if not zk.exists(self.zh, self.NODE_ME):
zk.create(self.zh, self.NODE_ME, '', PERM_WORLD)
# setup notification
zk.get_children(self.zh, self.NODE_SERVERS, self.__servers_watcher)
#zk.get_children(self.zh, self.NODE_ME, self.__watcher)
self.NODE_JOBS = self.NODE_ME+'/jobs'
zk.acreate(self.zh, self.NODE_JOBS, '', PERM_WORLD)
def getReadLock(self, metadata):
#Acquire read lock on file through ZooKeeper
fn = metadata.filename
print "Trying to acquire read lock for " + self.root + fn
self.zpath = self.createLockNode(fn, 'r')
myLock = self.zpath.rsplit('/', 1)[1]
while True:
children = sorted(zookeeper.get_children(self.zh, self.root + fn))
lastLock = ''
for child in children:
try:
if child == myLock:
break
elif zookeeper.get(self.zh,
self.root + fn + '/' + child)[0] == 'w':
lastLock = child
except:
pass
if lastLock != '':
def watcher(handle, type, state, path):
self.cv2.acquire()
self.cv2.notify()
self.cv2.release()
self.cv2.acquire()
if zookeeper.exists(self.zh, self.root + fn + '/' + lastLock,
watcher):
self.cv2.wait()
self.cv2.release()
else:
break
print "Acquired read lock for " + self.root + fn
def exists(path):
init_client()
global handle
try:
return zookeeper.exists(handle, path) is not None
except zookeeper.NoNodeException:
return False
def __init__(self, service_name, connect_string, timeout=DEFAULT_TIMEOUT, default_port=6664):
self.SERVICE_NODE = "/" + service_name
self.AVAILABILITY_NODE = self.SERVICE_NODE + "/available"
self.MEMBERSHIP_NODE = self.SERVICE_NODE + "/members"
self.connected = False
self.timeout = timeout
self.default_port = default_port
self.conn_cv = threading.Condition()
self.conn_cv.acquire()
self.handle = zookeeper.init(connect_string, self.connection_watcher, timeout)
self.conn_cv.wait(timeout / 1000)
self.conn_cv.release()
self.watcher_lock = threading.Lock()
self.logger = logging.getLogger("sincc")
if not self.connected:
raise SinClusterClientError("Unable to connect to %s" % connect_string)
for path in [self.SERVICE_NODE, self.AVAILABILITY_NODE, self.MEMBERSHIP_NODE]:
if not zookeeper.exists(self.handle, path):
zookeeper.create(self.handle, path, "", [ZOO_OPEN_ACL_UNSAFE], 0)
self.listeners = []
# Start to watch both /members and /available
zookeeper.get_children(self.handle, self.MEMBERSHIP_NODE, self.watcher)
available = zookeeper.get_children(self.handle, self.AVAILABILITY_NODE, self.watcher)
self.available_nodes = {}
for node_id in available:
self.available_nodes[int(node_id)] = Node(int(node_id),
zookeeper.get(self.handle, self.AVAILABILITY_NODE + "/" + node_id)[0])
def list_children(self, path):
"""
Returns a list of all the children nodes in the path. None is returned if the path does
not exist.
"""
if zookeeper.exists(self.handle, path):
value = zookeeper.get_children(self.handle, path)
return value
def watcher(zk, type, state, nodepath):
log('watcher', {
'type': type,
'state': state,
'nodepath': nodepath
}, 'DEBUG')
if type == zookeeper.SESSION_EVENT:
zookeeper.set_watcher(zk, watcher)
if state == zookeeper.CONNECTED_STATE:
for k in G.project:
if zookeeper.exists(zk, k, watcher):
# 启动时马上就通知一次，防止在断线过程中出现了数据更改，而服务又不知道
on_zookeeper_node_change(zookeeper.get(zk, k), k)
if zookeeper.exists(zk, ROOT_NODE, watcher):
config = zookeeper.get(zk, ROOT_NODE)[0]
if config != '':
G.config = json.loads(config)
if not isinstance(G.config, dict):
raise TypeError()
else:
G.config = {}
log('config', G.config, 'DEBUG')
elif type == zookeeper.CREATED_EVENT or type == zookeeper.CHANGED_EVENT:
nodevalue = zookeeper.get(zk, nodepath, watcher)
if nodepath == ROOT_NODE:
if nodevalue[0] != '':
G.config = json.loads(nodevalue[0])
if not isinstance(G.config, dict):
raise TypeError()
else:
G.config = {}
log('config', G.config, 'DEBUG')
# restart process
if G.config.has_key('restart') and G.config['restart'] in [
True, G.name
]:
G.queue.put((0, {'type': 'restart'}))
else:
on_zookeeper_node_change(nodevalue, nodepath)
elif type == zookeeper.DELETED_EVENT:
zookeeper.exists(zk, nodepath, watcher) # 期待再次创建节点
def register_service(self, service, port=0):
path = ""
if service == "master":
path = ZKConf.ZK_PATH_TEMP_MASTER
elif service == "stats":
path = ZKConf.ZK_PATH_TEMP_STATS
elif service == "proxy":
path = ZKConf.ZK_PATH_TEMP_PROXY
if not path:
raise Exception("service type '%s' doesn't exist" % (service, ))
node = zookeeper.exists(self._handler, path, None)
if not node:
if not zookeeper.exists(self._handler, ZKConf.ZK_PATH_ROOT, None):
zookeeper.create(self._handler, ZKConf.ZK_PATH_ROOT, "",
[ZKConf.ZOO_CREATOR_ALL_ACL], 0)
zookeeper.create(self._handler, path, "",
[ZKConf.ZOO_CREATOR_ALL_ACL], 0)
ip = utils.getip()
mepath = "".join([path, "/", ip, ":", str(port), "-"])
if not port:
mepath = "".join([path, "/", ip, "-"])
childs = zookeeper.get_children(self._handler, path, None)
is_created = False
pathme = mepath.split("/")[-1]
for child in childs:
if child.startswith(pathme):
is_created = child.split("-")[-1]
break
if is_created:
meflag = mepath + is_created
utils.log(utils.cur(),
"%s none-callback %s" % ("*" * 10, "*" * 10), meflag)
else:
utils.log(utils.cur(),
"%s %s %s" % ("*" * 10, self.register_node_cb, "*" * 10),
mepath)
meflag = zookeeper.create(self._handler, mepath, "",
[ZKConf.ZOO_CREATOR_ALL_ACL],
zookeeper.SEQUENCE | zookeeper.EPHEMERAL)
utils.log(utils.cur(), "-" * 10, meflag)
def read(self, path, default=None):
"""
Returns the conents in the path. default is returned if the path does not exists.
"""
value = default
if zookeeper.exists(self.handle, path):
value, timeinfo = zookeeper.get(self.handle, path)
return value
def registUpdatedKey(self, app_id, current_txid, target_txid, entity_key):
""" Regist valid transaction id for entity.
target_txid must be the latest valid transaction id for the entity.
"""
self.__waitForConnect()
vtxpath = self.__getValidTransactionPath(app_id, entity_key)
if zookeeper.exists(self.handle, vtxpath):
# just update transaction id for entity if there is valid transaction node.
zookeeper.aset(self.handle, vtxpath, str(target_txid))
else:
# store the updated key info into current transaction node.
value = PATH_SEPARATOR.join([urllib.quote_plus(entity_key), str(target_txid)])
txpath = self.__getTransactionPath(app_id, current_txid)
if zookeeper.exists(self.handle, txpath):
zookeeper.acreate(self.handle, PATH_SEPARATOR.join([txpath, TX_UPDATEDKEY_PREFIX]), value, ZOO_ACL_OPEN, zookeeper.SEQUENCE)
else:
raise ZKTransactionException(ZKTransactionException.TYPE_INVALID, "Transaction %d is not valid." % current_txid)
def __make_presence(self):
if not zookeeper.exists(self.zk, '/present/%s' % self.queue):
self.__create_node('/present/%s' % self.queue, '', perms, 0)
self.__create_node('/present/%s/%d' % (self.queue, self.zk),
'',
perms,
zookeeper.EPHEMERAL)
self.__log('Created presence node successfully')
def registUpdatedKey(self, app_id, current_txid, target_txid, entity_key):
""" Regist valid transaction id for entity.
target_txid must be the latest valid transaction id for the entity.
"""
self.__waitForConnect()
vtxpath = self.__getValidTransactionPath(app_id, entity_key)
if zookeeper.exists(self.handle, vtxpath):
# just update transaction id for entity if there is valid transaction node.
zookeeper.aset(self.handle, vtxpath, str(target_txid))
else:
# store the updated key info into current transaction node.
value = PATH_SEPARATOR.join([urllib.quote_plus(entity_key), str(target_txid)])
txpath = self.__getTransactionPath(app_id, current_txid)
if zookeeper.exists(self.handle, txpath):
zookeeper.acreate(self.handle, PATH_SEPARATOR.join([txpath, TX_UPDATEDKEY_PREFIX]), value, ZOO_ACL_OPEN, zookeeper.SEQUENCE)
else:
raise ZKTransactionException(ZKTransactionException.TYPE_INVALID, "Transaction %d is not valid." % current_txid)
def delete(dry_run=False):
wc = ctx.get_write_context()
LOG.info("Deleting recursively " + str(cnt.ZOOM_ZK_ROOTS) +
" from server(s) " + wc.zk_servers)
handle = zookeeper.init(wc.zk_servers)
for root in cnt.ZOOM_ZK_ROOTS:
if zookeeper.exists(handle, root):
_recursive_delete(handle, root, dry_run=dry_run)
def __get_message(self):
messages = self.__get_children('/queues/%s' % self.queue)
self.__log('got %d children of the queue' % len(messages), level=logging.Logger.debug)
for message in sorted(messages):
if not zookeeper.exists(self.zk, '/queues/%s/%s/taken' % (self.queue, message)):
self.__create_node('/queues/%s/%s/taken' % (self.queue, message), str(self.zk),
perms, zookeeper.EPHEMERAL)
self._current_message = '/queues/%s/%s' % (self.queue, message)
return self.__get_node('/queues/%s/%s' % (self.queue, message))
return None
def register_service(self, service, port=0):
path = ""
if service == "master":
path = ZKConf.ZK_PATH_TEMP_MASTER
elif service == "stats":
path = ZKConf.ZK_PATH_TEMP_STATS
elif service == "proxy":
path = ZKConf.ZK_PATH_TEMP_PROXY
if not path:
raise Exception("service type '%s' doesn't exist" % (service,))
node = zookeeper.exists(self._handler, path, None)
if not node:
if not zookeeper.exists(self._handler, ZKConf.ZK_PATH_ROOT, None):
zookeeper.create(self._handler, ZKConf.ZK_PATH_ROOT, "", [ZKConf.ZOO_CREATOR_ALL_ACL], 0)
zookeeper.create(self._handler, path, "", [ZKConf.ZOO_CREATOR_ALL_ACL], 0)
ip = utils.getip()
mepath = "".join([path, "/", ip, ":", str(port), "-"])
if not port:
mepath = "".join([path, "/", ip, "-"])
childs = zookeeper.get_children(self._handler, path, None)
is_created = False
pathme = mepath.split("/")[-1]
for child in childs:
if child.startswith(pathme):
is_created = child.split("-")[-1]
break
if is_created:
meflag = mepath + is_created
utils.log(utils.cur(), "%s none-callback %s" % ("*"*10, "*"*10), meflag)
else:
utils.log(utils.cur(), "%s %s %s" % ("*"*10, self.register_node_cb, "*"*10), mepath)
meflag = zookeeper.create(self._handler,
mepath,
"",
[ZKConf.ZOO_CREATOR_ALL_ACL],
zookeeper.SEQUENCE|zookeeper.EPHEMERAL)
utils.log(utils.cur(), "-"*10, meflag)
def exists(self, path, watch=None):
"""
Determine whether the ZooKeeper Node at `path` exists
Arguments:
- `path`: string
Return: dict of stats or None
Exceptions: None
"""
return zookeeper.exists(self._zk, path, watch)
def create_zookeeper_without_delete_children(zk, path, data):
try:
if zookeeper.exists(zk, path):
return
zookeeper.create(zk, path, data, [{
"perms": 15,
"scheme": "world",
"id": "anyone"
}], 0)
except zookeeper.NodeExistsException, e:
err_str = "path %s exist\n" % path
handle_output(False, err_str)
def clean_tree(zh, root):
"""Recursively removes the zookeeper subtree underneath :root given zookeeper handle :zh."""
try:
if not zookeeper.exists(zh, root):
return True
for child in zookeeper.get_children(zh, root):
if not ZookeeperUtil.clean_tree(zh, posixpath.join(root, child)):
return False
zookeeper.delete(zh, root)
except zookeeper.ZooKeeperException as e:
return False
return True
def cached_exists(self, path):
cache = self._caches.setdefault('exists', dict())
retval = cache.get(path, Ellipsis)
if retval is not Ellipsis:
return retval
def invalidator(event):
cache.pop(path, None)
retval = zookeeper.exists(self.handle, path, self._wrap_watcher(invalidator))
cache[path] = retval
return retval
def clean_tree(zh, root):
"""Recursively removes the zookeeper subtree underneath :root given zookeeper handle :zh."""
try:
if not zookeeper.exists(zh, root):
return True
for child in zookeeper.get_children(zh, root):
if not ZookeeperUtil.clean_tree(zh, posixpath.join(
root, child)):
return False
zookeeper.delete(zh, root)
except zookeeper.ZooKeeperException as e:
return False
return True
def post(self):
request_dict = self.request.arguments
cluster_name = (request_dict['cluster_name'])[0]
zk=zookeeper.init(self.zk_connect(cluster_name))
new_node = (request_dict['New_post_node'])[0]
new_value = (request_dict['new_node_value'])[0]
if zookeeper.exists(zk,new_node):
zookeeper.close(zk)
self.write("此节点存在")
else:
zookeeper.create(zk,new_node,new_value,[{"perms":0x1f,"scheme":"world","id":"anyone"}],0)
zookeeper.close(zk)
self.write("增加成功")
def checkTransaction(self, app_id, txid):
""" Get status of specified transaction.
Returns: True - If transaction is alive.
Raises: ZKTransactionException - If transaction is timeout or not exist.
"""
self.__waitForConnect()
txpath = self.__getTransactionPath(app_id, txid)
if self.isBlacklisted(app_id, txid):
raise ZKTransactionException(ZKTransactionException.TYPE_EXPIRED, "Transaction %d is timeout." % txid)
if not zookeeper.exists(self.handle, txpath):
raise ZKTransactionException(ZKTransactionException.TYPE_INVALID, "TransactionID %d is not valid." % txid)
return True
def watch_contents(self, path, fn, default_value="", clean=False):
if not zookeeper.exists(self.handle, path):
self.write(path, default_value)
self.cond.acquire()
try:
if clean:
self.watches[path] = []
if not(fn in self.watches.get(path, [])):
self.watches[path] = self.watches.get(path, []) + [fn]
value, timeinfo = zookeeper.get(self.handle, path, self.zookeeper_watch)
finally:
self.cond.release()
return value
def cached_exists(self, path):
cache = self._caches.setdefault('exists', dict())
retval = cache.get(path, Ellipsis)
if retval is not Ellipsis:
return retval
def invalidator(event):
cache.pop(path, None)
retval = zookeeper.exists(self.handle, path,
self._wrap_watcher(invalidator))
cache[path] = retval
return retval
def ready(self):
if not zookeeper.exists(self.zk, '/waiting/%s' % self.queue):
self.__create_node('/waiting/%s' % self.queue, '', perms, 0)
self.__log('Created waiting queue %s' % self.queue)
last_waiting = self.__get_nth_waiting(-1)
if last_waiting:
self.__set_watch(last_waiting, self.__in_line_watch)
self.waiting_node = self.__create_node('/waiting/%s/w-' % self.queue, str(self.zk), perms,
zookeeper.SEQUENCE | zookeeper.EPHEMERAL)
else:
self[0].waiting_node = self.__create_node('/waiting/%s/w-' % self.queue, str(self.zk), perms,
zookeeper.SEQUENCE | zookeeper.EPHEMERAL)
return self.__message_check()
def watch_children(self, path, fn, default_value="", clean=False):
self.cond.acquire()
if not zookeeper.exists(self.handle, path):
self.write(path, default_value)
try:
if clean:
self.watches[path] = []
if not(fn in self.watches.get(path, [])):
self.watches[path] = self.watches.get(path, []) + [fn]
rval = zookeeper.get_children(self.handle, path, self.zookeeper_watch)
finally:
self.cond.release()
return rval
def wait(self, file):
def ret_watcher(handle, event, state, path):
self.cv.acquire()
self.cv.notify()
self.cv.release()
while True:
self.cv.acquire()
data = zookeeper.exists(self.handle, file, ret_watcher)
if data != None:
data = self.get_and_delete(file)
self.cv.release()
return data
self.cv.wait()
self.cv.release()
def read(self, path, default=None):
"""
Returns the conents in the path. default is returned if the path does not exists.
"""
if not path:
raise BadArgumentsException("Invalid path: %s" % (path))
value = default
if zookeeper.exists(self.handle, path):
try:
value, _ = zookeeper.get(self.handle, path)
except zookeeper.NoNodeException:
pass
return value
def list_children(self, path):
"""
Returns a list of all the children nodes in the path. None is returned if the path does
not exist.
"""
if not path:
raise BadArgumentsException("Invalid path: %s" % (path))
if zookeeper.exists(self.handle, path):
try:
value = zookeeper.get_children(self.handle, path)
return value
except zookeeper.NoNodeException:
pass
return []
def checkTransaction(self, app_id, txid):
""" Get status of specified transaction.
Returns: True - If transaction is alive.
Raises: ZKTransactionException - If transaction is timeout or not exist.
"""
self.__waitForConnect()
txpath = self.__getTransactionPath(app_id, txid)
if self.isBlacklisted(app_id, txid):
raise ZKTransactionException(ZKTransactionException.TYPE_EXPIRED,
"zktransaction.checkTransaction: Transaction %d is timeout." % txid)
if not zookeeper.exists(self.handle, txpath):
raise ZKTransactionException(ZKTransactionException.TYPE_INVALID,
"zktransaction.checkTransaction: TransactionID %d is not valid." % txid)
return True
def delete(self, path):
"""
Delete the path.
"""
if zookeeper.exists(self.handle, path):
path_children = zookeeper.get_children(self.handle, path)
for child in path_children:
try:
self.delete(path + "/" + child)
except zookeeper.NoNodeException:
pass
try:
zookeeper.delete(self.handle, path)
except zookeeper.NoNodeException:
pass
def post(self):
request_dict = self.request.arguments
node_json_list = (eval(request_dict['node_json'][0])).values()
cluster_name = (request_dict['cluster_name'])[0]
current_node = (request_dict['current_node'])[0]
if current_node == "/" :
current_node = ""
zk=zookeeper.init(self.zk_connect(cluster_name))
for item in node_json_list:
key = current_node + "/" + item[0]
value = item[1]
if not zookeeper.exists(zk, key):
zookeeper.create(zk, key , value , [{"perms":0x1f,"scheme":"world","id":"anyone"}],0)
self.write("写入成功")
zookeeper.close(zk)
def list_children(self, path):
"""
Returns a list of all the children nodes in the path. None is returned if the path does
not exist.
"""
if not path:
raise BadArgumentsException("Invalid path: %s" % (path))
if zookeeper.exists(self.handle, path):
try:
value = zookeeper.get_children(self.handle, path)
return value
except zookeeper.NoNodeException:
pass
return []
def read(self, path, default=None):
"""
Returns the conents in the path. default is returned if the path does not exists.
"""
if not path:
raise BadArgumentsException("Invalid path: %s" % (path))
value = default
if zookeeper.exists(self.handle, path):
try:
value, _ = zookeeper.get(self.handle, path)
except zookeeper.NoNodeException:
pass
return value
def acquireLock(self, app_id, txid, entity_key = GLOBAL_LOCK_KEY):
""" Acquire lock for transaction.
You must call getTransactionID() first to obtain transaction ID.
You could call this method anytime if the root entity key is same.
Args:
app_id: The application ID to acquire a lock for.
txid: The transaction ID you are acquiring a lock for. Built into
the path.
entity_key: Used to get the root path.
Raises:
ZKTransactionException: If it could not get the lock.
"""
self.__waitForConnect()
txpath = self.__getTransactionPath(app_id, txid)
lockrootpath = self.__getLockRootPath(app_id, entity_key)
lockpath = None
if zookeeper.exists(self.handle, PATH_SEPARATOR.join([txpath, TX_LOCK_PATH])):
# use current lock
prelockpath = zookeeper.get(self.handle, PATH_SEPARATOR.join([txpath, TX_LOCK_PATH]), None)[0]
if not lockrootpath == prelockpath:
raise ZKTransactionException(ZKTransactionException.TYPE_DIFFERENT_ROOTKEY,
"zktransaction.acquireLock: You can not lock different root entity in same transaction.")
print "Already has lock: %s" % lockrootpath
return True
self.checkTransaction(app_id, txid)
# create new lock
retry = True
while retry:
retry = False
try:
lockpath = zookeeper.create(self.handle, lockrootpath, txpath, ZOO_ACL_OPEN)
except zookeeper.NoNodeException:
self.__forceCreatePath(PATH_SEPARATOR.join(lockrootpath.split(PATH_SEPARATOR)[:-1]))
retry = True
except zookeeper.NodeExistsException:
# fail to get lock
raise ZKTransactionException(ZKTransactionException.TYPE_CONCURRENT,
"zktransaction.acquireLock: There is already another transaction using this lock")
# set lockpath for transaction node
# TODO: we should think about atomic operation or recovery of
# inconsistent lockpath node.
zookeeper.acreate(self.handle, PATH_SEPARATOR.join([txpath, TX_LOCK_PATH]), lockpath, ZOO_ACL_OPEN)
return True
def watch_children(self, path, fn, clean=False):
if not (path and fn):
raise BadArgumentsException("Invalid path/fn: %s/%s" % (path, fn))
self.cond.acquire()
if not zookeeper.exists(self.handle, path):
self.write(path, "")
try:
if clean:
self.child_watches[path] = []
if not(fn in self.child_watches.get(path, [])):
self.child_watches[path] = self.child_watches.get(path, []) + [fn]
rval = zookeeper.get_children(self.handle, path, self.zookeeper_watch)
finally:
self.cond.release()
return rval
def node_modify():
path = request.args.get('path', '')
if request.method == 'POST':
data = request.form.get('data')
verify = True
if not zookeeper.exists(zk, CONFIG_NODE / path):
flash('Path already exists:"%s"' % (CONFIG_NODE / path))
verify = False
if verify:
try:
zookeeper.set(zk, CONFIG_NODE / path, data)
return redirect('/node')
except BaseException, e:
flash(e.message)
def main():
data = options.znode_data_size * "D"
zookeeper.set_debug_level(zookeeper.LOG_LEVEL_WARN)
s = zookeeper.init(options.server)
if zookeeper.exists(s, options.root_znode, None):
children = zookeeper.get_children(s, options.root_znode, None)
print "delete old entries: %d" % len(children)
for child in children:
zookeeper.delete(s, "%s/%s" % (options.root_znode, child))
else:
zookeeper.create(s, options.root_znode, "zkpy root znode", acl, 0)
evaluation(s, options.root_znode, data, options)
zookeeper.close(s)
print("Performance test complete")
def watch_children(self, path, fn, clean=False):
if not (path and fn):
raise BadArgumentsException("Invalid path/fn: %s/%s" % (path, fn))
if not zookeeper.exists(self.handle, path):
self.write(path, "")
self.cond.acquire()
try:
if clean:
self.child_watches[path] = []
if not (fn in self.child_watches.get(path, [])):
self.child_watches[path] = self.child_watches.get(path,
[]) + [fn]
finally:
self.cond.release()
rval = zookeeper.get_children(self.handle, path, self.zookeeper_watch)
return rval
def watch_contents(self, path, fn, default_value="", clean=False):
if not (path and fn):
raise BadArgumentsException("Invalid path/fn: %s/%s" % (path, fn))
if not zookeeper.exists(self.handle, path):
self.write(path, default_value)
self.cond.acquire()
try:
if clean:
self.content_watches[path] = []
if not (fn in self.content_watches.get(path, [])):
self.content_watches[path] = self.content_watches.get(
path, []) + [fn]
finally:
self.cond.release()
value, _ = zookeeper.get(self.handle, path, self.zookeeper_watch)
return value
def exists(self, path, watcher=None):
"""Blocking call to check if zookeeper node exists.
Args:
watcher: watcher method to be invoked upon node creation
or removal with Zookeeper.Event as its sole argument.
watcher will be invoked in the context of the underlying
zookeeper API threads.
Returns:
zookeeper state dict if node exists, otherwise None.
i.e.
{'pzxid': 4522L, 'ctime': 1333654407863L, 'aversion': 0, 'mzxid': 4522L,
'numChildren': 0, 'ephemeralOwner': 0L, 'version': 0, 'dataLength': 0,
'mtime': 1333654407863L, 'cversion': 0, 'czxid': 4522L}
Raises:
zookeeper.*Exception for other failure scenarios.
"""
return zookeeper.exists(self.handle, path,
self._watcher_proxy(watcher))
def server_restart():
name = request.args.get('name', '')
if name == '':
return redirect(request.referrer)
if not zookeeper.exists(zk, ONLINE_NODE / name):
flash('"%s" is not online and can not be restart' % (name))
return redirect(request.referrer)
nodevalue = zookeeper.get(zk, ROOT_NODE)
if nodevalue[0] != '':
config = json.loads(nodevalue[0])
config['restart'] = name
else:
config = {'restart': name}
zookeeper.set(zk, ROOT_NODE, json.dumps(config))
flash('Success')
return redirect(request.referrer)