def ListService(self, service_id = None, endpoint = None):
if service_id:
if endpoint:
ep = Endpoint(self.zk, Service(self.zk, service_id), endpoint)
print INFO(datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S] ") + " endpoint is:" + endpoint + " status is:" + ep.Status() + " healthy is:" + ep.Healthy())
return ep
else:
service = Service(self.zk, service_id)
service.Load()
service.ShowStatus()
return service
else:
zkinfolog = OpenZkLog("./log.txt")
departments = zookeeper.get_children(self.zk, '/com/renren/xoa', None)
CloseZkLog(zkinfolog)
service_ids = []
for department in departments:
dp_path = '/com/renren/xoa/' + department
zkinfolog = OpenZkLog("./log.txt")
services = zookeeper.get_children(self.zk, dp_path, None)
CloseZkLog(zkinfolog)
for service in services:
service_id = GetServiceIdByPath(dp_path + '/' + service)
print INFO(datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S] ") + "service id is:" + service_id)
service_ids.append(service_id)
return service_ids
def __init__(self, zkhosts, root=NODE_HQ_ROOT, alivenode='alive'):
"""zkhosts: a string or a list. list will be ','.join-ed into a string.
root: root node path (any parents must exist, if any)
"""
if not isinstance(zkhosts, basestring):
zkhosts = ','.join(zkhosts)
self.zkhosts = zkhosts
self.ROOT = root
self.alivenode = alivenode
self.nodename = os.uname()[1]
self.NODE_SERVERS = self.ROOT + '/servers'
self.NODE_ME = self.NODE_SERVERS+'/'+self.nodename
self.zh = zk.init(self.zkhosts, self.__watcher)
self.jobs = {}
self.initialize_node_structure()
if not zk.exists(self.zh, self.NODE_ME):
zk.create(self.zh, self.NODE_ME, '', PERM_WORLD)
# setup notification
zk.get_children(self.zh, self.NODE_SERVERS, self.__servers_watcher)
#zk.get_children(self.zh, self.NODE_ME, self.__watcher)
self.NODE_JOBS = self.NODE_ME+'/jobs'
zk.acreate(self.zh, self.NODE_JOBS, '', PERM_WORLD)
def __init__(self, service_name, connect_string, timeout=DEFAULT_TIMEOUT, default_port=6664):
self.SERVICE_NODE = "/" + service_name
self.AVAILABILITY_NODE = self.SERVICE_NODE + "/available"
self.MEMBERSHIP_NODE = self.SERVICE_NODE + "/members"
self.connected = False
self.timeout = timeout
self.default_port = default_port
self.conn_cv = threading.Condition()
self.conn_cv.acquire()
self.handle = zookeeper.init(connect_string, self.connection_watcher, timeout)
self.conn_cv.wait(timeout / 1000)
self.conn_cv.release()
self.watcher_lock = threading.Lock()
self.logger = logging.getLogger("sincc")
if not self.connected:
raise SinClusterClientError("Unable to connect to %s" % connect_string)
for path in [self.SERVICE_NODE, self.AVAILABILITY_NODE, self.MEMBERSHIP_NODE]:
if not zookeeper.exists(self.handle, path):
zookeeper.create(self.handle, path, "", [ZOO_OPEN_ACL_UNSAFE], 0)
self.listeners = []
# Start to watch both /members and /available
zookeeper.get_children(self.handle, self.MEMBERSHIP_NODE, self.watcher)
available = zookeeper.get_children(self.handle, self.AVAILABILITY_NODE, self.watcher)
self.available_nodes = {}
for node_id in available:
self.available_nodes[int(node_id)] = Node(int(node_id),
zookeeper.get(self.handle, self.AVAILABILITY_NODE + "/" + node_id)[0])
def notifyFailedTransaction(self, app_id, txid):
""" Notify failed transaction id.
This method will add the transaction id into black list.
After this call, the transaction becomes invalid.
"""
self.__waitForConnect()
self.checkTransaction(app_id, txid)
print "notify failed transaction app:%s, txid:%d" % (app_id, txid)
txpath = self.__getTransactionPath(app_id, txid)
lockpath = None
try:
lockpath = zookeeper.get(self.handle, PATH_SEPARATOR.join([txpath, TX_LOCK_PATH]), None)[0]
except zookeeper.NoNodeException:
# there is no lock. it means there is no need to rollback.
pass
if lockpath:
# add transacion id to black list.
now = str(time.time())
broot = self.__getBlacklistRootPath(app_id)
if not zookeeper.exists(self.handle, broot):
self.__forceCreatePath(broot)
zookeeper.acreate(self.handle, PATH_SEPARATOR.join([broot, str(txid)]), now, ZOO_ACL_OPEN)
# update local cache before notification
if app_id in self.blacklistcache:
with self.blacklistcv:
self.blacklistcache[app_id].add(str(txid))
# copy valid transaction id for each updated key into valid list.
for child in zookeeper.get_children(self.handle, txpath):
if re.match("^" + TX_UPDATEDKEY_PREFIX, child):
value = zookeeper.get(self.handle, PATH_SEPARATOR.join([txpath, child]), None)[0]
valuelist = value.split(PATH_SEPARATOR)
key = urllib.unquote_plus(valuelist[0])
vid = valuelist[1]
vtxroot = self.__getValidTransactionRootPath(app_id)
if not zookeeper.exists(self.handle, vtxroot):
self.__forceCreatePath(vtxroot)
vtxpath = self.__getValidTransactionPath(app_id, key)
zookeeper.acreate(self.handle, vtxpath, vid, ZOO_ACL_OPEN)
# release the lock
try:
zookeeper.adelete(self.handle, lockpath)
except zookeeper.NoNodeException:
# this should be retry.
pass
# just remove transaction node
try:
for item in zookeeper.get_children(self.handle, txpath):
zookeeper.adelete(self.handle, PATH_SEPARATOR.join([txpath, item]))
zookeeper.adelete(self.handle, txpath)
except zookeeper.NoNodeException:
# something wrong. next GC will take care of it.
return False
return True
def notifyFailedTransaction(self, app_id, txid):
""" Notify failed transaction id.
This method will add the transaction id into black list.
After this call, the transaction becomes invalid.
"""
self.__waitForConnect()
self.checkTransaction(app_id, txid)
print "notify failed transaction app:%s, txid:%d" % (app_id, txid)
txpath = self.__getTransactionPath(app_id, txid)
lockpath = None
try:
lockpath = zookeeper.get(self.handle, PATH_SEPARATOR.join([txpath, TX_LOCK_PATH]), None)[0]
except zookeeper.NoNodeException:
# there is no lock. it means there is no need to rollback.
pass
if lockpath:
# add transacion id to black list.
now = str(time.time())
broot = self.__getBlacklistRootPath(app_id)
if not zookeeper.exists(self.handle, broot):
self.__forceCreatePath(broot)
zookeeper.acreate(self.handle, PATH_SEPARATOR.join([broot, str(txid)]), now, ZOO_ACL_OPEN)
# update local cache before notification
if app_id in self.blacklistcache:
with self.blacklistcv:
self.blacklistcache[app_id].add(str(txid))
# copy valid transaction id for each updated key into valid list.
for child in zookeeper.get_children(self.handle, txpath):
if re.match("^" + TX_UPDATEDKEY_PREFIX, child):
value = zookeeper.get(self.handle, PATH_SEPARATOR.join([txpath, child]), None)[0]
valuelist = value.split(PATH_SEPARATOR)
key = urllib.unquote_plus(valuelist[0])
vid = valuelist[1]
vtxroot = self.__getValidTransactionRootPath(app_id)
if not zookeeper.exists(self.handle, vtxroot):
self.__forceCreatePath(vtxroot)
vtxpath = self.__getValidTransactionPath(app_id, key)
zookeeper.acreate(self.handle, vtxpath, vid, ZOO_ACL_OPEN)
# release the lock
try:
zookeeper.adelete(self.handle, lockpath)
except zookeeper.NoNodeException:
# this should be retry.
pass
# just remove transaction node
try:
for item in zookeeper.get_children(self.handle, txpath):
zookeeper.adelete(self.handle, PATH_SEPARATOR.join([txpath, item]))
zookeeper.adelete(self.handle, txpath)
except zookeeper.NoNodeException:
# something wrong. next GC will take care of it.
return False
return True
def handle_membership_changed(self):
members = zookeeper.get_children(self.handle, self.MEMBERSHIP_NODE, self.watcher)
# No need to watch /available here.
available = zookeeper.get_children(self.handle, self.AVAILABILITY_NODE)
self.available_nodes.clear()
for node_id in members:
if node_id in available:
self.available_nodes[int(node_id)] = Node(int(node_id),
zookeeper.get(self.handle, self.AVAILABILITY_NODE + "/" + node_id)[0])
self.notify_all()
def get_job_list(zk):
root_list = zookeeper.get_children(zk, '/')
job_list = {}
job_list['data'] = []
for root in root_list:
if root == "zookeeper":
continue
job_name = zookeeper.get_children(zk, '/' + root)
for job in job_name:
temp = {'{#JOBPATH}':"/%s/%s" % (root,job)}
job_list['data'].append(temp)
job_list_json = json.dumps(job_list, indent=4, sort_keys=True)
print job_list_json
def get_politicians(self, watcher=None, cell=None):
if cell is None:
cell = self._cell
if watcher is None:
children = zookeeper.get_children(self._z, '%s/politicians' % (cell))
else:
children = zookeeper.get_children(self._z, '%s/politicians' % (cell),
watcher)
children = sorted(children)
politicians = list()
for child in children:
politicians.append('%s/politicians/%s' % (cell, child))
return sorted(politicians)
def zookeeper_watch(self, zh, event, state, path):
self.cond.acquire()
try:
result = None
if event == zookeeper.CHILD_EVENT:
fns = self.child_watches.get(path, None)
if fns:
try:
result = zookeeper.get_children(self.handle, path, self.zookeeper_watch)
except zookeeper.NoNodeException:
result = None
elif event == zookeeper.CHANGED_EVENT:
fns = self.content_watches.get(path, None)
if fns:
try:
result, _ = zookeeper.get(self.handle, path, self.zookeeper_watch)
except zookeeper.NoNodeException:
result = None
else:
return
if result != None and fns != None:
for fn in fns:
# Don't allow an individual watch firing an exception to
# prevent all other watches from being fired. Just log an
# error message and moved on to the next callback.
try:
fn(result)
except Exception:
logging.exception("Error executing watch for %s.", path)
finally:
self.cond.release()
def __receiveNotify(self, handle, type, state, path):
# print "notify type:%s, state:%s, path:%s" % (type, state, path)
if type == zookeeper.SESSION_EVENT:
if state == zookeeper.CONNECTED_STATE:
# connected
with self.connectcv:
self.connected = True
self.connectcv.notifyAll()
else:
# disconnected
with self.connectcv:
self.connected = False
self.connectcv.notifyAll()
elif type == zookeeper.CHILD_EVENT:
pathlist = path.split(PATH_SEPARATOR)
if pathlist[-1] == TX_BLACKLIST_PATH:
# update blacklist cache
appid = urllib.unquote_plus(pathlist[-3])
try:
blist = zookeeper.get_children(self.handle, path, self.__receiveNotify)
# print "update blacklist: ",blist
with self.blacklistcv:
self.blacklistcache[appid] = set(blist)
except zookeeper.NoNodeException:
if appid in self.blacklistcache:
with self.blacklistcv:
del self.blacklistcache[appid]
def zookeeper_watch(self, zh, event, state, path):
self.cond.acquire()
try:
if event == zookeeper.CHILD_EVENT:
fns = self.child_watches.get(path, None)
elif event == zookeeper.CHANGED_EVENT:
fns = self.content_watches.get(path, None)
else:
fns = None
finally:
self.cond.release()
result = None
try:
if fns and event == zookeeper.CHILD_EVENT:
result = zookeeper.get_children(self.handle, path,
self.zookeeper_watch)
elif fns and event == zookeeper.CHANGED_EVENT:
result, _ = zookeeper.get(self.handle, path,
self.zookeeper_watch)
except zookeeper.NoNodeException:
pass
if result != None and fns != None:
for fn in fns:
# Don't allow an individual watch firing an exception to
# prevent all other watches from being fired. Just log an
# error message and moved on to the next callback.
try:
fn(result)
except Exception:
logging.exception("Error executing watch for %s.", path)
def list_problematic_nodes (zh, path, options):
path = path.replace('//', '/')
try:
children = list(zookeeper.get_children(zh, path))
if len(children) <= 0:
return
if options.ALONE and 1 == len(children):
yield path + '/' + children[0]
return
if options.NUM > len(children):
logging.info("SKIP only %d nodes in %s", len(children), path)
return
if options.SMART:
children.sort()
# check for services registered several times
group = {}
for n in children:
n = path + '/' + n
data, meta = tuple(zookeeper.get(zh, n))
print repr(data), repr(meta)
if data in group:
yield group[data]
group[data] = n;
else:
# systematical removal
for n in children:
yield path + '/' + n
except Exception as e:
logging.warn("ERROR list %s: %s", path, e)
def find(node_name):
global zh
yield node_name
children = zookeeper.get_children(zh, node_name)
for c in children:
for f in find(os.path.join(node_name, c)):
yield f
def _take_action(self, parsed_args):
import zookeeper
from oio.zk.client import get_meta0_paths, get_connected_handles
self.logger.debug("Checking meta0 services")
# TODO: tcp touch to the meta0 services
# check they are registered in the ZK
for zh in get_connected_handles(self.zookeeper()):
for p in get_meta0_paths(zh, self.app.options.ns):
try:
registered = set()
for n in zookeeper.get_children(zh.get(), p):
v, m = zookeeper.get(zh.get(), p + '/' + n)
registered.add(v)
known = set()
for t, i, p, s in self.filter_services(
self.catalog, 'meta0'):
known.add('%s:%d' % (i, p))
self.logger.info("meta0 known=%d zk_registered=%d",
len(known), len(registered))
assert registered == known
except Exception as ex:
self.logger.exception(
"Failed to list the meta0 services "
"from zookeeper: %s", ex)
finally:
zh.close()
yield ('OK', None)
def list(self, path=''):
path = '%s%s' % (self.prefix, path)
try:
return zookeeper.get_children(self.handle, path, None)
except:
self.log.exception('Failed to fetch subkeys for %s', path)
return []
def zookeeper_delete_node(zk, path):
children = zookeeper.get_children(zk, path)
for child in children:
zookeeper_delete_node(zk, path / child)
zookeeper.delete(zk, path)
def releaseLock(self, app_id, txid, key = None):
""" Release acquired lock.
You must call acquireLock() first.
if the transaction is not valid or it is expired, this raises Exception.
After the release lock, you could not use transaction ID again.
If there is no lock, this method returns False.
"""
self.__waitForConnect()
self.checkTransaction(app_id, txid)
txpath = self.__getTransactionPath(app_id, txid)
has_lock = False
try:
lockpath = zookeeper.get(self.handle, PATH_SEPARATOR.join([txpath, TX_LOCK_PATH]), None)[0]
if key:
lockroot = self.__getLockRootPath(app_id, key)
if not lockroot == lockpath:
raise ZKTransactionException(ZKTransactionException.TYPE_DIFFERENT_ROOTKEY, "You can not specify different root entity for release.")
zookeeper.adelete(self.handle, lockpath)
has_lock = True
except zookeeper.NoNodeException:
# there is no lock.
pass
# If the transaction doesn't have active lock or not, we should delete it.
# delete transaction node
for child in zookeeper.get_children(self.handle, txpath):
zookeeper.adelete(self.handle, PATH_SEPARATOR.join([txpath, child]))
zookeeper.adelete(self.handle, txpath)
return has_lock
def getReadLock(self, metadata):
#Acquire read lock on file through ZooKeeper
fn = metadata.filename
print "Trying to acquire read lock for " + self.root + fn
self.zpath = self.createLockNode(fn, 'r')
myLock = self.zpath.rsplit('/',1)[1]
while True:
children = sorted(zookeeper.get_children(self.zh, self.root + fn))
lastLock = ''
for child in children:
try:
if child == myLock:
break
elif zookeeper.get(self.zh, self.root+fn+'/'+child)[0] == 'w':
lastLock = child
except:
pass
if lastLock != '':
def watcher (handle, type, state, path):
self.cv2.acquire()
self.cv2.notify()
self.cv2.release()
self.cv2.acquire()
if zookeeper.exists(self.zh, self.root+fn+'/'+lastLock, watcher):
self.cv2.wait()
self.cv2.release()
else:
break
print "Acquired read lock for " + self.root + fn
def test_sync_getchildren(self):
self.ensureCreated("/zk-python-getchildrentest", flags=0)
self.ensureCreated("/zk-python-getchildrentest/child")
children = zookeeper.get_children(self.handle,
"/zk-python-getchildrentest")
self.assertEqual(len(children), 1,
"Expected to find 1 child, got " + str(len(children)))
def gc_runner(self):
""" Transaction ID garbage collection (GC) runner.
Note: This must be running as separate thread.
"""
logging.info("Starting GC thread.")
while self.gc_running:
if self.connected:
# Scan each application's last GC time.
try:
app_list = zookeeper.get_children(self.handle, APPS_PATH)
for app in app_list:
app_id = urllib.unquote_plus(app)
# App is already encoded, so we should not use
# self.get_app_root_path.
app_path = PATH_SEPARATOR.join([APPS_PATH, app])
self.try_garbage_collection(app_id, app_path)
except zookeeper.NoNodeException:
# There were no nodes for this application.
pass
except zookeeper.OperationTimeoutException, ote:
logging.warning("GC operation timed out while trying to get {0}"\
" with {1}".format(APPS_PATH, str(ote)))
except zookeeper.ZooKeeperException, zk_exception:
logging.error("ZK Exception: {0}".format(zk_exception))
self.reestablish_connection()
return
def find_leaves_first(node_name):
global zh
children = zookeeper.get_children(zh, node_name)
for c in children:
for n in find_leaves_first(os.path.join(node_name, c)):
yield n
yield node_name
def getReadLock(self, metadata):
#Acquire read lock on file through ZooKeeper
fn = metadata.filename
print "Trying to acquire read lock for " + self.root + fn
self.zpath = self.createLockNode(fn, 'r')
myLock = self.zpath.rsplit('/', 1)[1]
while True:
children = sorted(zookeeper.get_children(self.zh, self.root + fn))
lastLock = ''
for child in children:
try:
if child == myLock:
break
elif zookeeper.get(self.zh,
self.root + fn + '/' + child)[0] == 'w':
lastLock = child
except:
pass
if lastLock != '':
def watcher(handle, type, state, path):
self.cv2.acquire()
self.cv2.notify()
self.cv2.release()
self.cv2.acquire()
if zookeeper.exists(self.zh, self.root + fn + '/' + lastLock,
watcher):
self.cv2.wait()
self.cv2.release()
else:
break
print "Acquired read lock for " + self.root + fn
def find_leaves_first(node_name): global zh children = zookeeper.get_children(zh,node_name) for c in children: for n in find_leaves_first(os.path.join(node_name,c)): yield n yield node_name
def __init__(self,queuename, port, is_producer=False):
self.connected = False
self.queuename = "/" + queuename
self.cv = threading.Condition()
zookeeper.set_log_stream(open("/dev/null"))
def watcher(handle,type,state,path):
print "Connected"
self.cv.acquire()
self.connected = True
self.cv.notify()
self.cv.release()
self.cv.acquire()
self.handle = zookeeper.init("localhost:%d" % port, watcher, 10000)
self.cv.wait(10.0)
if not self.connected:
print "Connection to ZooKeeper cluster timed out - is a server running on localhost:%d?" % port
sys.exit()
self.cv.release()
if is_producer:
while True:
try:
zookeeper.create(self.handle,self.queuename,"queue top level", [ZOO_OPEN_ACL_UNSAFE],0)
print "Fila criada, OK"
return
except zookeeper.NodeExistsException:
print "Tratorando filas existentes"
while True:
children = sorted(zookeeper.get_children(self.handle, self.queuename,None))
if len(children) == 0:
(data,stat) = zookeeper.get(self.handle, self.queuename, None)
zookeeper.delete(self.handle, self.queuename, stat["version"])
break
for child in children:
data = self.get_and_delete(self.queuename + "/" + child)
def find(node_name): global zh yield node_name children = zookeeper.get_children(zh,node_name) for c in children: for f in find(os.path.join(node_name,c)): yield f
def LoadEndpoints(self):
group_path = self.GetActiveGroupPath()
try:
group_children = zookeeper.get_children(self.zk, group_path, None);
except zookeeper.NoNodeException, msg:
print ERROR(group_path + " node exist.")
return
def handle_availability_changed(self):
available = zookeeper.get_children(self.handle, self.AVAILABILITY_NODE, self.watcher)
self.available_nodes.clear()
for node_id in available:
self.available_nodes[int(node_id)] = Node(int(node_id),
zookeeper.get(self.handle, self.AVAILABILITY_NODE + "/" + node_id)[0])
self.notify_all()
def __receiveNotify(self, handle, type, state, path):
# print "notify type:%s, state:%s, path:%s" % (type, state, path)
if type == zookeeper.SESSION_EVENT:
if state == zookeeper.CONNECTED_STATE:
# connected
with self.connectcv:
self.connected = True
self.connectcv.notifyAll()
else:
# disconnected
with self.connectcv:
self.connected = False
self.connectcv.notifyAll()
elif type == zookeeper.CHILD_EVENT:
pathlist = path.split(PATH_SEPARATOR)
if pathlist[-1] == TX_BLACKLIST_PATH:
# update blacklist cache
appid = urllib.unquote_plus(pathlist[-3])
try:
blist = zookeeper.get_children(self.handle, path, self.__receiveNotify)
# print "update blacklist: ",blist
with self.blacklistcv:
self.blacklistcache[appid] = set(blist)
except zookeeper.NoNodeException:
if appid in self.blacklistcache:
with self.blacklistcv:
del self.blacklistcache[appid]
def delete_tree (zh, path):
path = path.replace('//', '/')
try:
for n in tuple(zookeeper.get_children(zh, path)):
delete_tree(zh, path + '/' + n)
zookeeper.delete(zh, path)
except:
pass
def list_children(self, path):
"""
Returns a list of all the children nodes in the path. None is returned if the path does
not exist.
"""
if zookeeper.exists(self.handle, path):
value = zookeeper.get_children(self.handle, path)
return value
def _recursive_delete(handle, root, dry_run=False):
children = zookeeper.get_children(handle, root)
for child in children:
path = root + "/" + child
_recursive_delete(handle, path, dry_run=dry_run)
LOG.debug("Deleting node " + root)
if not dry_run:
zookeeper.delete(handle, root)
def delete_recursive(handle, path):
try:
children = zookeeper.get_children(handle, path)
for child in children:
delete_recursive(handle, PATH_SEPARATOR.join([path, child]))
zookeeper.delete(handle, path, -1)
except zookeeper.NoNodeException:
pass
def test_sync_delete(self):
ZOO_OPEN_ACL_UNSAFE = {"perms":0x1f, "scheme":"world", "id" :"anyone"}
self.assertEqual(self.connected, True)
ret = zookeeper.create(self.handle, "/zk-python-deletetest", "nodecontents", [ZOO_OPEN_ACL_UNSAFE], zookeeper.EPHEMERAL)
self.assertEqual(ret, "/zk-python-deletetest")
ret = zookeeper.delete(self.handle,"/zk-python-deletetest")
self.assertEqual(ret, zookeeper.OK)
children = zookeeper.get_children(self.handle, "/")
self.assertEqual(False, "zk-python-deletetest" in children)
def deleteRecursive(self, path):
self.__waitForConnect()
try:
children = zookeeper.get_children(self.handle, path)
for child in children:
self.deleteRecursive(PATH_SEPARATOR.join([path, child]))
zookeeper.delete(self.handle, path, -1)
except zookeeper.NoNodeException:
pass
def delete_children(zh, path):
path = path.replace('//', '/')
try:
for n in tuple(zookeeper.get_children(zh, path)):
p = path + '/' + n
delete_children(zh, p)
zookeeper.delete(zh, p)
except:
pass
def print_recursive(handle, path):
try:
children = zookeeper.get_children(handle, path)
for child in children:
print_recursive(handle, PATH_SEPARATOR.join([path, child]))
value = zookeeper.get(handle, path)[0]
print "{0} = {1}".format(path, value)
except zookeeper.NoNodeException:
pass
def dequeue(self):
while True:
children = sorted(zookeeper.get_children(self.handle, self.queuename,None))
if len(children) == 0:
return None
for child in children:
data = self.get_and_delete(self.queuename + "/" + children[0])
if data:
return data
def deleteRecursive(self, path):
self.__waitForConnect()
try:
children = zookeeper.get_children(self.handle, path)
for child in children:
self.deleteRecursive(PATH_SEPARATOR.join([path, child]))
zookeeper.delete(self.handle, path, -1)
except zookeeper.NoNodeException:
pass
def _delete_children(zh, path):
logging.debug("Removing %s", path)
try:
for n in tuple(zookeeper.get_children(zh, path)):
p = path + '/' + n
_delete_children(zh, p)
zookeeper.delete(zh, p)
except Exception as ex:
logging.warn("Removal failed: %s", ex)
def LoadEndpoints(self):
group_path = self.GetActiveGroupPath()
try:
zkinfolog = OpenZkLog("./log.txt")
group_children = zookeeper.get_children(self.zk, group_path, None);
CloseZkLog(zkinfolog)
except zookeeper.NoNodeException, msg:
print ERROR(datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S] ") + group_path + " node exist.")
return
def getARset(self):
children = zookeeper.get_children(self.handle, "/root/segment", True)
#for i in range(1, self.nbr_agents + 1):
for child in children:
(data, stat) = zookeeper.get(self.handle,
"/root/segment/" + str(child), True)
#zookeeper keeps things as string, convert to a list
self.agent_registry[child] = eval(data)
def dequeue(self):
while True:
children = sorted(
zookeeper.get_children(self.handle, self.queuename, None))
if len(children) == 0:
return None
for child in children:
data = self.get_and_delete(self.queuename + "/" + children[0])
if data:
return data
def get_node_tree(node_key):
if node_key == "/":
for node in zookeeper.get_children(zk,node_key):
key = "/" + node
if (zookeeper.get(zk,key)[1])['numChildren'] > 0:
get_node_tree(key)
print key
else:
print key
else:
for node in zookeeper.get_children(zk,node_key):
key = node_key + "/" + node
if (zookeeper.get(zk,key)[1])['numChildren'] > 0:
get_node_tree(key)
data.append(key)
else:
data.append(key)
return data
def dumpTree(self, path):
self.__waitForConnect()
try:
value = zookeeper.get(self.handle, path, None)[0]
print "%s = \"%s\"" % (path, value)
children = zookeeper.get_children(self.handle, path)
for child in children:
self.dumpTree(PATH_SEPARATOR.join([path, child]))
except zookeeper.NoNodeException:
print "%s is not exist." % path
def _delete_children(zh, path, logger):
logger.debug("Removing %s", path)
for n in tuple(zookeeper.get_children(zh, path)):
p = path + '/' + n
_delete_children(zh, p, logger)
try:
zookeeper.delete(zh, p)
logger.info('Deleted %s', p)
except Exception as ex:
logger.warn("Removal failed on %s: %s", p, ex)
def dumpTree(self, path):
self.__waitForConnect()
try:
value = zookeeper.get(self.handle, path, None)[0]
print "%s = \"%s\"" % (path, value)
children = zookeeper.get_children(self.handle, path)
for child in children:
self.dumpTree(PATH_SEPARATOR.join([path, child]))
except zookeeper.NoNodeException:
print "%s is not exist." % path
def _list_nodes(self, zh, path):
path = path.replace('//', '/')
try:
children = list(zookeeper.get_children(zh, path))
if len(children) <= 0:
return
for child in children:
yield child
except Exception as e:
self.log.warn("ERROR list %s: %s", path, e)
def _getThriftServerFromZookeeper(self):
self.hd = zookeeper.init("%s:%s" % (self.zookeeper_host, self.zookeeper_port))
self.children = zookeeper.get_children(self.hd, self.hbase_thrift_znode)
# random get a thrift server
import random
index = random.randint(0, len(self.children)-1)
(self.thrift_host, self.thrift_port) = self.children[index].split(':')
log.msg("Thrift server got from zookeeper:[%s:%s]" % \
(self.thrift_host, self.thrift_port))
zookeeper.close(self.hd)
def master(self):
"""
get children, and check who is the smallest child
"""
@watchmethod
def watcher(event):
self.say("child changed, try to get master again.")
self.master()
self.children = zookeeper.get_children(self.zk,'/redis_master', watcher)
self.children.sort()
logger.debug("%s's children: %s" % ('/redis_master', self.children))
def block_dequeue(self):
"""
Similar to dequeue, but if the queue is empty, block until an item
is added and successfully removed.
"""
while True:
children = sorted(zookeeper.get_children(self.handle, self.queuename, self.__queueWatcher__))
for child in children:
data = self.get_and_delete(self.queuename + "/" + child)
if data != None:
return data
def delete_zookeeper(zk, path):
try:
zk_children = zookeeper.get_children(zk, path)
if len(zk_children) > 0:
for tp in zk_children:
tp = path + "/" + tp
delete_zookeeper(zk, tp)
zookeeper.delete(zk, path)
except zookeeper.NodeExistsException, e:
err_str = "path %s exist\n" % path
handle_output(False, err_str)
def reset(self):
"""Reset both MEMBERSHIP_NODE and AVAILABILITY_NODE to empty nodes."""
nodes = zookeeper.get_children(self.handle, self.MEMBERSHIP_NODE)
for node_id in nodes:
path = self.MEMBERSHIP_NODE + "/" + node_id
try:
zookeeper.delete(self.handle, path)
self.logger.info("Node %s is removed (because of reset)" % node_id)
except zookeeper.NoNodeException:
self.logger.warn("%s does not exist" % path)
def get_zk_data():
zk = zookeeper.init("zk.staging.srv:8080") # 初始化zookeeper
try:
zk_children = zookeeper.get_children(zk, "/company/cash/jira")
# 如果test节点存在,则不会运行config节点,方便对新配置进行测试
if 'test' in zk_children:
zk_test = zookeeper.get(zk, "/company/cash/jira/test")
if zk_test[0] is not '':
print '测试节点不为空,将使用测试节点的配置...'
return zk_test[0]
except Exception, e:
print e.message
def get_registered_nodes(self):
"""Get all registered nodes."""
nodes = {}
try:
members = zookeeper.get_children(self.handle, self.MEMBERSHIP_NODE)
for node_id in members:
nodes[int(node_id)] = Node(int(node_id),
zookeeper.get(self.handle, self.MEMBERSHIP_NODE + "/" + node_id)[0])
except:
pass
return nodes
def watcher_sms(self, h_zk, w_type, w_stat, w_path):
'''
监听 agent/slaverX/smsX
watch回调
h_zk:就是我们创建连接之后的返回值,我试了下,发现好像指的是连接的一个索引值,以0开始
w_type:事件类型,-1表示没有事件发生,1表示创建节点,2表示节点删除,3表示节点数据被改变,4表示子节点发生变化
w_state:客户端的状态,0:断开连接状态,3:正常连接的状态,4认证失败,-112:过期啦
w_path:这个状态就不用解释了,znode的路径
'''
sms_name = os.path.basename(w_path)
log = "%s watche sms change:t:%d s:%d p:%s" % (self.server_node_name,
w_type, w_stat, w_path)
self.loger.info(log)
if w_type == 3:
new_vaule = zookeeper.get(h_zk, w_path, self.watcher_sms)
self.agent_stat = new_vaule
msg_txt = '{"%s":["change","%s"]}' % (sms_name, new_vaule[0])
cur_role = getLocalhostRole()
log = 'cur role %s' % cur_role
self.loger.info(log)
if cur_role and cur_role != 'follower':
self.msg_queue.put(msg_txt) # 交由上层AgentMgr.process处理
else:
self.loger.info(msg_txt)
elif w_type == 2:
msg_txt = '{"%s":["delete"]}' % (sms_name)
cur_role = getLocalhostRole()
log = 'cur role %s' % cur_role
self.loger.info(log)
if cur_role and cur_role != 'follower':
self.msg_queue.put(msg_txt) # 交由上层AgentMgr.process处理
else:
self.loger.info(msg_txt)
elif w_type == 4: #如果等于4，则sms_name 的内容为‘sms’，不能使用sms_name
sms_ls = zookeeper.get_children(h_zk, w_path, self.watcher_sms)
bexist = False
#检查是否存在属于此主机下的sms
for sms in sms_ls:
agent, smsid = sms.split('@')
if agent == self.agent_name:
bexist = True
if bexist:
msg_txt = '{"%s":["child_change"]}' % self.agent_name
cur_role = getLocalhostRole()
log = 'cur role %s' % cur_role
self.loger.info(log)
if cur_role and cur_role != 'follower':
self.msg_queue.put(msg_txt) # 交由上层AgentMgr.process处理
else:
self.loger.info(msg_txt)
def list_nodes(zh, path, options):
path = path.replace('//', '/')
try:
children = list(zookeeper.get_children(zh, path))
if len(children) >= options.CHILDREN:
print "CHILDREN", len(children), path
for child in children:
n = path + '/' + child
_, meta = tuple(zookeeper.get(zh, n))
yield child, meta
except Exception as e:
logging.warn("ERROR list %s: %s", path, e)
