def __init__(self, verbose=True):
self.VERBOSE = verbose
self.masters = []
self.is_master = False
self.path = None
self.zk = ZKClient(self.ZK_HOST, timeout=self.TIMEOUT)
self.say("login ok!")
# init
self.__init_zk()
# register
self.register()
def __init__(self, hosts, root, count):
self.root = root
self.name = 'barriers'
self.count = count
self.condition = threading.Condition(threading.RLock())
ZKClient.__init__(self, hosts)
self.start()
@self.zk.ChildrenWatch(self.root)
def _on_children_change(children):
Barriers_logger.info('_on_children_change children are[%s]' %
children)
with self.condition:
self.condition.notifyAll()
def __init__(self, hosts, root, name='Lock'):
self.root = root
self.name = name
self.lock_name = None
self.condition = threading.Condition(threading.RLock())
ZKClient.__init__(self, hosts)
self.start()
@self.zk.ChildrenWatch(self.root)
def _on_children_change(children):
Lock_logger.info(
'_on_children_change children are[%s]' % children)
with self.condition:
self.condition.notifyAll()
def __init__(self,
verbose=True,
database='test',
table='targets',
zkhost='127.0.0.1:32181'):
self.VERBOSE = verbose
self.path = None
self.ZK_HOST = zkhost
self.database = "/" + database
self.table = os.path.join(self.database, table)
self.zk = ZKClient(self.ZK_HOST, timeout=self.TIMEOUT)
self.say("login ok")
self._init_zk()
def __init__(self, hosts, root):
self.root = root
self.name = 'queue'
self.condition = threading.Condition(threading.RLock())
ZKClient.__init__(self, hosts)
self.start()
@self.zk.ChildrenWatch(self.root)
def _on_children_change(children):
ZKQueue_logger.info('_on_children_change children are[%s]' %
children)
with self.condition:
self.condition.notifyAll()
self.zklock = ZKLock(hosts='127.0.0.1:2181', root='/my_Lock_test')
def __init__(self, verbose = True, database='/test', table='targets'):
self.VERBOSE = verbose
self.path = None
self.database = database
self.table = os.path.join(self.database, table)
self.zk = ZKClient(self.ZK_HOST, timeout = self.TIMEOUT)
self.say("login ok")
self._init_zk()
def __init__(self, verbose = True, database='test', table='targets', zkhost='127.0.0.1:32181'):
self.VERBOSE = verbose
self.path = None
self.ZK_HOST = zkhost
self.database = "/" + database
self.table = os.path.join(self.database, table)
self.zk = ZKClient(self.ZK_HOST, timeout = self.TIMEOUT)
self.say("login ok")
self._init_zk()
def __init__(self, verbose = True):
self.VERBOSE = verbose
self.path = None
self.is_master = True
self.zk = ZKClient(self.ZK_HOST, timeout = self.TIMEOUT)
self.say("login ok!")
# init
self.__init_zk()
# register
self.register()
def __init__(self, verbose = True):
self.VERBOSE = verbose
self.path = None
self.APP_ROOT = "/Applications"
self.APP_CONF = join(self.APP_ROOT,"NginxConf")
self.zk = ZKClient(self.ZK_HOST, timeout = self.TIMEOUT)
self.say("login zookeeper successfully!")
# init
self.create_roles_znode()
# register
self.register()
self.watch_app_znode()
def alert():
zk_dst = ZKClient(ZK_ADDR)
stat = dict()
redis_conn= []
title = zk_dst.get_children(MQ_PATH+"msgmap")
shard = zk_dst.get_children(MQ_PATH+"redis")
for i in shard:
child=zk_dst.get_children(MQ_PATH+"redis/"+i+"/.service_nodes")
for k in child:
addr=k.split(':')
redis_conn.append(redis.Redis(addr[0], int(addr[1])))
flag = False
for i in title:
for j in redis_conn:
if not stat.has_key(i):
stat[i]=0
len = j.llen(i)
stat[i] += len
for i in stat:
if stat[i]>ALERT_MSG_NUM:
flag=True
break
if not flag:
print "无积压消息"
return
subject = "MQ积压消息报警"
body = '\n'
body += time.asctime() + "\n"
body += 'Title \t Num' + "\n"
for i in stat:
body += 'title:' + i + '\t len:' + str(stat[i])+'\n'
send_mail(STAT_MAIL_TO, subject, body)
def set_tcp_keepalive(sock, opts=None):
# Warn if ZMQ < 3.2
try:
zmq_version_info = zmq.zmq_version_info()
except AttributeError:
# PyZMQ <= 2.1.9 does not have zmq_version_info, fall back to
# using zmq.zmq_version() and build a version info tuple.
zmq_version_info = tuple(
[int(x) for x in zmq.zmq_version().split('.')]
)
if zmq_version_info < (3, 2):
log.warning(
'You have a version of ZMQ less than ZMQ 3.2! There are '
'known connection keep-alive issues with ZMQ < 3.2 which '
'may result in loss of contact with minions. Please '
'upgrade your ZMQ!'
)
####
# try:
# ip=socket.gethostbyname(socket.gethostname())
# if ip == "127.0.0.1":
# raise Exception("127.0.0.1")
# except Exception,e:
# (status,ip) = commands.getstatusoutput("/sbin/ifconfig -a|grep inet|grep -v 127.0.0.1|grep -v inet6|awk '{print $2}'|tr -d 'addr:'")
# ip = ip
if opts['ha'] == True:
while True:
time.sleep(5)
try:
sys.path.append("/usr/lib/python2.7/site-packages/salt")
from zkclient import ZKClient, zookeeper, watchmethod
zk = ZKClient('172.16.163.14:2181',1000)
if zk.exists('/broker'):
value = zk.get('/broker')
logging.error(str(value[0]))
if value[0] == opts['id']:
break
else:
zk.create('/broker')
zk.set('/broker',data='%s'%opts['id'])
logging.error("create the zk path is successful !")
continue
except Exception,e:
logging.error(str(e))
continue
class ZookeeperStore(object):
TIMEOUT = 1000
def __init__(self,
verbose=True,
database='test',
table='targets',
zkhost='127.0.0.1:32181'):
self.VERBOSE = verbose
self.path = None
self.ZK_HOST = zkhost
self.database = "/" + database
self.table = os.path.join(self.database, table)
self.zk = ZKClient(self.ZK_HOST, timeout=self.TIMEOUT)
self.say("login ok")
self._init_zk()
def __del__(self):
self.zk.close()
def _init_zk(self):
nodes = (self.database, self.table)
for node in nodes:
if not self.zk.exists(node):
try:
self.zk.create(node, "")
except:
pass
def create(self, path, msg, flags=0):
msg = self.encode(msg)
self.path = os.path.join(self.table, path)
nodes = self.path.split("/")
# FIXME
for i in range(4, len(nodes)):
node = "/".join(nodes[:i])
if not self.zk.exists(node):
try:
self.zk.create(node, "")
except:
pass
if not self.zk.exists(self.path):
self.path = self.zk.create(self.path, msg, flags=flags)
else:
self.zk.set(self.path, msg)
#self.path = os.path.basename(self.path)
self.say("register ok! I'm %s" % self.path)
def get_children(self, path=""):
if not path:
path = self.table
children = self.zk.get_children(path)
return children
def get(self, node):
msg = self.zk.get(node)
return msg
def decode(self, obj):
return obj
def encode(self, msg):
return msg
def say(self, msg):
if self.VERBOSE:
if self.path:
log.info(msg)
else:
log.info(msg)
log_file_name = log_path + "/zk_client.log"
if os.path.exists(log_path):
if not os.path.isdir(log_path):
os.remove(log_path)
os.makedirs(log_path)
else:
os.makedirs(log_path)
fp = logging.handlers.RotatingFileHandler(log_file_name, maxBytes=10*1024*1024, mode="a", backupCount=100)
logger.addHandler(fp)
formatter = logging.Formatter("[%(asctime)s] [%(levelname)s] [%(filename)s] [%(lineno)d] - %(message)s")
fp.setFormatter(formatter)
logger.setLevel(logging.NOTSET)
return logger
#每个节点启动时去/work节点注册自身ip
zk = ZKClient(config.server_list)
print "dir zk", dir(zk)
#try:
# ret = zk.delete("/worker")
# print "delete ret:", ret
#except:
# pass
#try:
# ret = zk.create("/worker", "")
# print "create ret:", ret
#except Exception, reason:
# print "create failed", reason
# pass
try:
ret = zk.create("/worker/work", "100", zookeeper.EPHEMERAL | zookeeper.SEQUENCE)
print "create ret:", ret
class GJZookeeper(object):
ZK_HOST = "localhost:2181"
ROOT = "/app"
WORKERS_PATH = join(ROOT, "workers")
MASTERS_NUM = 1
TIMEOUT = 10000
def __init__(self, verbose = True):
self.VERBOSE = verbose
self.masters = []
self.is_master = False
self.path = None
self.zk = ZKClient(self.ZK_HOST, timeout = self.TIMEOUT)
self.say("login ok!")
# init
self.__init_zk()
# register
self.register()
def __init_zk(self):
"""
create the zookeeper node if not exist
"""
nodes = (self.ROOT, self.WORKERS_PATH)
for node in nodes:
if not self.zk.exists(node):
try:
self.zk.create(node, "")
except:
pass
@property
def is_slave(self):
return not self.is_master
def register(self):
"""
register a node for this worker
"""
self.path = self.zk.create(self.WORKERS_PATH + "/worker", "1", flags=zookeeper.EPHEMERAL | zookeeper.SEQUENCE)
self.path = basename(self.path)
self.say("register ok! I'm %s" % self.path)
# check who is the master
self.get_master()
def get_master(self):
"""
get children, and check who is the smallest child
"""
@watchmethod
def watcher(event):
self.say("child changed, try to get master again.")
self.get_master()
children = self.zk.get_children(self.WORKERS_PATH, watcher)
children.sort()
self.say("%s's children: %s" % (self.WORKERS_PATH, children))
# check if I'm master
self.masters = children[:self.MASTERS_NUM]
if self.path in self.masters:
self.is_master = True
self.say("I've become master!")
else:
self.say("%s is masters, I'm slave" % self.masters)
def say(self, msg):
"""
print messages to screen
"""
if self.VERBOSE:
if self.path:
if self.is_master:
log.info("[ %s(%s) ] %s" % (self.path, "master" , msg))
else:
log.info("[ %s(%s) ] %s" % (self.path, "slave", msg))
else:
log.info(msg)
for cb in callbacks:
cb.waitForSuccess()
timer2(func, "deleted %7d permanent znodes " % (options.znode_count))
def get_zk_servers():
return options.cluster
if __name__ == '__main__':
data = options.znode_size * "x"
servers = get_zk_servers()
session_gets = []
for x in xrange(options.watch_session):
session_gets.append(ZKClient(servers, options.timeout))
session_set = ZKClient(servers, options.timeout)
session_set.create(options.root_znode,
"smoketest root, delete after test done, created %s" % (datetime.datetime.now().ctime()))
create_nodes(session_set)
for s in session_gets:
add_wathches(s)
set_nodes(session_set, data)
delete_nodes(session_set)
session_set.delete(options.root_znode)
time.sleep(10)
for s in session_gets:
s.close()
session_set.close()
class GJZookeeper(object):
ZK_HOST = "localhost:2181"
ROOT = "/Roles"
WORKERS_PATH = join(ROOT, "agents")
MASTERS_NUM = 1
TIMEOUT = 10000
def __init__(self, verbose = True):
self.VERBOSE = verbose
self.masters = []
self.is_master = False
self.path = None
self.APP_ROOT = "/Applications"
self.APP_CONF = join(self.APP_ROOT,"NginxConf")
self.zk = ZKClient(self.ZK_HOST, timeout = self.TIMEOUT)
self.say("login zookeeper successfully!")
# init
self.create_roles_znode()
# register
self.register()
def create_roles_znode(self):
"""
create the zookeeper node if not exist
|-Roles
|-workers
"""
nodes = (self.ROOT, self.WORKERS_PATH)
for node in nodes:
if not self.zk.exists(node):
try:
self.zk.create(node, "")
except:
pass
@property
def is_slave(self):
return not self.is_master
def register(self):
"""
register a node for this worker,znode type : EPHEMERAL | SEQUENCE
|-Roles
|-workers
|-worker000000000x ==>>master
|-worker000000000x+1 ==>>worker
....
"""
self.path = self.zk.create(self.WORKERS_PATH + "/worker", "1", flags=zookeeper.EPHEMERAL | zookeeper.SEQUENCE)
self.path = basename(self.path)
self.say("I'm %s" % self.path)
# check who is the master
self.get_master()
def get_master(self):
"""
get children, and check who is the smallest child
"""
@watchmethod
def watcher(event):
self.say("child changed, try to get master again.")
self.get_master()
children = self.zk.get_children(self.WORKERS_PATH, watcher)
children.sort()
self.say("%s's children: %s" % (self.WORKERS_PATH, children))
# check if I'm master
self.masters = children[:self.MASTERS_NUM]
if self.path in self.masters:
self.is_master = True
self.say("I've become master!")
self.create_app_znode()
else:
self.say("%s is masters, I'm slave" % self.masters)
def create_app_znode(self):
"""
create the zookeeper node if not exist
|-Applications
|-NginxConf
"""
nodes = (self.APP_ROOT, self.APP_CONF)
for node in nodes:
if not self.zk.exists(node):
try:
self.say("Create znode [%s] ..."%(node))
self.zk.create(node, "")
except:
pass
def create_conf_znode(self, data):
"""
create the zookeeper node's children if not exist,contents is conf data
|-Applications
|-NginxConf
|-item-000000000x => data
"""
self.child_node = join(self.APP_CONF, "conf-")
path = self.zk.create(self.child_node, data, flags=zookeeper.SEQUENCE)
self.say("create znode %s"%path)
def say(self, msg):
"""
print messages to screen
"""
if self.VERBOSE:
if self.path:
log.info("[ %s(%s) ] %s" % (self.path, "master" if self.is_master else "slave", msg))
else:
log.info(msg)
pool.join()
print("Duration: ", str(time.time() - startTime))
print("Total writes: ", total_writes)
if __name__ == '__main__':
data = options.znode_size * "x"
servers = options.servers.split(",")
# create all the sessions first to ensure that all servers are
# at least available & quorum has been formed. otw this will
# fail right away (before we start creating nodes)
sessions = []
# create sessions to each of the servers in the ensemble
for server in servers:
for i in range(SESSIONS_NUM):
sessions.append(ZKClient(server, options.timeout))
# ensure root_znode doesn't exist
if sessions[0].exists(options.root_znode) and options.type == "create":
# unless forced by user
if not options.force:
raise SmokeError("Node %s already exists!" % (options.root_znode))
children = sessions[0].get_children(options.root_znode)
for child in children:
sessions[0].delete("%s/%s" % (options.root_znode, child))
elif options.type == "create":
sessions[0].create(
options.root_znode,
"smoketest root, delete after test done, created %s" %
(datetime.datetime.now().ctime()))
def help(err_info=""):
info="""
usage: sh rm_rf.sh -s 127.0.0.1:2181 -p /node_path"
"""
print err_info+info
def rm_rf(path):
if(path==None):
help("路径为空\n")
sys.exit(1)
if(not zk.exists(path)):
print "path(%s) is not exists!"%(path)
sys.exit(0)
the_childs=zk.get_children(path)
for a_child in the_childs:
rm_rf(path+"/"+a_child)
zk.delete(path)
pass
if __name__ == '__main__':
help()
if(options.servers is not None):
zk=ZKClient(options.servers, options.timeout)
rm_rf(options.path)
print "rm_rf the node (%s)done!"%(options.path)
else:
help("zookeeper服务为空!\n")
sys.exit(1)
logger.error(title)
def server_down(down_list):
if not down_list:
return
title = "zookeeper warning:检测到zookeeper服务器" + str(down_list) + "宕机"
content = title
email_util.send_mail(config.mail_to, title, content)
logger.error(title)
if __name__ == "__main__":
logger = init_logger()
server_set = config.server_node_ids
client_set = config.client_node_ids
r_dbhd = redis.Redis(host = config.redis_host, port = config.redis_port, db = config.redis_db)
zk = ZKClient(config.server_list)
#此处创建server端监视节点
safe_reg_self(config.server_root_path, config.node_id)
#try:
# l = zk.create(config.root_path + config.node_id, "", config.zknode_type_ephemeral)
# print l
#except Exception, reason:
# print "create node fail:", str(reason)
# logger.error("create node fail:" + str(reason))
# exit()
try:
l = zk.create(config.client_root_path, "{}", 0)
print l
except zookeeper.NodeExistsException, reason:
sys.path.append("../dep/ice/")
import Ice
Ice.loadSlice("../slice/mq.ice")
import com.renren.x2.mq
mq=com.renren.x2.mq
class MqServiceI(mq.MqService):
def forwardMq(self, mid, msg, current=None):
print msg
return 1
if __name__ == "__main__":
zookeeper.set_debug_level(zookeeper.LOG_LEVEL_ERROR)
zk = ZKClient("localhost:2181")
try:
path = zk.create("/3g/ice/x2/MqService/msgmap/test/.service_nodes/127.0.0.1:10015", flags=zookeeper.EPHEMERAL)
except:
pass
try:
ic = Ice.initialize(sys.argv)
addr ="default -h %s -p %d -t 200" % ('127.0.0.1',10015)
adapter = ic.createObjectAdapterWithEndpoints( "MqService", addr)
adapter.add(MqServiceI(), ic.stringToIdentity("MqService"))
adapter.activate()
ic.waitForShutdown()
except Exception,x:
conf.logger.error(str(x))
class GJZookeeper(object):
ZK_HOST = "localhost:2181"
ROOT = "/app"
WORKERS_PATH = join(ROOT, "workers")
MASTERS_NUM = 1
TIMEOUT = 10000
def __init__(self, verbose = True):
self.VERBOSE = verbose
self.path = None
self.is_master = True
self.zk = ZKClient(self.ZK_HOST, timeout = self.TIMEOUT)
self.say("login ok!")
# init
self.__init_zk()
# register
self.register()
def __init_zk(self):
"""
create the zookeeper node if not exist
"""
nodes = (self.ROOT, self.WORKERS_PATH)
for node in nodes:
if not self.zk.exists(node):
try:
self.zk.create(node, "")
except:
pass
def register(self):
@watchmethod
def watcher(event):
self.say("first watcher")
self.path = self.zk.create(self.WORKERS_PATH + "/worker", "1", flags=zookeeper.EPHEMERAL | zookeeper.SEQUENCE)
if not self.zk.exists(self.path, watcher):
print "not exist"
self.zk.set(self.path, "3", -1)
self.zk.exists(self.path, watcher)
self.zk.delete(self.path, -1)
print "_________________________"
def say(self, msg):
"""
print messages to screen
"""
if self.VERBOSE:
if self.path:
log.info("[ %s ] %s" % (self.path , msg))
else:
log.info(msg)
import zkclient
from zkclient import ZKClient
zk = ZKClient("10.187.176.194:2181")
newKeyList = [
"goods_homef~~~brand", "qrqm_hp_cfb~~~brand", "goods_homef~~~c3brand",
"qrqm_hp_cfb~~~c3brand_0", "goods_homef~~~c3", "qrqm_hp_cfb~~~c3",
"qrqm_hp_cfb~~~channelBrand_0", "qrqm_hp_cfb~~~channelC3Brand_0",
"qrqm_hp_cfb~~~channelC3_0", "qrqm_hp_cfb~~~channel", "cd3_fzxp_fs",
"cd3_jdjx_fs", "cd3_jdmarket_fs", "cd3_newsh_fs", "cd3_oversea_fs",
"cd3_7toreturn_fs", "seckill_cfb~~~brand_code",
"seckill_cfb~~~cid3_discount_ratio_cat", "seckill_cfb~~~cid3",
"seckill_cfb~~~cid3_pw3_ids", "seckill_cfb~~~discount_amount_cat",
"seckill_cfb~~~discount_ratio_cat"
]
created = zk.exists("/root/redis")
if created:
print 'deleting /root/redis ...'
oldKeyList = zk.get_children("/root/redis")
for k in oldKeyList:
zk.delete("/root/redis/" + k)
else:
zk.create("/root/redis", "")
for k in newKeyList:
zk.create("/root/redis/" + k, k)
print 'add keys on zk success!'
class GJZookeeper(object):
ZK_HOST = "localhost:2181"
ROOT = "/Roles"
WORKERS_PATH = join(ROOT, "workers")
TIMEOUT = 10000
RULE_PATH = "/opt/interlive/nginx/conf/self_rules.xml"
def __init__(self, verbose = True):
self.VERBOSE = verbose
self.path = None
self.APP_ROOT = "/Applications"
self.APP_CONF = join(self.APP_ROOT,"NginxConf")
self.zk = ZKClient(self.ZK_HOST, timeout = self.TIMEOUT)
self.say("login zookeeper successfully!")
# init
self.create_roles_znode()
# register
self.register()
self.watch_app_znode()
def create_roles_znode(self):
"""
create the zookeeper node if not exist
|-Roles
|-workers
"""
nodes = (self.ROOT, self.WORKERS_PATH)
for node in nodes:
if not self.zk.exists(node):
try:
self.zk.create(node, "")
except:
pass
def register(self):
"""
register a node for this worker,znode type : EPHEMERAL | SEQUENCE
|-Roles
|-workers
|-worker000000000x ==>>master
|-worker000000000x+1 ==>>worker
....
"""
self.path = self.zk.create(self.WORKERS_PATH + "/worker", "1", flags=zookeeper.EPHEMERAL | zookeeper.SEQUENCE)
self.path = basename(self.path)
self.say("I'm %s" % self.path)
def watch_app_znode(self):
@watchmethod
def watcher(event):
self.say("first watcher")
child_nodes = self.zk.get_children(self.APP_CONF)
child_nodes.sort(reverse=True)
data = self.zk.get(self.APP_CONF + "/" + child_nodes[0])
self.say(data[0])
file_object = open(self.RULE_PATH, 'w')
try:
file_object.write(data[0])
finally:
file_object.close()
self.zk.get_children(self.APP_CONF, watcher)
self.zk.get_children(self.APP_CONF, watcher)
def say(self, msg):
"""
print messages to screen
"""
if self.VERBOSE:
if self.path:
log.info("[ %s ] %s" % (self.path, msg))
else:
log.info(msg)
self.cond.release()
kongbai = kongbaige(cond, ' ')
ximi = ximige(cond, '西米')
#尼玛下面这2个启动标志是关键，虽然是空白哥先开的口，但是不能让他先启动，
#因为他先启动的可能直到发完notify通知了，西米哥才开始启动，
#西米哥启动后会一直处于44行的wait状态，因为空白哥已经发完notify通知了进入wait状态了，
#而西米哥没收到
#造成的结果就是2根线程就一直在那挂起，什么都不干，也不扯蛋了
ximi.start()
kongbai.start()
if 0:
import zk_lock
zk = ZKClient(config.server_list)
lock_name = zk_lock.get_lock(zk)
print lock_name
exit()
while 1:
l = zk_lock.get_lock(zk)
print l
raw_input(">>")
zk_lock.release_lock(zk, l)
else:
return options.servers.split(",")
if __name__ == '__main__':
servers = get_zk_servers(options.configfile)
# create all the sessions first to ensure that all servers are
# at least available & quorum has been formed. otw this will
# fail right away (before we start creating nodes)
sessions = []
# create one session to each of the servers in the ensemble
for i, server in enumerate(servers):
sessions.append(ZKClient(server, options.timeout))
# create on first server
zk = ZKClient(servers[0], options.timeout)
rootpath = "/zk-smoketest"
if zk.exists(rootpath):
if not options.quiet:
print("Node %s already exists, attempting reuse" % (rootpath))
else:
zk.create(rootpath,
"smoketest root, delete after test done, created %s" %
(datetime.datetime.now().ctime()))
# make sure previous cleaned up properly
children = zk.get_children(rootpath)
if len(children) > 0:
raise SmokeError("there should not be children in %s"
class GJZookeeper(object):
ZK_HOST = "localhost:2181"
ROOT = "/app"
WORKERS_PATH = join(ROOT, "workers")
MASTERS_NUM = 1
TIMEOUT = 10000
def __init__(self, verbose=True):
self.VERBOSE = verbose
self.masters = []
self.is_master = False
self.path = None
self.zk = ZKClient(self.ZK_HOST, timeout=self.TIMEOUT)
self.say("login ok!")
# init
self.__init_zk()
# register
self.register()
def __init_zk(self):
"""
create the zookeeper node if not exist
"""
nodes = (self.ROOT, self.WORKERS_PATH)
for node in nodes:
if not self.zk.exists(node):
try:
self.zk.create(node, "")
except:
pass
@property
def is_slave(self):
return not self.is_master
def register(self):
"""
register a node for this worker
"""
self.path = self.zk.create(self.WORKERS_PATH + "/worker",
"1",
flags=zookeeper.EPHEMERAL
| zookeeper.SEQUENCE)
self.path = basename(self.path)
self.say("register ok! I'm %s" % self.path)
# check who is the master
self.get_master()
def get_master(self):
"""
get children, and check who is the smallest child
"""
@watchmethod
def watcher(event):
self.say("child changed, try to get master again.")
self.get_master()
children = self.zk.get_children(self.WORKERS_PATH, watcher)
children.sort()
self.say("%s's children: %s" % (self.WORKERS_PATH, children))
# check if I'm master
self.masters = children[:self.MASTERS_NUM]
if self.path in self.masters:
self.is_master = True
self.say("I've become master!")
else:
self.say("%s is masters, I'm slave" % self.masters)
def say(self, msg):
"""
print messages to screen
"""
if self.VERBOSE:
if self.path:
log.info(
"[ %s(%s) ] %s" %
(self.path, "master" if self.is_master else "slave", msg))
else:
log.info(msg)
return sum(summary[summary_key]) / len(summary[summary_key])
if __name__ == '__main__':
data = options.znode_size * "x"
servers = get_zk_servers(options.configfile)
# create all the sessions first to ensure that all servers are
# at least available & quorum has been formed. otw this will
# fail right away (before we start creating nodes)
sessions = []
server_id = 0
# create one session to each of the servers in the ensemble
for server in servers:
sessions.append(
ZKClient(server, options.timeout, ssl_server_cert_id=server_id))
server_id += 1
# ensure root_znode doesn't exist
if sessions[0].exists(options.root_znode):
# unless forced by user
if not options.force:
raise SmokeError("Node %s already exists!" % (options.root_znode))
children = sessions[0].get_children(options.root_znode)
for child in children:
sessions[0].delete("%s/%s" % (options.root_znode, child))
else:
sessions[0].create(
options.root_znode,
"smoketest root, delete after test done, created %s" %
class ZookeeperStore(object):
ZK_HOST = "127.0.0.1:32181"
TIMEOUT = 1000
def __init__(self, verbose = True, database='/test', table='targets'):
self.VERBOSE = verbose
self.path = None
self.database = database
self.table = os.path.join(self.database, table)
self.zk = ZKClient(self.ZK_HOST, timeout = self.TIMEOUT)
self.say("login ok")
self._init_zk()
def __del__(self):
self.zk.close()
def _init_zk(self):
nodes = (self.database, self.table)
for node in nodes:
if not self.zk.exists(node):
try:
self.zk.create(node, "")
except:
pass
def create(self, path, msg, flags=0):
msg = self.encode(msg)
self.path = os.path.join(self.table, path)
nodes = self.path.split("/")
# FIXME
for i in range(4,len(nodes)):
node = "/".join(nodes[:i])
if not self.zk.exists(node):
try:
self.zk.create(node, "")
except:
pass
if not self.zk.exists(self.path):
self.path = self.zk.create(self.path, msg, flags=flags)
else:
self.zk.set(self.path, msg)
#self.path = os.path.basename(self.path)
self.say("register ok! I'm %s" % self.path)
def get_children(self, path=""):
if not path:
path = self.table
children = self.zk.get_children(path)
return children
def get(self, node):
msg = self.zk.get(node)
return msg
def decode(self, obj):
return obj
def encode(self, msg):
return msg
def say(self, msg):
if self.VERBOSE:
if self.path:
log.info(msg)
else:
log.info(msg)
self.value = value
def __str__(self):
return repr(self.value)
if __name__ == '__main__':
servers = options.servers.split(",")
# create all the sessions first to ensure that all servers are
# at least available & quorum has been formed. otw this will
# fail right away (before we start creating nodes)
sessions = []
# create one session to each of the servers in the ensemble
for i, server in enumerate(servers):
sessions.append(ZKClient(server, options.timeout))
# create on first server
zk = ZKClient(servers[0], options.timeout)
rootpath = "/zk-smoketest"
if zk.exists(rootpath):
if not options.quiet:
print("Node %s already exists, attempting reuse" % (rootpath))
else:
zk.create(
rootpath, "smoketest root, delete after test done, created %s" %
(datetime.datetime.now().ctime()))
# make sure previous cleaned up properly
cb.waitForSuccess()
timer2(func, "deleted %7d permanent znodes " % (options.znode_count))
def get_zk_servers():
return options.cluster
if __name__ == '__main__':
data = options.znode_size * "x"
servers = get_zk_servers()
session_gets = []
for x in xrange(options.watch_session):
session_gets.append(ZKClient(servers, options.timeout))
session_set = ZKClient(servers, options.timeout)
session_set.create(
options.root_znode,
"smoketest root, delete after test done, created %s" %
(datetime.datetime.now().ctime()))
create_nodes(session_set)
for s in session_gets:
add_wathches(s)
set_nodes(session_set, data)
delete_nodes(session_set)
session_set.delete(options.root_znode)
time.sleep(10)