def quit(code):
if G.zookeeper != None:
zookeeper.close(G.zookeeper)
G.zookeeper = None
log('quit', [], 'DEBUG')
sys.exit(code)
def testhandlereuse(self):
"""
Test a) multiple concurrent connections b) reuse of closed handles
"""
cv = threading.Condition()
self.connected = False
def connection_watcher(handle, type, state, path):
cv.acquire()
self.connected = True
self.assertEqual(zookeeper.CONNECTED_STATE, state)
self.handle = handle
cv.notify()
cv.release()
cv.acquire()
handles = [ zookeeper.init(self.host) for i in range(10) ]
ret = zookeeper.init(self.host, connection_watcher)
cv.wait(15.0)
cv.release()
self.assertEqual(self.connected, True, "Connection timed out to " + self.host)
self.assertEqual(True, self.all( [ zookeeper.state(handle) == zookeeper.CONNECTED_STATE for handle in handles ] ),
"Not all connections succeeded")
oldhandle = handles[3]
zookeeper.close(oldhandle)
newhandle = zookeeper.init(self.host)
# This assertion tests *internal* behaviour; i.e. that the module
# correctly reuses closed handles. This is therefore implementation
# dependent.
self.assertEqual(newhandle, oldhandle, "Didn't get reused handle")
def test_logging(self):
handler = zope.testing.loggingsupport.InstalledHandler('ZooKeeper')
try:
handle = zookeeper.init('zookeeper.example.com:2181')
zookeeper.close(handle)
except:
pass
wait(lambda:
[r for r in handler.records if 'environment' in r.getMessage()])
handler.clear()
# Test that the filter for the "Exceeded deadline by" noise works.
# cheat and bypass zk by writing to the pipe directly.
os.write(
zc.zk._logging_pipe[1],
'2012-01-06 16:45:44,572:43673(0x1004f6000):ZOO_WARN@'
'zookeeper_interest@1461: Exceeded deadline by 27747ms\n')
wait(lambda: [
r for r in handler.records
if ('Exceeded deadline by' in r.getMessage() and r.levelno ==
logging.DEBUG)
])
self.assert_(not [
r for r in handler.records
if ('Exceeded deadline by' in r.getMessage()
and r.levelno == logging.WARNING)
])
handler.uninstall()
def watch_session(handle, event_type, state, path):
assert event_type == zookeeper.SESSION_EVENT
assert not path
if state == zookeeper.CONNECTED_STATE:
if self.handle is None:
self.handle = handle
for watch in self.watches.clear():
self._watch(watch)
for path, data in list(self.ephemeral.items()):
zookeeper.create(self.handle, path, data['data'],
data['acl'], data['flags'])
else:
assert handle == self.handle
connected.set()
logger.info('connected %s', handle)
elif state == zookeeper.CONNECTING_STATE:
connected.clear()
elif state == zookeeper.EXPIRED_SESSION_STATE:
connected.clear()
if self.handle is not None:
zookeeper.close(self.handle)
self.handle = None
init()
else:
logger.critical('unexpected session event %s %s', handle, state)
def close(self):
"""Close underlying zookeeper connections."""
zookeeper.close(self.handle)
self.handle = None
self.session_id = -1
self.session_password = ""
self.connected = False
def post(self):
request_dict = self.request.arguments
node_key = (request_dict['node_tree'])[0]
cluster_name = (request_dict['cluster_name'])[0]
zk=zookeeper.init(self.zk_connect(cluster_name))
def get_node(node_key):
"""获取子节点生成快照存到MySQL"""
if node_key == "/":
for node in zookeeper.get_children(zk,node_key):
key = "/" + node
if (zookeeper.get(zk,key)[1])['numChildren'] > 0:
get_node(key)
else:
value = (zookeeper.get(zk,key))[0]
create_time = time.time()
table = ZdSnapshot(cluster_name= cluster_name ,path=key , data=value ,create_time=self.GetNowTime())
table.save()
else:
for node in zookeeper.get_children(zk,node_key):
key = node_key + "/" + node
if (zookeeper.get(zk,key)[1])['numChildren'] > 0:
get_node(key)
else:
value = (zookeeper.get(zk,key))[0]
create_time = time.time()
table = ZdSnapshot(cluster_name= cluster_name,path=key , data=value ,create_time=self.GetNowTime())
table.save()
get_node(node_key)
self.write("生成快照成功!!!!!")
zookeeper.close(zk)
def connect(self, timeout = None):
'''Connects to the zookeeper server'''
# if no timeout provided, thake the configured one
if timeout is None:
timeout = self._timeout
# check, if we're already connected
if self._handle is not None and not self.is_connected():
raise RuntimeError('Already connected')
return
condition = threading.Condition()
def connection_watch(handle, type, state, path):
condition.acquire()
condition.notify()
condition.release()
# try to connect
condition.acquire()
self._handle = zookeeper.init(
','.join(self._servers),
connection_watch,
self._timeout * 1000)
condition.wait(self._timeout)
condition.release()
if zookeeper.state(self._handle) != zookeeper.CONNECTED_STATE:
zookeeper.close(self._handle)
raise RuntimeError(
'unable to connect to %s ' % (' or '.join(self._servers)))
zookeeper.set_watcher(self._handle, self.__global_watch)
def post(self):
request_dict = self.request.arguments
node_key = (request_dict['node_key'])[0]
cluster_name = (request_dict['cluster_name'])[0]
zk=zookeeper.init(self.zk_connect(cluster_name))
data = []
def get_node_tree(node_key):
if node_key == "/":
for node in zookeeper.get_children(zk,node_key):
key = "/" + node
if (zookeeper.get(zk,key)[1])['numChildren'] > 0:
get_node_tree(key)
print key
else:
print key
else:
for node in zookeeper.get_children(zk,node_key):
key = node_key + "/" + node
if (zookeeper.get(zk,key)[1])['numChildren'] > 0:
get_node_tree(key)
data.append(key)
else:
data.append(key)
return data
get_node_tree(node_key)
data.append(node_key)
for items in data:
zookeeper.delete(zk,items)
zookeeper.close(zk)
self.write("删除成功")
def reestablish_connection(self):
""" Checks the connection and resets it as needed. """
try:
zookeeper.close(self.handle)
except zookeeper.ZooKeeperException, close_exception:
logging.error("Exception when closing ZK connection {0}".\
format(close_exception))
def main():
from optparse import OptionParser as OptionParser
parser = OptionParser()
parser.add_option('-v', '--verbose', action="store_true", dest="flag_verbose",
help='Triggers debugging traces')
(options, args) = parser.parse_args(sys.argv)
# Logging configuration
if options.flag_verbose:
logging.basicConfig(
format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %I:%M:%S',
level=logging.DEBUG)
else:
logging.basicConfig(
format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %I:%M:%S',
level=logging.INFO)
if len(args) < 2:
raise ValueError("not enough CLI arguments")
ns = args[1]
cnxstr = load_config().get(ns, 'zookeeper')
zookeeper.set_debug_level(zookeeper.LOG_LEVEL_INFO)
zh = zookeeper.init(cnxstr)
zookeeper.create(zh, PREFIX, '', acl_openbar, 0)
create_tree(zh, boot_tree())
init_namespace(zh, ns)
zookeeper.close(zh)
def close(self):
""" Stops the thread that cleans up failed transactions and closes its
connection to Zookeeper.
"""
logging.info("Closing ZK connection")
self.stop_gc()
zookeeper.close(self.handle)
def test_logging(self):
handler = zope.testing.loggingsupport.InstalledHandler(
'ZooKeeper')
try:
handle = zookeeper.init('zookeeper.example.com:2181')
zookeeper.close(handle)
except:
pass
wait(lambda : [r for r in handler.records
if 'environment' in r.getMessage()]
)
handler.clear()
# Test that the filter for the "Exceeded deadline by" noise works.
# cheat and bypass zk by writing to the pipe directly.
os.write(zc.zk._logging_pipe[1],
'2012-01-06 16:45:44,572:43673(0x1004f6000):ZOO_WARN@'
'zookeeper_interest@1461: Exceeded deadline by 27747ms\n')
wait(lambda : [r for r in handler.records
if ('Exceeded deadline by' in r.getMessage()
and r.levelno == logging.DEBUG)
]
)
self.assert_(not [r for r in handler.records
if ('Exceeded deadline by' in r.getMessage()
and r.levelno == logging.WARNING)
])
handler.uninstall()
def connection_handler(handle, type, state, path):
if self._zh != handle:
try:
# latent handle callback from previous connection
zookeeper.close(handle)
except:
pass
return
if self._stopped.is_set():
return
if self._watch:
self._watch(self, type, state, path)
if state == zookeeper.CONNECTED_STATE:
self._logger('Connection started, setting live.')
self._live.set()
self._clear_completions()
elif state == zookeeper.EXPIRED_SESSION_STATE:
self._logger('Session lost, clearing live state.')
self._live.clear()
self._zh = None
self._init_count = 0
self.reconnect()
else:
self._logger('Connection lost, clearing live state.')
self._live.clear()
def connection_handler(handle, type, state, path):
if self._zh != handle:
try:
# latent handle callback from previous connection
zookeeper.close(handle)
except:
pass
return
if self._stopped.is_set():
return
if self._watch:
self._watch(self, type, state, path)
if state == zookeeper.CONNECTED_STATE:
self._logger('Connection started, setting live.')
self._live.set()
self._clear_completions()
elif state == zookeeper.EXPIRED_SESSION_STATE:
self._logger('Session lost, clearing live state.')
self._live.clear()
self._zh = None
self._init_count = 0
self.reconnect()
else:
self._logger('Connection lost, clearing live state.')
self._live.clear()
def testhandlereuse(self):
"""
Test a) multiple concurrent connections b) reuse of closed handles
"""
cv = threading.Condition()
self.connected = False
def connection_watcher(handle, type, state, path):
cv.acquire()
self.connected = True
self.assertEqual(zookeeper.CONNECTED_STATE, state)
self.handle = handle
cv.notify()
cv.release()
cv.acquire()
handles = [ zookeeper.init(self.host) for i in xrange(10) ]
ret = zookeeper.init(self.host, connection_watcher)
cv.wait(15.0)
cv.release()
self.assertEqual(self.connected, True, "Connection timed out to " + self.host)
self.assertEqual(True, all( [ zookeeper.state(handle) == zookeeper.CONNECTED_STATE for handle in handles ] ),
"Not all connections succeeded")
oldhandle = handles[3]
zookeeper.close(oldhandle)
newhandle = zookeeper.init(self.host)
# This assertion tests *internal* behaviour; i.e. that the module
# correctly reuses closed handles. This is therefore implementation
# dependent.
self.assertEqual(newhandle, oldhandle, "Didn't get reused handle")
def connect(self, first=False):
try:
if self._handler:
zookeeper.close(self._handler)
except:
pass
self._handler = Connection(self._name, self.register_watch_cb, first, self.register_node_cb, *self._args, **self._kwargs).handler
def close(self):
"""Close underlying zookeeper connections."""
zookeeper.close(self.handle)
self.handle = None
self.session_id = -1
self.session_password = ""
self.connected = False
def close(self):
if self.connected:
try:
zookeeper.close(self.handle)
except:
self.log.exception('close')
self.connected = False
self.handle = None
def close(self):
"""
close the handle. potentially a blocking call?
"""
if self._zhandle is not None:
zookeeper.close(self._zhandle)
# if closed successfully
self._zhandle = None
def close(self):
if self.connected:
try:
zookeeper.close(self.handle)
except:
self.log.exception('close')
self.connected = False
self.handle = None
def close(self):
"""
close the handle. potentially a blocking call?
"""
if self._zhandle is not None:
zookeeper.close(self._zhandle)
# if closed successfully
self._zhandle = None
def close():
global handle
global init_flag
global connected
if not init_flag: return
zookeeper.close(handle)
init_flag = False
connected = False
def close():
global handle
global init_flag
global connected
if not init_flag: return
zookeeper.close(handle)
init_flag = False
connected = False
def _safe_close(self):
if self._zh is not None:
zh, self._zh = self._zh, None
try:
zookeeper.close(zh)
except zookeeper.ZooKeeperException:
# the session has been corrupted or otherwise disconnected
pass
self._live.clear()
def connect(self, first=False):
try:
if self._handler:
zookeeper.close(self._handler)
except:
pass
self._handler = Connection(self._name, self.register_watch_cb, first,
self.register_node_cb, *self._args,
**self._kwargs).handler
def _check_timeout():
# Close the handle
try:
if self.handle is not None:
zookeeper.close(self.handle)
except zookeeper.ZooKeeperException:
pass
d.errback(
ConnectionTimeoutException("could not connect before timeout"))
def _safe_close(self):
if self._zh is not None:
zh, self._zh = self._zh, None
try:
zookeeper.close(zh)
except zookeeper.ZooKeeperException:
# the session has been corrupted or otherwise disconnected
pass
self._live.clear()
def close(self):
"""
Close the connection to our ZooKeeper instance
Return: None
Exceptions: None
"""
if self._zk:
zookeeper.close(self._zk)
def post(self):
request_dict = self.request.arguments
node_id = (request_dict['choose_node'])[0]
cluster_name = (request_dict['cluster_name'])[0]
zk=zookeeper.init(self.zk_connect(cluster_name))
_value = (zookeeper.get(zk,node_id))[0]
zookeeper.close(zk)
self.write(_value)
logging.info('%s查看了集群%s的节点:%s'%(self.get_current_user(),cluster_name,node_id))
def post(self):
request_dict = self.request.arguments
snapshot_id = (request_dict['snapshot_id'])[0]
print snapshot_id
data = ZdSnapshot.get(ZdSnapshot.id == int(snapshot_id) )
zk = zookeeper.init(self.zk_connect(data.cluster_name))
zookeeper.set(zk, data.path ,data.data)
zookeeper.close(zk)
self.write("还原成功!!!!!")
def main():
from optparse import OptionParser as OptionParser
parser = OptionParser()
parser.add_option(
'-v', '--verbose',
action="store_true", dest="flag_verbose",
help='Triggers debugging traces')
parser.add_option(
'-s', '--smart',
action="store_true", dest="SMART", default=False,
help="Delete onle the members belong to services with multiple" \
" members")
parser.add_option(
'-d', '--dry-run',
action="store_true", dest="DRY", default=False,
help="Do not delete, just print")
parser.add_option(
'-n', '--min-services',
type=int, action="store", dest="NUM", default=4,
help="Do not delete election if less the NUM")
parser.add_option(
'-1', '--alone',
action="store_true", dest="ALONE", default=False,
help="Also consider members alone in their group")
(options, args) = parser.parse_args(sys.argv)
# Logging configuration
if options.flag_verbose:
logging.basicConfig(
format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %I:%M:%S',
level=logging.DEBUG)
else:
logging.basicConfig(
format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %I:%M:%S',
level=logging.INFO)
if len(args) < 2:
raise ValueError("not enough CLI arguments: NS TYPE [TYPE...]")
ns = args[1]
cnxstr = load_namespace_conf(ns)['zookeeper']
zookeeper.set_debug_level(zookeeper.LOG_LEVEL_INFO)
zh = zookeeper.init(cnxstr)
for srvtype in args[2:]:
for group in namespace_tree(ns, srvtype):
logging.debug(">DIR %s", group)
for node in list_problematic_nodes(zh, group, options):
delete_node(zh, node, options)
zookeeper.close(zh)
def post(self):
request_dict = self.request.arguments
node_value = (request_dict['node_value'])[0]
node_name = (request_dict['node_name'])[0]
cluster_name = (request_dict['cluster_name'])[0]
zk=zookeeper.init(self.zk_connect(cluster_name))
zookeeper.set(zk,node_name,node_value)
zookeeper.close(zk)
logging.info('%s修改了集群%s的节点:%s值为%s'%(self.get_current_user(),cluster_name,node_name ,node_value ))
self.write("修改成功")
def __watcher(self, zh, evtype, state, path):
"""connection/session watcher"""
logging.debug('event%s', str((evtype, state, path)))
if evtype == zk.SESSION_EVENT:
if state == zk.CONNECTED_STATE:
self.publish_alive()
elif state == zk.EXPIRED_SESSION_STATE:
# unrecoverable state - close and reconnect
zk.close(self.zh)
self.zh = zk.init(self.zkhosts, self.__watcher)
def _getThriftServerFromZookeeper(self):
self.hd = zookeeper.init("%s:%s" % (self.zookeeper_host, self.zookeeper_port))
self.children = zookeeper.get_children(self.hd, self.hbase_thrift_znode)
# random get a thrift server
import random
index = random.randint(0, len(self.children)-1)
(self.thrift_host, self.thrift_port) = self.children[index].split(':')
log.msg("Thrift server got from zookeeper:[%s:%s]" % \
(self.thrift_host, self.thrift_port))
zookeeper.close(self.hd)
def main():
zookeeper.set_debug_level(zookeeper.LOG_LEVEL_WARN)
zh = zookeeper.init('127.0.0.1:2181')
ensure_basedirs(zh)
for f in sys.argv[1:]:
try:
manage_type(zh, f)
print "OK", f
except Exception as e:
print "KO", f, e
zookeeper.close(zh)
def main():
zookeeper.set_debug_level(zookeeper.LOG_LEVEL_WARN)
zh = zookeeper.init('127.0.0.1:2181')
ensure_basedirs(zh)
for f in sys.argv[1:]:
try:
manage_type(zh, f)
print "OK", f
except Exception as e:
print "KO", f, e
zookeeper.close(zh)
def post(self):
request_dict = self.request.arguments
node_tree = (request_dict['node_tree'])[0]
cluster_name = (request_dict['cluster_name'])[0]
zk=zookeeper.init(self.zk_connect(cluster_name))
_value = (zookeeper.get(zk,node_tree))[0]
create_time = time.time()
table = ZdSnapshot(cluster_name= cluster_name ,path=node_tree , data=_value ,create_time = self.GetNowTime())
table.save()
zookeeper.close(zk)
self.write("生成快照成功!!!!!")
def main():
from optparse import OptionParser as OptionParser
parser = OptionParser()
parser.add_option('-v',
'--verbose',
action="store_true",
dest="flag_verbose",
help='Triggers debugging traces')
parser.add_option(
'-c',
'--min-children',
type=int,
action="store",
dest="CHILDREN",
default=15,
help="Do not print the children number if less than that value")
parser.add_option('-s',
'--min-size',
type=int,
action="store",
dest="SIZE",
default=0,
help="Do not print unless the size is over that value")
(options, args) = parser.parse_args(sys.argv)
# Logging configuration
if options.flag_verbose:
logging.basicConfig(format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %I:%M:%S',
level=logging.DEBUG)
else:
logging.basicConfig(format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %I:%M:%S',
level=logging.INFO)
if len(args) < 2:
raise ValueError("not enough CLI arguments: NS TYPE [TYPE...]")
ns = args[1]
cnxstr = load_namespace_conf(ns)['zookeeper']
zookeeper.set_debug_level(zookeeper.LOG_LEVEL_INFO)
zh = zookeeper.init(cnxstr)
for srvtype in args[2:]:
for group in namespace_tree(ns, srvtype):
for child, meta in list_nodes(zh, group, options):
if meta['dataLength'] > options.SIZE:
print "NODE", meta['dataLength'], child
zookeeper.close(zh)
def deadlock():
cv = threading.Condition()
def callback(*args):
cv.acquire()
cv.notifyAll()
cv.release()
time.sleep(1)
cv.acquire()
zookeeper.aget(handle, "/", None, callback)
cv.wait()
zookeeper.close(handle)
def close(self):
self.cond.acquire()
# Forget current watches, otherwise it's easy for
# circular references to keep this object around.
self.watches = {}
try:
if self.handle:
zookeeper.close(self.handle)
self.handle = None
except:
logging.warn("Error closing Zookeeper handle.")
finally:
self.cond.release()
def deadlock():
cv = threading.Condition()
def callback(*args):
cv.acquire()
cv.notifyAll()
cv.release()
time.sleep(1)
cv.acquire()
zookeeper.aget(handle, "/", None, callback)
cv.wait()
zookeeper.close(handle)
def post(self):
request_dict = self.request.arguments
node_key = (request_dict['node_key'])[0]
cluster_name = (request_dict['cluster_name'])[0]
zk=zookeeper.init(self.zk_connect(cluster_name))
try:
zookeeper.delete(zk,node_key)
msg = '删除成功'
except:
msg = '无法删除节点'
finally:
zookeeper.close(zk)
self.write(msg)
def post(self):
request_dict = self.request.arguments
cluster_name = (request_dict['cluster_name'])[0]
zk=zookeeper.init(self.zk_connect(cluster_name))
new_node = (request_dict['New_post_node'])[0]
new_value = (request_dict['new_node_value'])[0]
if zookeeper.exists(zk,new_node):
zookeeper.close(zk)
self.write("此节点存在")
else:
zookeeper.create(zk,new_node,new_value,[{"perms":0x1f,"scheme":"world","id":"anyone"}],0)
zookeeper.close(zk)
self.write("增加成功")
def main():
from optparse import OptionParser as OptionParser
parser = OptionParser()
parser.add_option(
'-v',
'--verbose',
action="store_true",
dest="flag_verbose",
help='Triggers debugging traces')
parser.add_option(
'--slow',
action="store_true",
dest="SLOW",
help='Only play with small batches to avoid timeouts on slow hosts.')
parser.add_option(
'--avoid',
action="append",
type="string",
dest="AVOID_TYPES",
help='Do not populate entries for the specified service types')
(options, args) = parser.parse_args(sys.argv)
# Logging configuration
if options.flag_verbose:
logging.basicConfig(
format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %I:%M:%S',
level=logging.DEBUG)
else:
logging.basicConfig(
format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %I:%M:%S',
level=logging.INFO)
if len(args) < 2:
raise ValueError("not enough CLI arguments")
ns = args[1]
cnxstr = load_namespace_conf(ns)['zookeeper']
zookeeper.set_debug_level(zookeeper.LOG_LEVEL_INFO)
zh = zookeeper.init(cnxstr)
# synchronous creation of the root
try:
zookeeper.create(zh, PREFIX, '', acl_openbar, 0)
except zookeeper.NodeExistsException:
pass
create_tree(zh, namespace_tree(ns, options), options)
zookeeper.close(zh)
def __init__(self, connection_string="127.0.0.1:2181", session_timeout=None,
wait=False):
self.watches = WatchManager()
self.ephemeral = {}
self.handle = None
connected = self.connected = threading.Event()
def watch_session(handle, event_type, state, path):
assert event_type == zookeeper.SESSION_EVENT
assert not path
if state == zookeeper.CONNECTED_STATE:
if self.handle is None:
self.handle = handle
for watch in self.watches.clear():
self._watch(watch)
for path, data in list(self.ephemeral.items()):
zookeeper.create(self.handle, path, data['data'],
data['acl'], data['flags'])
else:
assert handle == self.handle
connected.set()
logger.info('connected %s', handle)
elif state == zookeeper.CONNECTING_STATE:
connected.clear()
elif state == zookeeper.EXPIRED_SESSION_STATE:
connected.clear()
if self.handle is not None:
zookeeper.close(self.handle)
self.handle = None
init()
else:
logger.critical('unexpected session event %s %s', handle, state)
if session_timeout:
init = (lambda : zookeeper.init(connection_string, watch_session,
session_timeout)
)
else:
init = lambda : zookeeper.init(connection_string, watch_session)
handle = init()
connected.wait(self.initial_connection_wait)
if not connected.is_set():
if wait:
while not connected.is_set():
logger.critical("Can't connect to ZooKeeper at %r",
connection_string)
connected.wait(1)
else:
zookeeper.close(handle)
raise FailedConnect(connection_string)
def close(self):
self.cond.acquire()
# Forget current watches, otherwise it's easy for
# circular references to keep this object around.
self.content_watches = {}
self.child_watches = {}
try:
if hasattr(self, 'handle') and self.handle:
zookeeper.close(self.handle)
self.handle = None
except Exception:
logging.warn("Error closing Zookeeper handle.")
finally:
self.cond.release()
def main():
from optparse import OptionParser as OptionParser
parser = OptionParser()
parser.add_option('-v',
'--verbose',
action="store_true",
dest="flag_verbose",
help='Triggers debugging traces')
parser.add_option('-d',
'--data',
action="store",
dest="VALUE",
default='',
help="Data to force")
(options, args) = parser.parse_args(sys.argv)
# Logging configuration
if options.flag_verbose:
logging.basicConfig(format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %I:%M:%S',
level=logging.DEBUG)
else:
logging.basicConfig(format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %I:%M:%S',
level=logging.INFO)
if len(args) < 2:
raise ValueError("not enough CLI arguments: NS TYPE [TYPE...]")
ns = args[1]
cnxstr = load_namespace_conf(ns)['zookeeper']
zookeeper.set_debug_level(zookeeper.LOG_LEVEL_INFO)
zh = zookeeper.init(cnxstr)
for srvtype in args[2:]:
for group in namespace_tree(ns, srvtype):
logging.debug(">DIR %s", group)
for mom, key, first, last in list_groups(zh, group, options):
tail = str(1 + int(last)).rjust(10, '0')
path = mom + '/' + key + '-' + tail
create_node(zh, options, path)
logging.debug("Please send a signal to remove the ephemeral nodes")
logging.warn("PAUSED ...")
signal.pause()
zookeeper.close(zh)
def main():
from optparse import OptionParser as OptionParser
parser = OptionParser()
parser.add_option('-v',
'--verbose',
action="store_true",
dest="flag_verbose",
help='Triggers debugging traces')
parser.add_option(
'--slow',
action="store_true",
dest="SLOW",
help='Only play with small batches to avoid timeouts on slow hosts.')
parser.add_option(
'--avoid',
action="append",
type="string",
dest="AVOID_TYPES",
help='Do not populate entries for the specified service types')
(options, args) = parser.parse_args(sys.argv)
# Logging configuration
if options.flag_verbose:
logging.basicConfig(format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %I:%M:%S',
level=logging.DEBUG)
else:
logging.basicConfig(format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %I:%M:%S',
level=logging.INFO)
if len(args) < 2:
raise ValueError("not enough CLI arguments")
ns = args[1]
cnxstr = load_namespace_conf(ns)['zookeeper']
zookeeper.set_debug_level(zookeeper.LOG_LEVEL_INFO)
zh = zookeeper.init(cnxstr)
# synchronous creation of the root
try:
zookeeper.create(zh, PREFIX, '', acl_openbar, 0)
except zookeeper.NodeExistsException:
pass
create_tree(zh, namespace_tree(ns, options), options)
zookeeper.close(zh)
def testconnection(self):
cv = threading.Condition()
self.connected = False
def connection_watcher(handle, type, state, path):
cv.acquire()
self.connected = True
self.assertEqual(zookeeper.CONNECTED_STATE, state)
self.handle = handle
cv.notify()
cv.release()
cv.acquire()
ret = zookeeper.init(self.host, connection_watcher)
cv.wait(15.0)
cv.release()
self.assertEqual(self.connected, True,
"Connection timed out to " + self.host)
self.assertEqual(zookeeper.CONNECTED_STATE,
zookeeper.state(self.handle))
self.assertEqual(zookeeper.close(self.handle), zookeeper.OK)
# Trying to close the same handle twice is an error, and the C library will segfault on it
# so make sure this is caught at the Python module layer
self.assertRaises(zookeeper.ZooKeeperException, zookeeper.close,
self.handle)
self.assertRaises(zookeeper.ZooKeeperException, zookeeper.get,
self.handle, "/")
def testmanyhandles(self):
"""
Test the ability of the module to support many handles.
"""
# We'd like to do more, but currently the C client doesn't
# work with > 83 handles (fails to create a pipe) on MacOS 10.5.8
handles = [zookeeper.init(self.host) for i in xrange(63)]
cv = threading.Condition()
self.connected = False
def connection_watcher(handle, type, state, path):
cv.acquire()
self.connected = True
self.assertEqual(zookeeper.CONNECTED_STATE, state)
self.handle = handle
cv.notify()
cv.release()
cv.acquire()
ret = zookeeper.init(self.host, connection_watcher)
cv.wait(15.0)
cv.release()
self.assertEqual(self.connected, True,
"Connection timed out to " + self.host)
for i, h in enumerate(handles):
path = "/zkpython-test-handles-%s" % str(i)
self.assertEqual(
path,
zookeeper.create(h, path, "", [ZOO_OPEN_ACL_UNSAFE],
zookeeper.EPHEMERAL))
self.assertEqual(
True, all(zookeeper.close(h) == zookeeper.OK for h in handles))
def close(self):
LOGGER.info("close ZookeeperRegistry")
try:
if self._register_handler is not None:
zookeeper.close(self._register_handler)
except BaseException:
traceback.print_exc()
try:
if self._discovery_handler is not None:
zookeeper.close(self._discovery_handler)
except BaseException:
traceback.print_exc()
self._register_handler = None
self._discovery_handler = None
self._local_cache.clear()
def watcher_conn(self, handler, mode, state, path):
self._conn_cv.acquire()
event = ClientEvent(mode, state, path)
if state == zookeeper.EXPIRED_SESSION_STATE:
utils.log(utils.cur(), self._name, event.state_name)
try:
zookeeper.close(self._handler)
except:
pass
self._connected = False
self.safe_connect()
elif state == zookeeper.CONNECTED_STATE:
utils.log(utils.cur(), self._name, event.state_name)
self._connected = True
else:
utils.log(utils.cur(), self._name, event.state_name)
self._conn_cv.notifyAll()
self._conn_cv.release()
def close(self):
"""Disconnect from ZooKeeper
"""
if self._connected:
code = zookeeper.close(self._handle)
self._handle = None
self._connected = False
if code != zookeeper.OK:
raise err_to_exception(code)
def main():
usage = "usage: %prog [options] NS"
from optparse import OptionParser as OptionParser
parser = OptionParser(usage=usage)
parser.add_option(
'-v', '--verbose',
action="store_true", dest="flag_verbose",
help='Triggers debugging traces')
parser.add_option(
'-a', '--all',
action="store_true", dest="flag_all",
help='Remove all oio-sds nodes (not only meta0)')
(options, args) = parser.parse_args(sys.argv)
# Logging configuration
if options.flag_verbose:
logging.basicConfig(
format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %I:%M:%S',
level=logging.DEBUG)
else:
logging.basicConfig(
format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %I:%M:%S',
level=logging.INFO)
if len(args) < 2:
raise ValueError("not enough CLI arguments")
ns = args[1]
cnxstr = load_namespace_conf(ns)['zookeeper']
zookeeper.set_debug_level(zookeeper.LOG_LEVEL_INFO)
for shard in cnxstr.split(";"):
logging.info("ZK=%s", shard)
zh = zookeeper.init(shard)
if options.flag_all:
logging.warn("FLUSHING all the oio-sds entries in the ZK server")
delete_children(zh, "/hc")
else:
logging.info("Cleaning only the meta0 registrations in ZK server")
delete_children(zh, "/hc/ns/"+ns+"/srv/meta0")
zookeeper.close(zh)
def run(self):
self.locate()
while True:
try:
self.lock.acquire()
if self.safeMode == True:
time.sleep(timeout)
zookeeper.close(self.zh)
conn_cv.acquire()
self.zh = zookeeper.init(self.zkservers, self.conn_callback, session_time)
conn_cv.wait()
conn_cv.release()
self.locate()
if self.safeMode == False:
if zookeeper.OK != zookeeper.aget_children(self.zh, self.actionNode, self.queue_watcher, None):
logger.error('Fail to monitor action queue for: '+self.actionNode+', activate safe mode.')
self.safeMode = True
except Exception, err:
logger.exception(err)
def main():
data = options.znode_data_size * "D"
zookeeper.set_debug_level(zookeeper.LOG_LEVEL_WARN)
s = zookeeper.init(options.server)
if zookeeper.exists(s, options.root_znode, None):
children = zookeeper.get_children(s, options.root_znode, None)
print "delete old entries: %d" % len(children)
for child in children:
zookeeper.delete(s, "%s/%s" % (options.root_znode, child))
else:
zookeeper.create(s, options.root_znode, "zkpy root znode", acl, 0)
evaluation(s, options.root_znode, data, options)
zookeeper.close(s)
print("Performance test complete")
def main():
from optparse import OptionParser as OptionParser
parser = OptionParser()
parser.add_option(
'-v', '--verbose',
action="store_true", dest="flag_verbose",
help='Triggers debugging traces')
(options, args) = parser.parse_args(sys.argv)
# Logging configuration
if options.flag_verbose:
logging.basicConfig(
format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %I:%M:%S',
level=logging.DEBUG)
else:
logging.basicConfig(
format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %I:%M:%S',
level=logging.INFO)
if len(args) < 2:
raise ValueError("not enough CLI arguments: NS SRVTYPE [SRVTYPE...]")
ns = args[1]
cnxstr = load_namespace_conf(ns)['zookeeper']
zookeeper.set_debug_level(zookeeper.LOG_LEVEL_INFO)
zh = zookeeper.init(cnxstr)
for srvtype in args[2:]:
for group in namespace_tree(ns, srvtype):
children = list(list_nodes(zh, group, options))
if len(children) > 0:
logging.info("> %s", group)
for k in children:
data, meta = zookeeper.get(zh, group + '/' + k)
logging.info(" %s : %s", k, data)
zookeeper.close(zh)
def watcher(zh, event, state, path):
logger.debug("watcher callback zh:%d event:%d state:%d path:%s", zh, event, state, path)
if event == zookeeper.SESSION_EVENT and state == zookeeper.CONNECTING_STATE:
zookeeper.close(zh)
zh = zookeeper.init(ZOOKEEPERADDR, watcher, 30000)
elif event == zookeeper.SESSION_EVENT and state == zookeeper.EXPIRED_SESSION_STATE:
zookeeper.close(zh)
zh = zookeeper.init(ZOOKEEPERADDR, watcher, 30000)
elif event == zookeeper.SESSION_EVENT and state == zookeeper.CONNECTED_STATE: #init connection complete
pass
elif event == zookeeper.CREATED_EVENT and state == zookeeper.CONNECTED_STATE:
InitConnPool(zh, PROXYPATH)
elif event == zookeeper.DELETED_EVENT and state == zookeeper.CONNECTED_STATE:
InitConnPool(zh, PROXYPATH)
elif event == zookeeper.CHANGED_EVENT and state == zookeeper.CONNECTED_STATE:
InitConnPool(zh, PROXYPATH)
elif event == zookeeper.CHILD_EVENT and state == zookeeper.CONNECTED_STATE:
InitConnPool(zh, PROXYPATH)
else:
logger.error("zookeeper connection state changed but not implemented: event:%d state:%d path:%s", event, state, path)
def start_zk(args, zk_path):
print zk_path + '/bin/zkServer.sh start'
subprocess.Popen(zk_path + '/bin/zkServer.sh start', shell=True)
hostname = socket.gethostname()
ip = socket.gethostbyname(hostname)
print ip + ':' + str(args.port)
if args.myid == 1:
while True:
time.sleep(2)
handler = zookeeper.init(ip + ':' + str(args.port))
try:
zookeeper.create(handler, "/tera", "", [{
"perms": 0x7f,
"scheme": "world",
"id": "anyone"
}], 0)
zookeeper.create(handler, "/tera/master-lock", "",
[{
"perms": 0x7f,
"scheme": "world",
"id": "anyone"
}], 0)
zookeeper.create(handler, "/tera/ts", "", [{
"perms": 0x7f,
"scheme": "world",
"id": "anyone"
}], 0)
zookeeper.create(handler, "/tera/kick", "", [{
"perms": 0x7f,
"scheme": "world",
"id": "anyone"
}], 0)
zookeeper.close(handler)
break
except:
print '#######################\nkeep trying!!\n##########################\n'
traceback.print_exc()
continue
def close(self):
return zookeeper.close(self.handle)
