def testhandlereuse(self):
"""
Test a) multiple concurrent connections b) reuse of closed handles
"""
cv = threading.Condition()
self.connected = False
def connection_watcher(handle, type, state, path):
cv.acquire()
self.connected = True
self.assertEqual(zookeeper.CONNECTED_STATE, state)
self.handle = handle
cv.notify()
cv.release()
cv.acquire()
handles = [ zookeeper.init(self.host) for i in xrange(10) ]
ret = zookeeper.init(self.host, connection_watcher)
cv.wait(15.0)
cv.release()
self.assertEqual(self.connected, True, "Connection timed out to " + self.host)
self.assertEqual(True, all( [ zookeeper.state(handle) == zookeeper.CONNECTED_STATE for handle in handles ] ),
"Not all connections succeeded")
oldhandle = handles[3]
zookeeper.close(oldhandle)
newhandle = zookeeper.init(self.host)
# This assertion tests *internal* behaviour; i.e. that the module
# correctly reuses closed handles. This is therefore implementation
# dependent.
self.assertEqual(newhandle, oldhandle, "Didn't get reused handle")
def _connect(self):
"""Creates a connection to a zookeeper instance."""
s = []
for host, port in self._servers:
try:
_, _, ips = socket.gethostbyname_ex(host)
for ip in ips:
s.append('%s:%s' % (ip, port))
except socket.gaierror:
logging.error('Hostname not known: %s', host)
except socket.herror:
logging.error('Unable to resolve %s', host)
if not s:
logging.error('No IPs found to connect to.. trying again in 1 second.')
t = threading.Timer(1.0, self._connect)
t.daemon = True
t.start()
return
if self._clientid is not None:
# Existing connections get registered with the same clientid that was
# used before.
self._zookeeper = zookeeper.init(
','.join(s), self._global_watch, None, clientid)
else:
self._zookeeper = zookeeper.init(','.join(s), self._global_watch)
def testmanyhandles(self):
"""
Test the ability of the module to support many handles.
"""
# We'd like to do more, but currently the C client doesn't
# work with > 83 handles (fails to create a pipe) on MacOS 10.5.8
handles = [ zookeeper.init(self.host) for i in xrange(63) ]
cv = threading.Condition()
self.connected = False
def connection_watcher(handle, type, state, path):
cv.acquire()
self.connected = True
self.assertEqual(zookeeper.CONNECTED_STATE, state)
self.handle = handle
cv.notify()
cv.release()
cv.acquire()
ret = zookeeper.init(self.host, connection_watcher)
cv.wait(15.0)
cv.release()
self.assertEqual(self.connected, True, "Connection timed out to " + self.host)
for i,h in enumerate(handles):
path = "/zkpython-test-handles-%s" % str(i)
self.assertEqual(path, zookeeper.create(h, path, "", [ZOO_OPEN_ACL_UNSAFE], zookeeper.EPHEMERAL))
self.assertEqual(True, all( zookeeper.close(h) == zookeeper.OK for h in handles ))
def __init__(self, connection_string="127.0.0.1:2181", session_timeout=None,
wait=False):
self.watches = WatchManager()
self.ephemeral = {}
self.handle = None
connected = self.connected = threading.Event()
def watch_session(handle, event_type, state, path):
assert event_type == zookeeper.SESSION_EVENT
assert not path
if state == zookeeper.CONNECTED_STATE:
if self.handle is None:
self.handle = handle
for watch in self.watches.clear():
self._watch(watch)
for path, data in list(self.ephemeral.items()):
zookeeper.create(self.handle, path, data['data'],
data['acl'], data['flags'])
else:
assert handle == self.handle
connected.set()
logger.info('connected %s', handle)
elif state == zookeeper.CONNECTING_STATE:
connected.clear()
elif state == zookeeper.EXPIRED_SESSION_STATE:
connected.clear()
if self.handle is not None:
zookeeper.close(self.handle)
self.handle = None
init()
else:
logger.critical('unexpected session event %s %s', handle, state)
if session_timeout:
init = (lambda : zookeeper.init(connection_string, watch_session,
session_timeout)
)
else:
init = lambda : zookeeper.init(connection_string, watch_session)
handle = init()
connected.wait(self.initial_connection_wait)
if not connected.is_set():
if wait:
while not connected.is_set():
logger.critical("Can't connect to ZooKeeper at %r",
connection_string)
connected.wait(1)
else:
zookeeper.close(handle)
raise FailedConnect(connection_string)
def connect(self, servers=None, timeout=10, client_id=None):
"""
Establish a connection to the given zookeeper server(s).
@param servers: A string specifying the servers and their ports to
connect to. Multiple servers can be specified in
comma separated fashion.
@param timeout: How many seconds to wait on a connection to the
zookeeper servers.
@param session_id:
@returns A deferred that's fired when the connection is established.
"""
d = defer.Deferred()
if self.connected:
return defer.fail(
zookeeper.ZooKeeperException("Already Connected"))
# Use a scheduled function to ensure a timeout.
def _check_timeout():
# Close the handle
try:
if self.handle is not None:
zookeeper.close(self.handle)
except zookeeper.ZooKeeperException:
pass
d.errback(
ConnectionTimeoutException("could not connect before timeout"))
scheduled_timeout = reactor.callLater(timeout, _check_timeout)
# Assemble an on connect callback with closure variable access.
callback = partial(self._cb_connected, scheduled_timeout, d)
callback = self._zk_thread_callback(callback)
if self._session_timeout is None:
self._session_timeout = DEFAULT_SESSION_TIMEOUT
if servers is not None:
self._servers = servers
# Use client id if specified.
if client_id:
self.handle = zookeeper.init(
self._servers, callback, self._session_timeout, client_id)
else:
self.handle = zookeeper.init(
self._servers, callback, self._session_timeout)
return d
def connect_async(self):
"""Asynchronously initiate connection to ZK
@return: AsyncResult object set on connection success
@rtype AsyncResult
"""
cb = self._wrap_session_callback(self._session_callback)
if self._provided_client_id:
self._handle = zookeeper.init(self._hosts, cb, self._timeout,
self._provided_client_id)
else:
self._handle = zookeeper.init(self._hosts, cb, self._timeout)
return self._connected_async_result
def connect(self, timeout = None):
'''Connects to the zookeeper server'''
# if no timeout provided, thake the configured one
if timeout is None:
timeout = self._timeout
# check, if we're already connected
if self._handle is not None and not self.is_connected():
raise RuntimeError('Already connected')
return
condition = threading.Condition()
def connection_watch(handle, type, state, path):
condition.acquire()
condition.notify()
condition.release()
# try to connect
condition.acquire()
self._handle = zookeeper.init(
','.join(self._servers),
connection_watch,
self._timeout * 1000)
condition.wait(self._timeout)
condition.release()
if zookeeper.state(self._handle) != zookeeper.CONNECTED_STATE:
zookeeper.close(self._handle)
raise RuntimeError(
'unable to connect to %s ' % (' or '.join(self._servers)))
zookeeper.set_watcher(self._handle, self.__global_watch)
def main():
from optparse import OptionParser as OptionParser
parser = OptionParser()
parser.add_option('-v', '--verbose', action="store_true", dest="flag_verbose",
help='Triggers debugging traces')
(options, args) = parser.parse_args(sys.argv)
# Logging configuration
if options.flag_verbose:
logging.basicConfig(
format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %I:%M:%S',
level=logging.DEBUG)
else:
logging.basicConfig(
format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %I:%M:%S',
level=logging.INFO)
if len(args) < 2:
raise ValueError("not enough CLI arguments")
ns = args[1]
cnxstr = load_config().get(ns, 'zookeeper')
zookeeper.set_debug_level(zookeeper.LOG_LEVEL_INFO)
zh = zookeeper.init(cnxstr)
zookeeper.create(zh, PREFIX, '', acl_openbar, 0)
create_tree(zh, boot_tree())
init_namespace(zh, ns)
zookeeper.close(zh)
def __init__(self, host=DEFAULT_HOST, start_gc=True):
""" Creates a new ZKTransaction, which will communicate with Zookeeper
on the given host.
Args:
host: A str that indicates which machine runs the Zookeeper service.
start_gc: A bool that indicates if we should start the garbage collector
for timed out transactions.
"""
logging.basicConfig(format='%(asctime)s %(levelname)s %(filename)s:' \
'%(lineno)s %(message)s ', level=logging.INFO)
logging.debug("Started logging")
# Connection instance variables.
self.connect_cv = threading.Condition()
self.connected = False
self.host = host
self.handle = zookeeper.init(host, self.receive_and_notify)
# for blacklist cache
self.blacklist_cv = threading.Condition()
self.blacklist_cache = {}
# for gc
self.gc_running = False
self.gc_cv = threading.Condition()
if start_gc:
self.start_gc()
def __init__(self, service_name, connect_string, timeout=DEFAULT_TIMEOUT, default_port=6664):
self.SERVICE_NODE = "/" + service_name
self.AVAILABILITY_NODE = self.SERVICE_NODE + "/available"
self.MEMBERSHIP_NODE = self.SERVICE_NODE + "/members"
self.connected = False
self.timeout = timeout
self.default_port = default_port
self.conn_cv = threading.Condition()
self.conn_cv.acquire()
self.handle = zookeeper.init(connect_string, self.connection_watcher, timeout)
self.conn_cv.wait(timeout / 1000)
self.conn_cv.release()
self.watcher_lock = threading.Lock()
self.logger = logging.getLogger("sincc")
if not self.connected:
raise SinClusterClientError("Unable to connect to %s" % connect_string)
for path in [self.SERVICE_NODE, self.AVAILABILITY_NODE, self.MEMBERSHIP_NODE]:
if not zookeeper.exists(self.handle, path):
zookeeper.create(self.handle, path, "", [ZOO_OPEN_ACL_UNSAFE], 0)
self.listeners = []
# Start to watch both /members and /available
zookeeper.get_children(self.handle, self.MEMBERSHIP_NODE, self.watcher)
available = zookeeper.get_children(self.handle, self.AVAILABILITY_NODE, self.watcher)
self.available_nodes = {}
for node_id in available:
self.available_nodes[int(node_id)] = Node(int(node_id),
zookeeper.get(self.handle, self.AVAILABILITY_NODE + "/" + node_id)[0])
def connect(host=None):
global conn_cv, connected, handle
global reconnect_host
if host is None:
host = "localhost:2181"
reconnect_host = host
conn_cv.acquire()
handle = zookeeper.init(host, my_connection_watcher, 10000)
while not connected:
conn_cv.wait()
conn_cv.release()
connected = False
while not connected:
try:
zookeeper.create(handle, ROOT, "1", [ZOO_OPEN_ACL_UNSAFE], 0)
connected = True
except zookeeper.NodeExistsException as e:
# No worries
connected = True
except zookeeper.ConnectionLossException:
continue
except:
raise
def __init__(self,queuename):
self.connected = False
self.queuename = "/" + queuename
self.cv = threading.Condition()
zookeeper.set_log_stream(open("/dev/null"))
def watcher(handle,type,state,path):
print "Connected"
self.cv.acquire()
self.connected = True
self.cv.notify()
self.cv.release()
self.cv.acquire()
self.handle = zookeeper.init("localhost:2181", watcher, 10000, 0)
self.cv.wait(10.0)
if not self.connected:
print "Connection to ZooKeeper cluster timed out - is a server running on localhost:2181?"
sys.exit()
self.cv.release()
try:
zookeeper.create(self.handle,self.queuename,"queue top level", [ZOO_OPEN_ACL_UNSAFE],0)
except IOError, e:
if e.message == zookeeper.zerror(zookeeper.NODEEXISTS):
print "Queue already exists"
else:
raise e
def post(self):
request_dict = self.request.arguments
node_key = (request_dict['node_tree'])[0]
cluster_name = (request_dict['cluster_name'])[0]
zk=zookeeper.init(self.zk_connect(cluster_name))
def get_node(node_key):
"""获取子节点生成快照存到MySQL"""
if node_key == "/":
for node in zookeeper.get_children(zk,node_key):
key = "/" + node
if (zookeeper.get(zk,key)[1])['numChildren'] > 0:
get_node(key)
else:
value = (zookeeper.get(zk,key))[0]
create_time = time.time()
table = ZdSnapshot(cluster_name= cluster_name ,path=key , data=value ,create_time=self.GetNowTime())
table.save()
else:
for node in zookeeper.get_children(zk,node_key):
key = node_key + "/" + node
if (zookeeper.get(zk,key)[1])['numChildren'] > 0:
get_node(key)
else:
value = (zookeeper.get(zk,key))[0]
create_time = time.time()
table = ZdSnapshot(cluster_name= cluster_name,path=key , data=value ,create_time=self.GetNowTime())
table.save()
get_node(node_key)
self.write("生成快照成功!!!!!")
zookeeper.close(zk)
def post(self):
request_dict = self.request.arguments
node_key = (request_dict['node_key'])[0]
cluster_name = (request_dict['cluster_name'])[0]
zk=zookeeper.init(self.zk_connect(cluster_name))
data = []
def get_node_tree(node_key):
if node_key == "/":
for node in zookeeper.get_children(zk,node_key):
key = "/" + node
if (zookeeper.get(zk,key)[1])['numChildren'] > 0:
get_node_tree(key)
print key
else:
print key
else:
for node in zookeeper.get_children(zk,node_key):
key = node_key + "/" + node
if (zookeeper.get(zk,key)[1])['numChildren'] > 0:
get_node_tree(key)
data.append(key)
else:
data.append(key)
return data
get_node_tree(node_key)
data.append(node_key)
for items in data:
zookeeper.delete(zk,items)
zookeeper.close(zk)
self.write("删除成功")
def __init__(self, setting):
#Connect to ZK servers
print "Connecting to ZooKeeper ... "
self.connected = False
self.log = open("zookeeper.log", 'a')
self.log.write("\n\n=============\nZOOKEEPER LOG\n=============\n")
self.log.write(datetime.now().__str__())
zookeeper.set_log_stream(self.log)
self.cv = threading.Condition()
self.cv2 = threading.Condition()
def watcher (handle, type, state, path):
self.cv.acquire()
if state == zookeeper.CONNECTED_STATE:
print "Connected!"
self.connected = True
else:
print "Disconnected from ZooKeeper: ",
print zookeeper.zerror(state)
self.cv.notify()
self.cv.release()
self.cv.acquire()
self.zh = zookeeper.init(setting.zookeeperloc, watcher, 10000)
self.cv.wait(10.0)
if not self.connected:
print "Cannot connect to ZooKeeper. ",
print "Check that server(s) are on " + setting.zookeeperloc
sys.exit()
self.cv.release()
self.root = setting.zookeeperroot
self.createRootNode(self.root)
self.zpath = '/'
def test_logging(self):
handler = zope.testing.loggingsupport.InstalledHandler(
'ZooKeeper')
try:
handle = zookeeper.init('zookeeper.example.com:2181')
zookeeper.close(handle)
except:
pass
wait(lambda : [r for r in handler.records
if 'environment' in r.getMessage()]
)
handler.clear()
# Test that the filter for the "Exceeded deadline by" noise works.
# cheat and bypass zk by writing to the pipe directly.
os.write(zc.zk._logging_pipe[1],
'2012-01-06 16:45:44,572:43673(0x1004f6000):ZOO_WARN@'
'zookeeper_interest@1461: Exceeded deadline by 27747ms\n')
wait(lambda : [r for r in handler.records
if ('Exceeded deadline by' in r.getMessage()
and r.levelno == logging.DEBUG)
]
)
self.assert_(not [r for r in handler.records
if ('Exceeded deadline by' in r.getMessage()
and r.levelno == logging.WARNING)
])
handler.uninstall()
def __init__(self,queuename, port, is_producer=False):
self.connected = False
self.queuename = "/" + queuename
self.cv = threading.Condition()
zookeeper.set_log_stream(open("/dev/null"))
def watcher(handle,type,state,path):
print "Connected"
self.cv.acquire()
self.connected = True
self.cv.notify()
self.cv.release()
self.cv.acquire()
self.handle = zookeeper.init("localhost:%d" % port, watcher, 10000)
self.cv.wait(10.0)
if not self.connected:
print "Connection to ZooKeeper cluster timed out - is a server running on localhost:%d?" % port
sys.exit()
self.cv.release()
if is_producer:
while True:
try:
zookeeper.create(self.handle,self.queuename,"queue top level", [ZOO_OPEN_ACL_UNSAFE],0)
print "Fila criada, OK"
return
except zookeeper.NodeExistsException:
print "Tratorando filas existentes"
while True:
children = sorted(zookeeper.get_children(self.handle, self.queuename,None))
if len(children) == 0:
(data,stat) = zookeeper.get(self.handle, self.queuename, None)
zookeeper.delete(self.handle, self.queuename, stat["version"])
break
for child in children:
data = self.get_and_delete(self.queuename + "/" + child)
def __init__(self, config_server_="http://xxx.xxx.com/"):
self.config_server = config_server_
self.cache_config = []
self.zk_path = "/config"
self.zklist = self.getZkServer()
zookeeper.set_debug_level(zookeeper.LOG_LEVEL_ERROR)
self.zookeeper = zookeeper.init(self.zklist)
def __init__(self, servers,port=2181, timeout=TIMEOUT):
"""
create connection to zookeeper
@param server e.g. "localhost"
@param port (std 2181)
"""
self.connected = False
self.conn_cv = threading.Condition( )
self.handle = -1
#@todo make cross platform
zookeeper.set_log_stream(open("/dev/null"))
self.conn_cv.acquire()
q.logger.log("Connecting to %s" % (servers))
start = time.time()
self.handle = zookeeper.init(servers, self._connection_watcher, 30000)
self.conn_cv.wait(timeout)
self.conn_cv.release()
if not self.connected:
raise RuntimeError("Unable to connect to %s" % (servers))
q.logger.log("Connected in %d ms, handle is %d" % (int((time.time() - start) * 1000), self.handle))
def __init__(self, barriername, number_of_workers):
self.cv = threading.Condition()
self.connected = False
self.barrier = "/" + barriername
self.workers = number_of_workers
zookeeper.set_log_stream(open('/dev/null'))
def watcher(handle, type, state, path):
self.cv.acquire()
self.connected = True
self.cv.notify()
self.cv.release()
self.cv.acquire()
self.handle = zookeeper.init("localhost:2181", watcher, 10000)
self.cv.wait(10.0)
if not self.connected:
print "Connection to ZooKeeper cluster timed out - is a server running on localhost:2181?"
sys.exit()
self.cv.release()
try:
zookeeper.create(self.handle, self.barrier, '\x00', [ZOO_OPEN_ACL_UNSAFE], 0)
except zookeeper.NodeExistsException:
pass
except Exception, ex:
print ex
raise ex
def __init__(self, servers, acl, timeout=DEFAULT_TIMEOUT):
"""
zookeeper 操作类的封装
"""
self.epoch = 0
self.acl = acl
self.timeout = timeout
self.connected = False
#线程同步对象
self.conn_cv = threading.Condition()
self.handle = -1
#线程尝试获取锁，如果拿到就执行下面的代码，否则等待其他线程通知
self.conn_cv.acquire()
self.handle = zookeeper.init(servers, self.conn_watcher, timeout)
#wait方法会释放锁，然后阻塞，等待其他线程执行notify或者notifyAll方法，如果指定的时间没有得到通知，线程重新获取锁，如果获取到锁，继续执行下面的代码
self.conn_cv.wait(timeout / 10000)
#释放锁
self.conn_cv.release()
#检查连接状态
if not self.connected:
raise ZKClientError("Unable to connect to %s" % (servers))
if not self.add_auth(scheme=acl["scheme"], id=acl["id"]):
raise ZKClientError("add_auth to zookeeper fail")
def __init__(self,queuename):
self.connected = False
self.queuename = "/" + queuename
self.cv = threading.Condition()
#zookeeper.set_log_stream(sys.stdout)
zookeeper.set_log_stream(open("/dev/null"))
def watcher(handle,type,state,path):
print "Connected"
self.cv.acquire()
self.connected = True
self.cv.notify()
self.cv.release()
self.cv.acquire()
self.handle = zookeeper.init("localhost:2181,localhost:2182,"\
"localhost:2183,localhost:2184,localhost:2185", watcher, 10000)
self.cv.wait(10.0)
if not self.connected:
print "Connection to ZooKeeper cluster timed out - is a server running on localhost:2181?"
sys.exit()
self.cv.release()
try:
retry_on_loss(zookeeper.create)\
(self.handle,self.queuename,"queue top level", [ZOO_OPEN_ACL_UNSAFE],0)
except zookeeper.NodeExistsException:
print "Queue already exists"
def __init__(self, zkhosts, root=NODE_HQ_ROOT, alivenode='alive'):
"""zkhosts: a string or a list. list will be ','.join-ed into a string.
root: root node path (any parents must exist, if any)
"""
if not isinstance(zkhosts, basestring):
zkhosts = ','.join(zkhosts)
self.zkhosts = zkhosts
self.ROOT = root
self.alivenode = alivenode
self.nodename = os.uname()[1]
self.NODE_SERVERS = self.ROOT + '/servers'
self.NODE_ME = self.NODE_SERVERS+'/'+self.nodename
self.zh = zk.init(self.zkhosts, self.__watcher)
self.jobs = {}
self.initialize_node_structure()
if not zk.exists(self.zh, self.NODE_ME):
zk.create(self.zh, self.NODE_ME, '', PERM_WORLD)
# setup notification
zk.get_children(self.zh, self.NODE_SERVERS, self.__servers_watcher)
#zk.get_children(self.zh, self.NODE_ME, self.__watcher)
self.NODE_JOBS = self.NODE_ME+'/jobs'
zk.acreate(self.zh, self.NODE_JOBS, '', PERM_WORLD)
def __init__(self, server, root_node, host='localhost', port=2181):
self.root = root_node
self.host = host
self.port = port
self.counter = random.randint(0,2**30)
self.server = server
self.zcv = Condition()
def watcher(handle,type,state,path):
print "Z connected (%s:%s)" % (self.host, self.port)
self.zcv.acquire()
self.connected = True
self.zcv.notify()
self.zcv.release()
self.zcv.acquire()
self.connected = False
self.handle = z.init("%s:%s"%(self.host,self.port), watcher, 10000, 0)
self.zcv.wait(10.0)
if not self.connected:
print "Connection to Z cluster timed out - is a server running on %s:%s?" % (self.host,self.port)
self.connected = False
self.zcv.release()
return
self.zcv.release()
# make sure root node exists
try:
self.create(self.root,"RACS root node", [ZOO_OPEN_ACL_UNSAFE])
except IOError, e:
if e.args[0] != z.zerror(z.NODEEXISTS):
raise e
def connect(servers):
cond = threading.Condition()
connected = [False]
# We attempt a connection for 10 seconds here. This is a long timeout
# for servicing a web request, so hopefully it is successful.
def connect_watcher(zh, event, state, path):
logging.debug("CONNECT WATCHER: event=%s, state=%s, path=%s" % (event, state, path))
try:
cond.acquire()
if state == zookeeper.CONNECTED_STATE:
# We only want to notify the main thread once the state has been
# connected. We store the connected variable in an odd way because
# of the way variables are bound in the local scope for functions.
connected[0] = True
cond.notify()
finally:
cond.release()
cond.acquire()
try:
# We default to port 2181 if no port is provided as part of the host specification.
server_list = ",".join(map(lambda x: (x.find(":") > 0 and x) or "%s:2181" % x, servers))
handle = zookeeper.init(server_list, connect_watcher, 10000)
cond.wait(60.0)
finally:
# Save whether we were successful or not.
is_connected = connected[0]
cond.release()
if not(is_connected):
raise ZookeeperException("Unable to connect.")
return handle
def testconnection(self):
cv = threading.Condition()
self.connected = False
def connection_watcher(handle, type, state, path):
cv.acquire()
self.connected = True
self.assertEqual(zookeeper.CONNECTED_STATE, state)
self.handle = handle
cv.notify()
cv.release()
cv.acquire()
ret = zookeeper.init(self.host, connection_watcher)
cv.wait(15.0)
cv.release()
self.assertEqual(self.connected, True, "Connection timed out to " + self.host)
self.assertEqual(zookeeper.CONNECTED_STATE, zookeeper.state(self.handle))
self.assertEqual(zookeeper.close(self.handle), zookeeper.OK)
# Trying to close the same handle twice is an error, and the C library will segfault on it
# so make sure this is caught at the Python module layer
self.assertRaises(zookeeper.ZooKeeperException,
zookeeper.close,
self.handle)
self.assertRaises(zookeeper.ZooKeeperException,
zookeeper.get,
self.handle,
"/")
def start(self):
self.handle = zookeeper.init(self.servers, self.connection_watcher, self.timeout * 1000)
self.conn_cv.acquire()
self.conn_cv.wait(self.timeout)
self.conn_cv.release()
if not self.connected:
raise TimeoutException
def __init__(self, zkservers, lockname):
self.connected = False
self.lockname = "/" + lockname
self.uuid = str(uuid.uuid4())
self.cv = threading.Condition()
self.log = logging.getLogger('zookeeper')
def connection_watcher(_handle, _type, _state, _path):
self.cv.acquire()
self.connected = True
self.cv.notify()
self.cv.release()
self.cv.acquire()
try:
self.handle = zookeeper.init(",".join(zkservers), connection_watcher, 4000)
except:
self.log.exception('Failed to connect Zookeeper cluster (%r)' % zkservers)
self.cv.release()
raise SyncError(SYNC_LOCK_CONNECTION_FAILURE, "Failed to connect to Zookeeper cluster")
self.cv.wait(4.0)
if not self.connected:
self.log.error('Failed to connect to Zookeeper cluster (%r)' % zkservers)
self.cv.release()
raise SyncError(SYNC_LOCK_CONNECTION_FAILURE, "Failed to connect to Zookeeper cluster")
self.cv.release()
def initialize(zkservers): global connected global zh zh = zookeeper.init(zkservers,check_connected) while True: if connected: break time.sleep(0.5)
def prepare_zookeeper():
handle = zookeeper.init( host );
host_ip = socket.gethostbyname(socket.gethostname());
domain_node = "%s/%s"%(rootnode, domain)
create_node_with_parent_node( handle, domain_node );
nodename = "%s/%s/%s"%( rootnode, domain, host_ip )
return { "handle": handle, "nodename":nodename }
def start_zk(args, zk_path):
print zk_path + '/bin/zkServer.sh start'
subprocess.Popen(zk_path + '/bin/zkServer.sh start', shell=True)
hostname = socket.gethostname()
ip = socket.gethostbyname(hostname)
print ip + ':' + str(args.port)
if args.myid == 1:
while True:
time.sleep(2)
handler = zookeeper.init(ip + ':' + str(args.port))
try:
zookeeper.create(handler, "/tera", "", [{
"perms": 0x7f,
"scheme": "world",
"id": "anyone"
}], 0)
zookeeper.create(handler, "/tera/master-lock", "",
[{
"perms": 0x7f,
"scheme": "world",
"id": "anyone"
}], 0)
zookeeper.create(handler, "/tera/ts", "", [{
"perms": 0x7f,
"scheme": "world",
"id": "anyone"
}], 0)
zookeeper.create(handler, "/tera/kick", "", [{
"perms": 0x7f,
"scheme": "world",
"id": "anyone"
}], 0)
zookeeper.close(handler)
break
except:
print '#######################\nkeep trying!!\n##########################\n'
traceback.print_exc()
continue
def __init__(self, host, port, queuename):
self.connected = False
self.queuename = "/" + queuename
self.cv = threading.Condition()
zookeeper.set_log_stream(open("/dev/null"))
def watcher(handle, type, state, path):
# print "Connected"
self.cv.acquire()
self.connected = True
self.cv.notify()
self.cv.release()
self.cv.acquire()
host_s = host + ":" + str(port)
self.handle = zookeeper.init(host_s, watcher, 2000000)
self.cv.wait(10.0)
if not self.connected:
print(
"Connection to ZooKeeper cluster timed out "
"- is a server running on %s?" % host_s)
sys.exit()
self.cv.release()
def set_up(self):
self.cv = threading.Condition()
self.connected = False
def connection_watcher(handle, type, state, path):
self.cv.acquire()
self.connected = True
self.cv.notify()
self.cv.release()
self.cv.acquire()
self.handle = zookeeper.init(self.host, connection_watcher)
self.cv.wait(10.0)
self.cv.release()
if not self.connected:
raise Exception("Couldn't connect to host -", self.host)
# set up the NG2znode look-up table:
if not zookeeper.exists(self.handle, '/look-up', None):
d = { 'default-graph' : '/dg' }
zookeeper.create(self.handle, '/look-up', json.dumps(d), [ZOO_OPEN_ACL_UNSAFE])
else:
(self.NG2znode, stat) = zookeeper.get(self.handle, '/look-up', None)
self.NG2znode = json.loads(str(self.NG2znode))
_logger.debug('%s' %self.NG2znode['default-graph'])
def __init__(self, servers, acl, timeout=DEFAULT_TIMEOUT):
"""
zookeeper 操作类的封装
"""
self.epoch = 0
self.acl = acl
self.timeout = timeout
self.connected = False
#线程同步对象
self.conn_cv = threading.Condition()
self.handle = -1
#线程尝试获取锁，如果拿到就执行下面的代码，否则等待其他线程通知
self.conn_cv.acquire()
self.handle = zookeeper.init(servers, self.conn_watcher, timeout)
#wait方法会释放锁，然后阻塞，等待其他线程执行notify或者notifyAll方法，如果指定的时间没有得到通知，线程重新获取锁，如果获取到锁，继续执行下面的代码
self.conn_cv.wait(timeout / 10000)
#释放锁
self.conn_cv.release()
#检查连接状态
if not self.connected:
raise ZKClientError("Unable to connect to %s" % (servers))
if not self.add_auth(scheme=acl["scheme"], id=acl["id"]):
raise ZKClientError("add_auth to zookeeper fail")
def main():
from optparse import OptionParser as OptionParser
parser = OptionParser()
parser.add_option('-v',
'--verbose',
action="store_true",
dest="flag_verbose",
help='Triggers debugging traces')
parser.add_option(
'-s', '--smart',
action="store_true", dest="SMART", default=False,
help="Delete onle the members belong to services with multiple" \
" members")
parser.add_option('-d',
'--dry-run',
action="store_true",
dest="DRY",
default=False,
help="Do not delete, just print")
parser.add_option('-n',
'--min-services',
type=int,
action="store",
dest="NUM",
default=4,
help="Do not delete election if less the NUM")
parser.add_option('-1',
'--alone',
action="store_true",
dest="ALONE",
default=False,
help="Also consider members alone in their group")
(options, args) = parser.parse_args(sys.argv)
# Logging configuration
if options.flag_verbose:
logging.basicConfig(format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %I:%M:%S',
level=logging.DEBUG)
else:
logging.basicConfig(format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %I:%M:%S',
level=logging.INFO)
if len(args) < 2:
raise ValueError("not enough CLI arguments: NS TYPE [TYPE...]")
ns = args[1]
cnxstr = load_namespace_conf(ns)['zookeeper']
zookeeper.set_debug_level(zookeeper.LOG_LEVEL_INFO)
zh = zookeeper.init(cnxstr)
for srvtype in args[2:]:
for group in namespace_tree(ns, srvtype):
logging.debug(">DIR %s", group)
for node in list_problematic_nodes(zh, group, options):
delete_node(zh, node, options)
zookeeper.close(zh)
len1 = len(runtimecontainer[i].split('='))
#print len1
if (len1 == 2):
dic1 = runtimecontainer[i].split('=')[0]
value1 = runtimecontainer[i].split('=')[1]
items.append((str(dic1), str(value1)))
#print items
d = dict(items)
return d
#函数结束
#初始化zookeeper连接
zk = zoo.init("10.245.0.96:2181")
zoo.set_debug_level(zoo.LOG_LEVEL_ERROR)
#获取所有的应用
path = "/UCMP/instances"
try:
application = zoo.get_children(zk, path)
except:
print
print "1===没有相关节点信息,请确认程序名称或者版本信息"
zoo.close(zk)
os._exit(0)
applicationlist = []
for index in range(len(application)):
#if ( list(application[index])[-1] != '_' and list(application[index])[0] != '&'):
if (not (application[index].endswith('_')
or application[index].startswith('&'))):
import os
import zookeeper
import pykeeper
host = os.environ['DOTCLOUD_ZOOKEEPER_CONNECTION_HOST']
port = os.environ['DOTCLOUD_ZOOKEEPER_CONNECTION_PORT']
connection_str = '%s:%s' % (host, port)
#client = pykeeper.ZooKeeper(connection_str)
def callback(*args, **kwargs):
print args
print kwargs
handle = zookeeper.init(connection_str, callback)
zookeeper.client_id(handle)
zookeeper.exists(handle, '/')
zookeeper.get(handle, '/')
zookeeper.get_children(handle, '/')
zookeeper.state(handle)
def _initialize_register(self):
if self._register_handler is None:
self._register_handler = zookeeper.init(self._hosts,
self._register_watcher)
def _initialize_discovery(self):
if self._discovery_handler is None:
self._discovery_handler = zookeeper.init(self._hosts,
self._discovery_watcher)
import zookeeper
import time
def myWatch(handler, type, state, path):
print "handler:" + str(handler) + ",type:" + str(type) + ",state:" + str(
state) + ",path:" + path
# print 'a'
# print str(type)
# print 'b'
path = '/haha'
handler = zookeeper.init("10.0.100.131:6181")
#zookeeper.get(handler,path,myWatch)
while True:
data = zookeeper.get(handler, path, myWatch)
# print data
time.sleep(50)
def main():
from optparse import OptionParser as OptionParser
parser = OptionParser()
parser.add_option('-v',
'--verbose',
action="store_true",
dest="flag_verbose",
default=False,
help='Triggers debugging traces')
parser.add_option('--lazy',
action="store_true",
dest="LAZY",
default=False,
help='Quickly check if things seem OK.')
parser.add_option(
'--slow',
action="store_true",
dest="SLOW",
default=False,
help='Send small batches to avoid timeouts on slow hosts.')
parser.add_option('--avoid',
action="append",
type="string",
dest="AVOID_TYPES",
help='Avoid entries for the specified service types')
(options, args) = parser.parse_args(sys.argv)
# Logging configuration
if options.flag_verbose:
logging.basicConfig(format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %I:%M:%S',
level=logging.DEBUG)
else:
logging.basicConfig(format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %I:%M:%S',
level=logging.INFO)
if len(args) < 2:
raise ValueError("not enough CLI arguments")
ns = args[1]
cnxstr = load_namespace_conf(ns)['zookeeper']
zookeeper.set_debug_level(zookeeper.LOG_LEVEL_INFO)
for shard in cnxstr.split(";"):
logging.info("ZK=%s", shard)
zh = zookeeper.init(shard)
# synchronous creation of the root
try:
zookeeper.create(zh, PREFIX, '', acl_openbar, 0)
except zookeeper.NodeExistsException:
pass
missing = True
if options.LAZY:
_m = False
for t, _, _ in SRVTYPES:
try:
_, _ = zookeeper.get(zh, PREFIX_NS + '/' + ns + '/el/' + t)
except Exception:
_m = True
missing = _m
if missing:
create_tree(zh, namespace_tree(ns, options), options)
zookeeper.close(zh)
#/usr/bin/env python
#coding=utf8
"""
# Author: kellanfan
# Created Time : Tue 27 Nov 2018 01:52:20 PM CST
# File Name: myzktest.py
# Description:
"""
import zookeeper
def myWatch(zk, type, state, path):
print("zk:", str(type), str(state), str(path))
zookeeper.get(zk, path, myWatch)
zk = zookeeper.init("192.168.10.10:2181")
data = zookeeper.get(zk, "/zk-bbb", myWatch)
print(data)
import zookeeper
import time
import sys
import threading
#zk = zookeeper.init("10.10.1.201:6181,10.10.1.212:6181,10.10.1.213:6181")
zk = zookeeper.init("10.0.100.131:6181")
#a=zk.get_children(zk, "/", None)
#zookeeper.create(zk,"/test3","hehe",[{"perms":0x1f,"scheme":"world","id":"anyone"}],0)
def set(i):
nodeinfo = '/node%d' % i
datainfo = 'datajfajdfkjaslkjdfajsfiwerkjkjelkjkljafd%d' % i
zookeeper.create(zk, nodeinfo, datainfo, [{
"perms": 0x1f,
"scheme": "world",
"id": "anyone"
}], 0)
print 'insert number %d is ok! ' % i
def get(i):
nodeinfo = '/node%d' % i
data = zookeeper.get(zk, nodeinfo)
print data
start_time = time.time()
for i in range(1100, 2100):
t = threading.Thread(target=get, args=(i, ))
def connect(self):
self.handle = zookeeper.init(self.servers, self._global_watcher)
self.on_state(self, self.state_name)
def get_connected_handles(cnxstr):
zookeeper.set_debug_level(zookeeper.LOG_LEVEL_INFO)
for shard in cnxstr.split(";"):
zh = zookeeper.init(shard)
yield ZkHandle(zh)
def reconnect(self):
"""Attempt to reconnect to ZK."""
if self._stopped.is_set():
self._safe_close()
return
def safe_close(zh):
try:
zookeeper.close(zh)
except:
# TODO(wickman) When the SystemError bug is fixed in zkpython, narrow this except clause.
pass
def activate():
self._authenticated.set()
self._live.set()
def on_authentication(zh, rc):
if self._zh != zh:
safe_close(zh)
return
if rc == zookeeper.OK:
activate()
def maybe_authenticate():
if self._authenticated.is_set() or not self._credentials:
activate()
return
try:
scheme, credentials = self._credentials
zookeeper.add_auth(self._zh, scheme, credentials,
on_authentication)
except zookeeper.ZooKeeperException as e:
self._logger('Failed to authenticate: %s' % e)
def connection_handler(handle, type, state, path):
if self._zh != handle:
safe_close(handle)
return
if self._stopped.is_set():
return
if self._watch:
self._watch(self, type, state, path)
if state == zookeeper.CONNECTED_STATE:
self._logger('Connection started, setting live.')
maybe_authenticate()
self._clear_completions()
elif state == zookeeper.EXPIRED_SESSION_STATE:
self._logger('Session lost, clearing live state.')
self._session_expirations.increment()
self._live.clear()
self._authenticated.clear()
self._zh = None
self._init_count = 0
self.reconnect()
else:
self._logger('Connection lost, clearing live state.')
self._connection_losses.increment()
self._live.clear()
# this closure is exposed for testing only -- in order to simulate session events.
self._handler = connection_handler
timeout_ms = int(self._timeout_secs * 1000)
while True:
self._safe_close()
servers = self.expand_ensemble(self._servers)
self._log('Connecting to ZK hosts at %s' % servers)
self._zh = zookeeper.init(servers, connection_handler, timeout_ms)
self._init_count += 1
self._live.wait(self._timeout_secs + 1)
if self._live.is_set():
break
elif self._max_reconnects > 0 and self._init_count >= self._max_reconnects:
self._safe_close()
raise ZooKeeper.ConnectionTimeout(
'Timed out waiting for ZK connection to %s' % servers)
self._log('Successfully connected to ZK at %s' % servers)
def start(self):
if self._handler is None:
self._handler = zookeeper.init(self._hosts,
self._connection_state_watcher)
LOGGER.info("create zookeeper client successfully, "
"handler is: %s" % self._handler)
zookeeper.set_log_stream(f)
connected = False
conn_cv = threading.Condition( )
def my_connection_watcher(handle,type,state,path):
global connected, conn_cv
print("Connected, handle is ", handle)
conn_cv.acquire()
connected = True
conn_cv.notifyAll()
conn_cv.release()
conn_cv.acquire()
print("Connecting to localhost:2181 -- ")
handle = zookeeper.init("localhost:2181", my_connection_watcher, 10000, 0)
while not connected:
conn_cv.wait()
conn_cv.release()
def my_getc_watch( handle, type, state, path ):
print("Watch fired -- ")
print(type, state, path)
ZOO_OPEN_ACL_UNSAFE = {"perms":0x1f, "scheme":"world", "id" :"anyone"};
try:
zookeeper.create(handle, "/zk-python", "data", [ZOO_OPEN_ACL_UNSAFE], 0)
zookeeper.get_children(handle, "/zk-python", my_getc_watch)
for i in xrange(5):
print("Creating sequence node ", i, " ", zookeeper.create(handle, "/zk-python/sequencenode", "data", [ZOO_OPEN_ACL_UNSAFE], zookeeper.SEQUENCE ))
def _init_handle(self):
if hasattr(self, '_zh'):
zookeeper.close(self._zh)
self._zh = zookeeper.init(self._hosts, self._global_watcher(), 1000)
_pos = -1
else:
item[1]['__sep__'] = sep + '├─'
_pos = 0
html = html + filter_node_treegrid(item[1], depth + 1, _pos, sep)
return html
######################################################################
# 开始代码执行 #
######################################################################
# zookeeper连接
zk = zookeeper.init(config.zookeeper['server'], watcher)
# 初始化、配置app
app = Flask(__name__)
app.config.from_mapping(config.site)
# 注册模板函数到jinja2
app.jinja_env.filters['datetime'] = filter_datetime
app.jinja_env.filters['json_dumps'] = filter_json_dumps
app.jinja_env.filters['url_encode'] = filter_url_encode
app.jinja_env.filters['url_quote'] = filter_url_quote
app.jinja_env.filters['node_treegrid'] = filter_node_treegrid
# 在所有的页面请求前执行
@app.before_request
for k in path:
p = path[k]
_project = project.copy()
_project.update({'key': k})
if not os.path.exists(G.tmp_dir + p):
os.makedirs(G.tmp_dir + p)
if not G.project.has_key(p):
G.project[p] = []
G.project[p].append(_project)
# zookeeper 连接
G.zookeeper = zookeeper.init(G.zookeeper_server, watcher)
# /service/<name>
zookeeper_node_set(SERVIE_NODE / G.name, json.dumps(G.project),
zookeeper.PERM_ALL)
# /online/<name>
nodepath = ONLINE_NODE / G.name
if zookeeper.exists(G.zookeeper, nodepath):
raise BaseException('The same service has been run in other places')
zookeeper_node_set(nodepath,
'',
zookeeper.PERM_READ,
ephemeral=True,
parent_perms=zookeeper.PERM_ALL)
except BaseException, e:
print e.message
import zookeeper
handler = zookeeper.init('localhost:2181')
#0表示持久化的,1表示持久化+序号,2表示瞬时的,3表示瞬时加序号型的
zookeeper.create(handler, '/zkpython_create_node', 'mydata1')
