def __init__(self, **config):
logger_name = config.get("logger_name")
self.logger = logging.getLogger(logger_name) if logger_name else Null()
self.zkhandle = None
self.auth = None
self.cv = threading.Condition()
try:
auth_config = config.get("auth")
if auth_config is not None:
auth_scheme = auth_config["scheme"]
auth_data = auth_config["data"]
self.auth = (auth_scheme, auth_data)
zklogfile_path, zklog_level = config.get("ZookeeperLog", ("/dev/stderr", "WARN"))
self.connection_timeout = config["timeout"]
self.zkhosts = ",".join(config["host"])
except KeyError as err:
self.logger.exception("Missing configuration option: %s", err)
raise
except Exception as err:
self.logger.exception("Unknown configuration error: %s", err)
raise
try:
_f = open(zklogfile_path, "a")
except IOError as err:
self.logger.error("Unable to open logfile %s %s", zklogfile_path, err)
else:
zookeeper.set_log_stream(_f)
zookeeper.set_debug_level(LOG_LEVELS.get(zklog_level.upper(), zookeeper.LOG_LEVEL_WARN))
self.connect()
if zookeeper.state(self.zkhandle) == zookeeper.CONNECTED_STATE:
self.logger.info("Connected to Zookeeper successfully")
else:
raise zookeeper.ZooKeeperException("Unable to connect " "to Zookeeper")
def on_auth_callback(state, result):
with self.cv:
if result == zookeeper.AUTHFAILED:
self.logger.error(zookeeper.zerror(zookeeper.AUTHFAILED))
self.logger.info("on_auth: state %s, result %s", state, result)
self.cv.notify()
if self.auth:
self.logger.info("Auth using %s", self.auth[0])
with self.cv:
res = zookeeper.add_auth(self.zkhandle, self.auth[0], self.auth[1], on_auth_callback)
if res != zookeeper.OK:
self.logger.error("Invalid status %d", zookeeper.zerror(res))
raise Exception("Invalid status")
self.cv.wait(self.connection_timeout)
if zookeeper.state(self.zkhandle) == zookeeper.AUTH_FAILED_STATE:
raise zookeeper.ZooKeeperException("authentication failed")
def state(self):
"""
What's the current state of this connection, result is an
integer value corresponding to zoookeeper module constants.
"""
if self.connected:
return zookeeper.state(self.handle)
def testconnection(self):
cv = threading.Condition()
self.connected = False
def connection_watcher(handle, type, state, path):
cv.acquire()
self.connected = True
self.assertEqual(zookeeper.CONNECTED_STATE, state)
self.handle = handle
cv.notify()
cv.release()
cv.acquire()
ret = zookeeper.init(self.host, connection_watcher)
cv.wait(15.0)
cv.release()
self.assertEqual(self.connected, True, "Connection timed out to " + self.host)
self.assertEqual(zookeeper.CONNECTED_STATE, zookeeper.state(self.handle))
self.assertEqual(zookeeper.close(self.handle), zookeeper.OK)
# Trying to close the same handle twice is an error, and the C library will segfault on it
# so make sure this is caught at the Python module layer
self.assertRaises(zookeeper.ZooKeeperException,
zookeeper.close,
self.handle)
self.assertRaises(zookeeper.ZooKeeperException,
zookeeper.get,
self.handle,
"/")
def connect(self, timeout = None):
'''Connects to the zookeeper server'''
# if no timeout provided, thake the configured one
if timeout is None:
timeout = self._timeout
# check, if we're already connected
if self._handle is not None and not self.is_connected():
raise RuntimeError('Already connected')
return
condition = threading.Condition()
def connection_watch(handle, type, state, path):
condition.acquire()
condition.notify()
condition.release()
# try to connect
condition.acquire()
self._handle = zookeeper.init(
','.join(self._servers),
connection_watch,
self._timeout * 1000)
condition.wait(self._timeout)
condition.release()
if zookeeper.state(self._handle) != zookeeper.CONNECTED_STATE:
zookeeper.close(self._handle)
raise RuntimeError(
'unable to connect to %s ' % (' or '.join(self._servers)))
zookeeper.set_watcher(self._handle, self.__global_watch)
def testhandlereuse(self):
"""
Test a) multiple concurrent connections b) reuse of closed handles
"""
cv = threading.Condition()
self.connected = False
def connection_watcher(handle, type, state, path):
cv.acquire()
self.connected = True
self.assertEqual(zookeeper.CONNECTED_STATE, state)
self.handle = handle
cv.notify()
cv.release()
cv.acquire()
handles = [ zookeeper.init(self.host) for i in xrange(10) ]
ret = zookeeper.init(self.host, connection_watcher)
cv.wait(15.0)
cv.release()
self.assertEqual(self.connected, True, "Connection timed out to " + self.host)
self.assertEqual(True, all( [ zookeeper.state(handle) == zookeeper.CONNECTED_STATE for handle in handles ] ),
"Not all connections succeeded")
oldhandle = handles[3]
zookeeper.close(oldhandle)
newhandle = zookeeper.init(self.host)
# This assertion tests *internal* behaviour; i.e. that the module
# correctly reuses closed handles. This is therefore implementation
# dependent.
self.assertEqual(newhandle, oldhandle, "Didn't get reused handle")
def ensureCreated(self,path,data="",flags=zookeeper.EPHEMERAL):
"""
It's possible not to get the flags you want here if the node already exists
"""
self.assertEqual(zookeeper.CONNECTED_STATE, zookeeper.state(self.handle), "Not connected!")
try:
self.assertEqual(path, zookeeper.create(self.handle, path, data, [ZOO_OPEN_ACL_UNSAFE], flags))
except zookeeper.NodeExistsException:
pass
def state(self):
"""
返回zk的连接状况
"""
try:
stat = zookeeper.state(self.handle)
return STATE_NAME_MAPPING.get(stat)
except:
log.error(traceback.format_exc())
return "unknwon"
def close(self):
'''Overwrites the zookeeper.close() method'''
if self._handle is None:
logger.error('Can not close an uninitialized connection.')
return
logger.debug('closing connection')
try:
_state = zookeeper.state(self._handle)
except zookeeper.ZooKeeperException:
logger.warn('Connection is already closed')
return True
for _ in range(3):
try:
return zookeeper.close(self._handle) == zookeeper.OK
except: # zookeeper.ConnectionLossException:
logger.info('Got exception while closing. Retrying...')
logger.error('Failed closing the zookeeper connection')
return False
def __check_handle__(self) :
#self.conn_handler.acquire()
try :
#if self.handle != -1 :
# return
if self.handle != -1 :
state = zookeeper.state(self.handle)
if (state == zookeeper.CONNECTED_STATE) or (state == zookeeper.CONNECTING_STATE) :
return
print "handle != -1, BUT the state is not 'CONNECTED_STATE' ..."
self.__close__()
self.reconnect_time = time.time() + random.randint(RECONNECT_MIN_MIILISECS, RECONNECT_MAX_MIILISECS)
return
now = time.time()
if (self.reconnect_time < now) or (self.reconnect_time > (now + RECONNECT_MAX_MIILISECS)) :
if VERBOSE: print("Connecting to %s" % (self.servers))
start_time = time.time()
self.handle = zookeeper.init(self.servers, self.__event_watcher__, self.timeout)
self.connected = False
if VERBOSE:
print("Connected in %d ms, handle is %d"
% (int((time.time() - start_time) ), self.handle))
except Exception, e :
print "Exception in __check_handle__() :", Exception, e
self.reconnect_time = time.time() + random.randint(RECONNECT_MIN_MIILISECS, RECONNECT_MAX_MIILISECS)
def connect(self):
def connect_watcher(handle, w_type, state, path):
"""Callback for connect()"""
with self.cv:
if state == zookeeper.CONNECTED_STATE:
self.logger.debug("connect_watcher: CONNECTED_STATE")
else:
self.logger.debug("connect_watcher: state %d", state)
self.cv.notify()
with self.cv:
try:
# zookeeper.init accepts timeout in ms
recv_timeout = int(self.connection_timeout * 1e3)
self.zkhandle = zookeeper.init(self.zkhosts, connect_watcher, recv_timeout)
except Exception as err:
self.logger.exception("Unable to init zookeeper: %s", err)
raise err
else:
while True:
self.logger.debug("Connecting to Zookeeper... Wait %d", self.connection_timeout)
self.cv.wait(self.connection_timeout)
if zookeeper.state(self.zkhandle) != zookeeper.CONNECTING_STATE:
break
def testhandlereuse(self):
"""
Test a) multiple concurrent connections b) reuse of closed handles
"""
cv = threading.Condition()
self.connected = False
def connection_watcher(handle, type, state, path):
cv.acquire()
self.connected = True
self.assertEqual(zookeeper.CONNECTED_STATE, state)
self.handle = handle
cv.notify()
cv.release()
cv.acquire()
handles = [zookeeper.init(self.host) for i in xrange(10)]
ret = zookeeper.init(self.host, connection_watcher)
cv.wait(15.0)
cv.release()
self.assertEqual(self.connected, True,
"Connection timed out to " + self.host)
self.assertEqual(
True,
all([
zookeeper.state(handle) == zookeeper.CONNECTED_STATE
for handle in handles
]), "Not all connections succeeded")
oldhandle = handles[3]
zookeeper.close(oldhandle)
newhandle = zookeeper.init(self.host)
# This assertion tests *internal* behaviour; i.e. that the module
# correctly reuses closed handles. This is therefore implementation
# dependent.
self.assertEqual(newhandle, oldhandle, "Didn't get reused handle")
def connected(self):
return self.zkhandle and zookeeper.state(self.zkhandle) == zookeeper.CONNECTED_STATE
import os
import zookeeper
import pykeeper
host = os.environ['DOTCLOUD_ZOOKEEPER_CONNECTION_HOST']
port = os.environ['DOTCLOUD_ZOOKEEPER_CONNECTION_PORT']
connection_str = '%s:%s' % (host, port)
#client = pykeeper.ZooKeeper(connection_str)
def callback(*args, **kwargs):
print args
print kwargs
handle = zookeeper.init(connection_str, callback)
zookeeper.client_id(handle)
zookeeper.exists(handle, '/')
zookeeper.get(handle, '/')
zookeeper.get_children(handle, '/')
zookeeper.state(handle)
def ensureDeleted(self,path):
self.assertEqual(zookeeper.CONNECTED_STATE, zookeeper.state(self.handle), "Not connected!")
try:
self.assertEqual(zookeeper.OK, zookeeper.delete(self.handle, path))
except zookeeper.NoNodeException:
pass
def state(self):
if self.handle is None:
return zookeeper.CONNECTING_STATE
return zookeeper.state(self.handle)
def state(self):
return zookeeper.state(self._zhandle)
def __init__(self, **config):
logger_name = config.get('logger_name')
self.logger = logging.getLogger(logger_name) if logger_name else Null()
self.zkhandle = None
self.auth = None
self.cv = threading.Condition()
try:
auth_config = config.get("auth")
if auth_config is not None:
auth_scheme = auth_config["scheme"]
auth_data = auth_config["data"]
self.auth = (auth_scheme, auth_data)
zklogfile_path, zklog_level = config.get("ZookeeperLog",
("/dev/stderr", "WARN"))
self.connection_timeout = config['timeout']
self.zkhosts = ','.join(config['host'])
except KeyError as err:
self.logger.exception("Missing configuration option: %s", err)
raise
except Exception as err:
self.logger.exception("Unknown configuration error: %s", err)
raise
try:
_f = open(zklogfile_path, 'a')
except IOError as err:
self.logger.error("Unable to open logfile %s %s",
zklogfile_path, err)
else:
zookeeper.set_log_stream(_f)
zookeeper.set_debug_level(LOG_LEVELS.get(zklog_level.upper(),
zookeeper.LOG_LEVEL_WARN))
self.connect()
if zookeeper.state(self.zkhandle) == zookeeper.CONNECTED_STATE:
self.logger.info('Connected to Zookeeper successfully')
else:
raise zookeeper.ZooKeeperException('Unable to connect '
'to Zookeeper')
def on_auth_callback(state, result):
with self.cv:
if result == zookeeper.AUTHFAILED:
self.logger.error(zookeeper.zerror(zookeeper.AUTHFAILED))
self.logger.info("on_auth: state %s, result %s",
state, result)
self.cv.notify()
if self.auth:
self.logger.info("Auth using %s", self.auth[0])
with self.cv:
res = zookeeper.add_auth(self.zkhandle, self.auth[0],
self.auth[1], on_auth_callback)
if res != zookeeper.OK:
self.logger.error("Invalid status %d",
zookeeper.zerror(res))
raise Exception("Invalid status")
self.cv.wait(self.connection_timeout)
if zookeeper.state(self.zkhandle) == zookeeper.AUTH_FAILED_STATE:
raise zookeeper.ZooKeeperException('authentication failed')
def connected(self):
return self.zkhandle and\
zookeeper.state(self.zkhandle) == zookeeper.CONNECTED_STATE
def state_name(self):
if self.handle is None:
return None
return STATE_NAME_MAPPING[zookeeper.state(self.handle)]
def state(self):
"""Returns zookeeper api state."""
return zookeeper.state(self.handle)
def state_name(self):
if self.handle is None:
return None
return STATE_NAME_MAPPING[zookeeper.state(self.handle)]
def ensureDeleted(self,path):
self.assertEqual(zookeeper.CONNECTED_STATE, zookeeper.state(self.handle), "Not connected!")
try:
self.assertEqual(zookeeper.OK, zookeeper.delete(self.handle, path))
except zookeeper.NoNodeException:
pass
def __del__(self):
'''Makes sure, that the connection is not left open'''
logger.debug('ConnectionWatcher: __del__')
if self._handle and zookeeper.state(self._handle) == zookeeper.CONNECTED_STATE:
self.logger.warn('Closing open zookeeper connection')
self.close()
def state(self):
"""Returns zookeeper api state."""
return zookeeper.state(self.handle)
import os
import zookeeper
import pykeeper
host = os.environ['DOTCLOUD_ZOOKEEPER_CONNECTION_HOST']
port = os.environ['DOTCLOUD_ZOOKEEPER_CONNECTION_PORT']
connection_str = '%s:%s' % (host, port)
#client = pykeeper.ZooKeeper(connection_str)
def callback(*args, **kwargs):
print args
print kwargs
handle = zookeeper.init(connection_str, callback)
zookeeper.client_id(handle)
zookeeper.exists(handle, '/')
zookeeper.get(handle, '/')
zookeeper.get_children(handle, '/')
zookeeper.state(handle)
def state(self):
return zookeeper.state(self.handle)
