def __init__(self, servers,port=2181, timeout=TIMEOUT):
"""
create connection to zookeeper
@param server e.g. "localhost"
@param port (std 2181)
"""
self.connected = False
self.conn_cv = threading.Condition( )
self.handle = -1
#@todo make cross platform
zookeeper.set_log_stream(open("/dev/null"))
self.conn_cv.acquire()
q.logger.log("Connecting to %s" % (servers))
start = time.time()
self.handle = zookeeper.init(servers, self._connection_watcher, 30000)
self.conn_cv.wait(timeout)
self.conn_cv.release()
if not self.connected:
raise RuntimeError("Unable to connect to %s" % (servers))
q.logger.log("Connected in %d ms, handle is %d" % (int((time.time() - start) * 1000), self.handle))
def __init__(self,queuename, port, is_producer=False):
self.connected = False
self.queuename = "/" + queuename
self.cv = threading.Condition()
zookeeper.set_log_stream(open("/dev/null"))
def watcher(handle,type,state,path):
print "Connected"
self.cv.acquire()
self.connected = True
self.cv.notify()
self.cv.release()
self.cv.acquire()
self.handle = zookeeper.init("localhost:%d" % port, watcher, 10000)
self.cv.wait(10.0)
if not self.connected:
print "Connection to ZooKeeper cluster timed out - is a server running on localhost:%d?" % port
sys.exit()
self.cv.release()
if is_producer:
while True:
try:
zookeeper.create(self.handle,self.queuename,"queue top level", [ZOO_OPEN_ACL_UNSAFE],0)
print "Fila criada, OK"
return
except zookeeper.NodeExistsException:
print "Tratorando filas existentes"
while True:
children = sorted(zookeeper.get_children(self.handle, self.queuename,None))
if len(children) == 0:
(data,stat) = zookeeper.get(self.handle, self.queuename, None)
zookeeper.delete(self.handle, self.queuename, stat["version"])
break
for child in children:
data = self.get_and_delete(self.queuename + "/" + child)
def main(argv=sys.argv, quiet=False):
args = parse(argv[1:])
d = args.__dict__
prog = os.path.basename(argv[0])
title = "%s - %s" % (prog, DESCRIPTION.strip('.').lower())
sys.stderr.write(title + "\n" + "-" * len(title) + "\n")
# Set up logging
levels = tuple(reversed(LEVELS))
verbosity = d.pop('verbosity')
log.setLevel(levels[min(verbosity, len(levels) - 1)])
if verbosity:
FORMAT = '%(asctime)-15s %(message)s'
else:
FORMAT = '>>> %(message)s'
logging.basicConfig(format=FORMAT)
logging.getLogger().setLevel(logging.WARN)
set_log_stream(open('/dev/null', 'w'))
log.debug("all arguments parsed.")
package = pkg_resources.get_distribution("pop")
log.debug("%s system initialized." % package.egg_name().lower())
# Invoke command
run(d.pop('func'), verbosity > 1, d)
def _getLogObserver(self):
if self._logfilename == "-":
log_file = sys.stdout
else:
log_file = open(self._logfilename, "a")
# Setup file logger
log_handler = logging.StreamHandler(log_file)
formatter = logging.Formatter(
"%(asctime)s: %(name)s@%(levelname)s: %(message)s")
log_handler.setFormatter(formatter)
# Also capture zookeeper logs (XXX not compatible with rotation)
zookeeper.set_log_stream(log_file)
# Configure logging.
root = logging.getLogger()
root.addHandler(log_handler)
root.setLevel(logging.getLevelName(self._loglevel))
# Twisted logging is painfully verbose on twisted.web, and
# there isn't a good way to distinguish different channels
# within twisted, so just utlize error level logging only for
# all of twisted.
twisted_log = logging.getLogger("twisted")
twisted_log.setLevel(logging.ERROR)
observer = PythonLoggingObserver()
return observer.emit
def initialize(self):
#self.handle = zookeeper.init(communication_config.NETWORK_OPTIONS['zookeeper-server'])
devnull = open(os.devnull,"w")
zookeeper.set_log_stream(devnull)
self.handle = zookeeper.init(maneuver_negotiator_config.GENERAL_OPTIONS['zookeeper-server'])
#id1 = int(input())
#self.id1 = int(args.id)
#setup network:
# global host_ip
# host_ip = communication_config.NETWORK_OPTIONS['network-prefix'] + str(id1)
#aID = id1
#agent[0] = id1
#port = communication_config.NETWORK_OPTIONS['port-start'] + id1
#print(id1)
self.t_update = Timer(self.self.TA, self.update)
self.t_update.start()
self.setup_ros()
#start thread that listens on network for udp packets from other cars
net_thread = threading.Thread(target= self.udp_msg_processor)
net_thread.start()
def silence(__once=[]):
if __once:
return
__once.append(0)
zookeeper.set_debug_level(zookeeper.LOG_LEVEL_ERROR)
zookeeper.set_log_stream(file("/dev/null", "w"))
def __init__(self,queuename):
self.connected = False
self.queuename = "/" + queuename
self.cv = threading.Condition()
#zookeeper.set_log_stream(sys.stdout)
zookeeper.set_log_stream(open("/dev/null"))
def watcher(handle,type,state,path):
print "Connected"
self.cv.acquire()
self.connected = True
self.cv.notify()
self.cv.release()
self.cv.acquire()
self.handle = zookeeper.init("localhost:2181,localhost:2182,"\
"localhost:2183,localhost:2184,localhost:2185", watcher, 10000)
self.cv.wait(10.0)
if not self.connected:
print "Connection to ZooKeeper cluster timed out - is a server running on localhost:2181?"
sys.exit()
self.cv.release()
try:
retry_on_loss(zookeeper.create)\
(self.handle,self.queuename,"queue top level", [ZOO_OPEN_ACL_UNSAFE],0)
except zookeeper.NodeExistsException:
print "Queue already exists"
def __init__(self, setting):
#Connect to ZK servers
print "Connecting to ZooKeeper ... "
self.connected = False
self.log = open("zookeeper.log", 'a')
self.log.write("\n\n=============\nZOOKEEPER LOG\n=============\n")
self.log.write(datetime.now().__str__())
zookeeper.set_log_stream(self.log)
self.cv = threading.Condition()
self.cv2 = threading.Condition()
def watcher(handle, type, state, path):
self.cv.acquire()
if state == zookeeper.CONNECTED_STATE:
print "Connected!"
self.connected = True
else:
print "Disconnected from ZooKeeper: ",
print zookeeper.zerror(state)
self.cv.notify()
self.cv.release()
self.cv.acquire()
self.zh = zookeeper.init(setting.zookeeperloc, watcher, 10000)
self.cv.wait(10.0)
if not self.connected:
print "Cannot connect to ZooKeeper. ",
print "Check that server(s) are on " + setting.zookeeperloc
sys.exit()
self.cv.release()
self.root = setting.zookeeperroot
self.createRootNode(self.root)
self.zpath = '/'
def __init__(self,queuename):
self.connected = False
self.queuename = "/" + queuename
self.cv = threading.Condition()
zookeeper.set_log_stream(open("/dev/null"))
def watcher(handle,type,state,path):
print "Connected"
self.cv.acquire()
self.connected = True
self.cv.notify()
self.cv.release()
self.cv.acquire()
self.handle = zookeeper.init("localhost:2181", watcher, 10000, 0)
self.cv.wait(10.0)
if not self.connected:
print "Connection to ZooKeeper cluster timed out - is a server running on localhost:2181?"
sys.exit()
self.cv.release()
try:
zookeeper.create(self.handle,self.queuename,"queue top level", [ZOO_OPEN_ACL_UNSAFE],0)
except IOError, e:
if e.message == zookeeper.zerror(zookeeper.NODEEXISTS):
print "Queue already exists"
else:
raise e
def __init__(self, setting):
#Connect to ZK servers
print "Connecting to ZooKeeper ... "
self.connected = False
self.log = open("zookeeper.log", 'a')
self.log.write("\n\n=============\nZOOKEEPER LOG\n=============\n")
self.log.write(datetime.now().__str__())
zookeeper.set_log_stream(self.log)
self.cv = threading.Condition()
self.cv2 = threading.Condition()
def watcher (handle, type, state, path):
self.cv.acquire()
if state == zookeeper.CONNECTED_STATE:
print "Connected!"
self.connected = True
else:
print "Disconnected from ZooKeeper: ",
print zookeeper.zerror(state)
self.cv.notify()
self.cv.release()
self.cv.acquire()
self.zh = zookeeper.init(setting.zookeeperloc, watcher, 10000)
self.cv.wait(10.0)
if not self.connected:
print "Cannot connect to ZooKeeper. ",
print "Check that server(s) are on " + setting.zookeeperloc
sys.exit()
self.cv.release()
self.root = setting.zookeeperroot
self.createRootNode(self.root)
self.zpath = '/'
def __init__(self, queuename):
self.connected = False
self.queuename = "/" + queuename
self.cv = threading.Condition()
zookeeper.set_log_stream(open("/dev/null"))
def watcher(handle, type, state, path):
print "Connected"
self.cv.acquire()
self.connected = True
self.cv.notify()
self.cv.release()
self.cv.acquire()
self.handle = zookeeper.init("localhost:2181", watcher, 10000, 0)
self.cv.wait(10.0)
if not self.connected:
print "Connection to ZooKeeper cluster timed out - is a server running on localhost:2181?"
sys.exit()
self.cv.release()
try:
zookeeper.create(self.handle, self.queuename, "queue top level",
[ZOO_OPEN_ACL_UNSAFE], 0)
except IOError, e:
if e.message == zookeeper.zerror(zookeeper.NODEEXISTS):
print "Queue already exists"
else:
raise e
def _getLogObserver(self):
if self._logfilename == "-":
log_file = sys.stdout
else:
log_file = open(self._logfilename, "a")
# Setup file logger
log_handler = logging.StreamHandler(log_file)
formatter = logging.Formatter(
"%(asctime)s: %(name)s@%(levelname)s: %(message)s")
log_handler.setFormatter(formatter)
# Also capture zookeeper logs (XXX not compatible with rotation)
zookeeper.set_log_stream(log_file)
zookeeper.set_debug_level(0)
# Configure logging.
root = logging.getLogger()
root.addHandler(log_handler)
root.setLevel(logging.getLevelName(self._loglevel))
# Twisted logging is painfully verbose on twisted.web, and
# there isn't a good way to distinguish different channels
# within twisted, so just utlize error level logging only for
# all of twisted.
twisted_log = logging.getLogger("twisted")
twisted_log.setLevel(logging.ERROR)
observer = PythonLoggingObserver()
return observer.emit
def __init__(self, queuename):
self.connected = False
self.queuename = "/" + queuename
self.cv = threading.Condition()
#zookeeper.set_log_stream(sys.stdout)
zookeeper.set_log_stream(open("/dev/null"))
def watcher(handle, type, state, path):
print "Connected"
self.cv.acquire()
self.connected = True
self.cv.notify()
self.cv.release()
self.cv.acquire()
self.handle = zookeeper.init("localhost:2181,localhost:2182,"\
"localhost:2183,localhost:2184,localhost:2185", watcher, 10000)
self.cv.wait(10.0)
if not self.connected:
print "Connection to ZooKeeper cluster timed out - is a server running on localhost:2181?"
sys.exit()
self.cv.release()
try:
retry_on_loss(zookeeper.create)\
(self.handle,self.queuename,"queue top level", [ZOO_OPEN_ACL_UNSAFE],0)
except zookeeper.NodeExistsException:
print "Queue already exists"
def __init__(self, barriername, number_of_workers):
self.cv = threading.Condition()
self.connected = False
self.barrier = "/" + barriername
self.workers = number_of_workers
zookeeper.set_log_stream(open('/dev/null'))
def watcher(handle, type, state, path):
self.cv.acquire()
self.connected = True
self.cv.notify()
self.cv.release()
self.cv.acquire()
self.handle = zookeeper.init("localhost:2181", watcher, 10000)
self.cv.wait(10.0)
if not self.connected:
print "Connection to ZooKeeper cluster timed out - is a server running on localhost:2181?"
sys.exit()
self.cv.release()
try:
zookeeper.create(self.handle, self.barrier, '\x00', [ZOO_OPEN_ACL_UNSAFE], 0)
except zookeeper.NodeExistsException:
pass
except Exception, ex:
print ex
raise ex
def __init__(self, **config):
if config.has_key('logger_name'):
self.logger = logging.getLogger(config['logger_name'])
else:
self.logger = Null()
# zookeeper client log
try:
zklogfile_path, zklog_level = config.get("ZookeeperLog", ("/dev/stderr", "WARN"))
_f = open(zklogfile_path,'a')
except Exception as err:
self.logger.error("In init ZKeeperClient: %s" % str(err))
else:
zookeeper.set_log_stream(_f)
zookeeper.set_debug_level(LOG_LEVELS.get(zklog_level, zookeeper.LOG_LEVEL_WARN))
try:
self.connection_timeout = config['timeout']
self.zkhosts = config['host']
except KeyError as err:
self.logger.exception("Cann't init ZKeeperClient: %s" % str(err))
self.connected = False
self.zkhandle = None
if self.connect():
self.logger.info('Connected to Zookeeper succesfully')
def main(argv):
usage = "usage: %prog [options]"
parser = OptionParser(usage=usage)
parser.add_option("-f", "--force", action="store_true", dest="force", help="Overwrite existing Sensei node info")
parser.add_option("-c", "--reset", action="store_true", dest="reset", help="Remove all registered nodes and then exit")
parser.add_option("-v", "--verbose", action="store_true", dest="verbose", help="Verbose mode")
(options, args) = parser.parse_args()
logging.basicConfig(format='[%(asctime)s] %(levelname)-8s"%(message)s"', datefmt='%Y-%m-%d %a %H:%M:%S')
if options.verbose:
logger.setLevel(logging.NOTSET)
initialize()
zookeeper.set_log_stream(open("/dev/null"))
cc = SinClusterClient(settings.SIN_SERVICE_NAME, settings.ZOOKEEPER_URL, settings.ZOOKEEPER_TIMEOUT)
cc.logger.setLevel(logging.INFO)
cc.logger.addHandler(logging.StreamHandler())
cc.add_listener(SinClusterListener())
if options.force or options.reset:
cc.reset()
Node.objects.all().delete()
logger.info("Removed all registered nodes from the system.")
logger.info("You may want to shut down all the agents.")
if options.reset:
return
# Reset online status. Some node(s) might have gone offline while Sin
# server was down, therefore the server did not get notified and still
# keeps the old "online" status for the node(s). If the node(s) are
# still online, we will send the start-store commands to them anyway.
# If a store is already running on a node, the start-store command
# will simply become a no-op.
Node.objects.filter(online=True).update(online=False)
for node in settings.SENSEI_NODES["nodes"]:
if not Node.objects.filter(id=node["node_id"]).exists():
Node.objects.create(id=node["node_id"], host=node["host"], agent_port=node["port"],
online=False, group=Group(pk=1))
cc.register_node(node["node_id"], node["host"], port=node["port"])
static_files = static.File(os.path.join(os.path.join(SIN_HOME, 'admin')))
WSGI = wsgi.WSGIResource(reactor, pool, WSGIHandler())
root = Root(WSGI)
root.putChild('static', static_files)
log.startLogging(sys.stdout)
site = server.Site(root)
reactor.listenTCP(settings.SIN_LISTEN, site)
pool.start()
def post_initialization():
cc.notify_all()
reactor.callInThread(post_initialization)
reactor.run(installSignalHandlers=False)
def __init__(self):
self.host = "localhost:%d" % self.SERVER_PORT
self.connected = False
self.handle = -1
try:
f = open('gk.log','w')
zookeeper.set_log_stream(f)
except IOError:
_logger.debug('Couldn\'t open logfile for writing')
def __init__(self,methodName='runTest'):
unittest.TestCase.__init__(self,methodName)
self.host = "localhost:22182"
self.connected = False
self.handle = -1
try:
f = open("build/test/logs/" + self.__class__.__name__ +".log","w")
zookeeper.set_log_stream(f)
except IOError:
print "Couldn't open build/test/logs/" + self.__class__.__name__ +".log for writing"
def __init__(self, **config):
logger_name = config.get("logger_name")
self.logger = logging.getLogger(logger_name) if logger_name else Null()
self.zkhandle = None
self.auth = None
self.cv = threading.Condition()
try:
auth_config = config.get("auth")
if auth_config is not None:
auth_scheme = auth_config["scheme"]
auth_data = auth_config["data"]
self.auth = (auth_scheme, auth_data)
zklogfile_path, zklog_level = config.get("ZookeeperLog", ("/dev/stderr", "WARN"))
self.connection_timeout = config["timeout"]
self.zkhosts = ",".join(config["host"])
except KeyError as err:
self.logger.exception("Missing configuration option: %s", err)
raise
except Exception as err:
self.logger.exception("Unknown configuration error: %s", err)
raise
try:
_f = open(zklogfile_path, "a")
except IOError as err:
self.logger.error("Unable to open logfile %s %s", zklogfile_path, err)
else:
zookeeper.set_log_stream(_f)
zookeeper.set_debug_level(LOG_LEVELS.get(zklog_level.upper(), zookeeper.LOG_LEVEL_WARN))
self.connect()
if zookeeper.state(self.zkhandle) == zookeeper.CONNECTED_STATE:
self.logger.info("Connected to Zookeeper successfully")
else:
raise zookeeper.ZooKeeperException("Unable to connect " "to Zookeeper")
def on_auth_callback(state, result):
with self.cv:
if result == zookeeper.AUTHFAILED:
self.logger.error(zookeeper.zerror(zookeeper.AUTHFAILED))
self.logger.info("on_auth: state %s, result %s", state, result)
self.cv.notify()
if self.auth:
self.logger.info("Auth using %s", self.auth[0])
with self.cv:
res = zookeeper.add_auth(self.zkhandle, self.auth[0], self.auth[1], on_auth_callback)
if res != zookeeper.OK:
self.logger.error("Invalid status %d", zookeeper.zerror(res))
raise Exception("Invalid status")
self.cv.wait(self.connection_timeout)
if zookeeper.state(self.zkhandle) == zookeeper.AUTH_FAILED_STATE:
raise zookeeper.ZooKeeperException("authentication failed")
def __init__(self, methodName='runTest'):
unittest.TestCase.__init__(self, methodName)
self.host = "localhost:22182"
self.connected = False
self.handle = -1
try:
f = open("build/test/logs/" + self.__class__.__name__ + ".log",
"w")
zookeeper.set_log_stream(f)
except IOError:
print "Couldn't open build/test/logs/" + self.__class__.__name__ + ".log for writing"
def __init__(self, host, timeout=None, recv_timeout=10000,
ident=(-1, ""), zklog_fd=None,
init_cbs=None):
"""
@param host: comma separated host:port pairs, each corresponding to
a zk server. e.g. '127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002'
(subsequent parameters are optional)
@param timeout: None by default means asynchronous session
establishment, or the time to wait until the session is established.
@param recv_timeout: ZK clients detects server failures in 2/3 of
recv_timeout, and then it retries the same IP at every recv_timeout
period if only one of ensemble is given. If more than two ensemble IP
are given, ZK clients will try next IP immediately.
@param ident: (clientid, passwd)
clientid the id of a previously established session that this
client will be reconnecting to. Clients can access the session id of
an established, valid, connection by calling zoo_client_id. If
the specified clientid has expired, or if the clientid is invalid for
any reason, the returned zhandle_t will be invalid -- the zhandle_t
state will indicate the reason for failure (typically
EXPIRED_SESSION_STATE).
@param zklog_fd: the file descriptor to redirect zookeeper logs.
By default, it redirects to /dev/null
@param init_cbs: initial callback objects of type StatePipeCondition
"""
LOG.debug("Create a new ZKSession: timeout=%(timeout)s "
"recv_timeout=%(recv_timeout)s,"
"ident=%(ident)s,"
"zklog_fd=%(zklog_fd)s, init_cbs=%(init_cbs)s",
locals())
self._host = host
self._recv_timeout = recv_timeout
self._ident = ident
if zklog_fd is None:
zklog_fd = open("/dev/null")
self._zklog_fd = zklog_fd
self._zhandle = None
self._conn_cbs = set([])
if init_cbs:
self._conn_cbs.update(init_cbs)
zookeeper.set_log_stream(self._zklog_fd)
self._conn_watchers = set([])
conn_spc = utils.StatePipeCondition()
self.add_connection_callback(conn_spc)
self._conn_spc = conn_spc
eventlet.spawn(self._session_thread)
self.connect(timeout=timeout)
def uninstall():
if not is_installed():
return
global _installed, _relay_thread
_installed = False
zookeeper.set_log_stream(sys.stderr)
# TODO: make sure the thread is actually stopped
_relay_thread.join()
def uninstall():
if not is_installed():
return
global _installed, _relay_thread
_installed = False
zookeeper.set_log_stream(sys.stderr)
# TODO: make sure the thread is actually stopped
_relay_thread.join()
def __init__(self,methodName='runTest'):
unittest.TestCase.__init__(self,methodName)
self.host = "localhost:22182"
self.connected = False
self.handle = -1
logdir = os.environ.get("ZKPY_LOG_DIR")
logfile = os.path.join(logdir, self.__class__.__name__ + ".log")
try:
f = open(logfile,"w")
zookeeper.set_log_stream(f)
except IOError:
print "Couldn't open " + logfile + " for writing"
def __init__(self, methodName='runTest'):
unittest.TestCase.__init__(self, methodName)
self.host = "localhost:22182"
self.connected = False
self.handle = -1
logdir = os.environ.get("ZKPY_LOG_DIR")
logfile = os.path.join(logdir, self.__class__.__name__ + ".log")
try:
f = open(logfile, "w")
zookeeper.set_log_stream(f)
except IOError:
print "Couldn't open " + logfile + " for writing"
def __init__(self, methodName='runTest'):
unittest.TestCase.__init__(self, methodName)
self.host = "localhost:%d" % self.SERVER_PORT
self.sslhost = "localhost:%d" % self.SERVER_SSL_PORT
self.sslcert = "./target/zkpython_tests/ssl/server.crt,./target/zkpython_tests/ssl/client.crt,./target/zkpython_tests/ssl/clientkey.pem,password"
self.connected = False
self.handle = -1
logdir = os.environ.get("ZKPY_LOG_DIR")
logfile = os.path.join(logdir, self.__class__.__name__ + ".log")
try:
f = open(logfile, "w")
zookeeper.set_log_stream(f)
except IOError:
print("Couldn't open " + logfile + " for writing")
def install():
global _installed, _relay_thread, _logging_pipe
if is_installed():
return
_logging_pipe = os.pipe()
zookeeper.set_log_stream(os.fdopen(_logging_pipe[1], 'w'))
_relay_thread = threading.Thread(target=_relay_log)
_relay_thread.setDaemon(True) # die along with the interpreter
_relay_thread.start()
_installed = True
def install():
global _installed, _relay_thread, _logging_pipe
if is_installed():
return
_logging_pipe = os.pipe()
zookeeper.set_log_stream(os.fdopen(_logging_pipe[1], 'w'))
_relay_thread = threading.Thread(target=_relay_log)
_relay_thread.setDaemon(True) # die along with the interpreter
_relay_thread.start()
_installed = True
def setup_logging(use_gevent=False):
global _logging_setup
if _logging_setup:
return
if use_gevent:
import gevent
fcntl.fcntl(_logging_pipe[0], fcntl.F_SETFL, os.O_NONBLOCK)
fcntl.fcntl(_logging_pipe[1], fcntl.F_SETFL, os.O_NONBLOCK)
gevent.spawn(_logging_greenlet)
else:
thread = get_realthread()
thread.start_new_thread(_logging_thread, ())
zookeeper.set_log_stream(os.fdopen(_logging_pipe[1], 'w'))
_logging_setup = True
def divert_zoolog(logfile='/dev/null'):
"""
Divert the ZooKeeper logging away from stderr.
This is particularly useful in interactive sessions where
the default logging behaviour is distinctly disruptive, but
also useful when logging that output to a file is desired.
Arguments:
- `logfile`: string - path to the file we'll log to
Return: None
Exceptions: None
"""
fh = open(logfile, 'w')
zookeeper.set_log_stream(fh)
return
def __init__(self,
host,
timeout=10,
recv_timeout=10000,
ident=(-1, ""),
zklog_fd=None):
"""
This method creates a new handle and a zookeeper session that corresponds
to that handle. Session establishment is asynchronous, meaning that the
session should not be considered established until (and unless) an
event of state CONNECTED_STATE is received.
PARAMETERS:
host: comma separated host:port pairs, each corresponding to a zk
server. e.g. '127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002'
(subsequent parameters are optional)
fn: the global watcher callback function. When notifications are
triggered this function will be invoked.
recv_timeout:
ident = (clientid, passwd)
clientid the id of a previously established session that this
client will be reconnecting to. Clients can access the session id of an
established, valid,
connection by calling zoo_client_id. If
the specified clientid has expired, or if the clientid is invalid for
any reason, the returned zhandle_t will be invalid -- the zhandle_t
state will indicate the reason for failure (typically
EXPIRED_SESSION_STATE).
zklog_fd is the file descriptor to redirect zookeeper logs.
By default, it redirects to /dev/null
"""
self._zhandle = None
pc = utils.PipeCondition()
def init_watcher(handle, event_type, stat, path):
#called when init is successful
pc.notify()
if zklog_fd is None:
zklog_fd = open("/dev/null")
zookeeper.set_log_stream(zklog_fd)
self._zhandle = zookeeper.init(host, init_watcher, recv_timeout, ident)
pc.wait(timeout)
def __init__(self, zk_ip, nodename):
# Frequency to check for change of status ( in seconds )
self.frequency = 10
self.connected = False
# Name of the master znode
self.nodename = nodename
# Used for checking if connection is established.
self.cv = threading.Condition()
self.isLeader = False
zookeeper.set_log_stream(open('/dev/null'))
Thread.__init__(self)
# Call this watch method when something is executed.
# Acquire condition variable so that the program waits until
# notified by watcher.
def watcher(handle, type, state, path):
log.info(" Connected to zookeeper")
self.cv.acquire()
self.connected = True
self.cv.notify()
self.cv.release()
self.cv.acquire()
self.handle = zookeeper.init(zk_ip, watcher, 10000)
# If watcher does not notify after 10 seconds, they proceed. It should probably error out
# in the next check.
self.cv.wait(10.0)
if not self.connected:
log.error("Cannot connect to the zookeeper instance. is a server running on "+zk_ip)
sys.exit()
self.cv.release()
# Create the master zNode here.
try:
# master node cannot be ephemeral because you cannot create childrens for ephemeral nodes.
zookeeper.create(self.handle,self.nodename,"Election Master Node", [ZOO_OPEN_ACL_UNSAFE],0)
except zookeeper.NodeExistsException:
log.debug("Election node already exists")
except zookeeper.BadArgumentsException:
log.error('Did you forget a / at the begging of the queue name?')
sys.exit(0)
def test_zookeeper_logging_enabled(self):
"""By default zookeeper logging is turned off, unless in verbose
mode.
"""
log_file = self.makeFile()
zookeeper.set_debug_level(10)
zookeeper.set_log_stream(open(log_file, "w"))
def reset_zk_log():
zookeeper.set_debug_level(0)
zookeeper.set_log_stream(sys.stdout)
self.addCleanup(reset_zk_log)
finished = self.setup_cli_reactor()
self.setup_exit(0)
self.mocker.replay()
main(["-v", "remove-unit", "mysql/3"])
yield finished
output = open(log_file).read()
self.assertTrue(output)
self.assertIn("ZOO_DEBUG", output)
def test_zookeeper_logging_enabled(self):
"""By default zookeeper logging is turned off, unless in verbose
mode.
"""
log_file = self.makeFile()
zookeeper.set_debug_level(10)
zookeeper.set_log_stream(open(log_file, "w"))
def reset_zk_log():
zookeeper.set_debug_level(0)
zookeeper.set_log_stream(sys.stdout)
self.addCleanup(reset_zk_log)
finished = self.setup_cli_reactor()
self.setup_exit(0)
self.mocker.replay()
main(["-v", "remove-unit", "mysql/3"])
yield finished
output = open(log_file).read()
self.assertTrue(output)
self.assertIn("ZOO_DEBUG", output)
def __init__(self, host, timeout=10, recv_timeout=10000, ident=(-1, ""), zklog_fd=None):
"""
This method creates a new handle and a zookeeper session that corresponds
to that handle. Session establishment is asynchronous, meaning that the
session should not be considered established until (and unless) an
event of state CONNECTED_STATE is received.
PARAMETERS:
host: comma separated host:port pairs, each corresponding to a zk
server. e.g. '127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002'
(subsequent parameters are optional)
fn: the global watcher callback function. When notifications are
triggered this function will be invoked.
recv_timeout:
ident = (clientid, passwd)
clientid the id of a previously established session that this
client will be reconnecting to. Clients can access the session id of an
established, valid,
connection by calling zoo_client_id. If
the specified clientid has expired, or if the clientid is invalid for
any reason, the returned zhandle_t will be invalid -- the zhandle_t
state will indicate the reason for failure (typically
EXPIRED_SESSION_STATE).
zklog_fd is the file descriptor to redirect zookeeper logs.
By default, it redirects to /dev/null
"""
self._zhandle = None
pc = utils.PipeCondition()
def init_watcher(handle, event_type, stat, path):
# called when init is successful
pc.notify()
if zklog_fd is None:
zklog_fd = open("/dev/null")
zookeeper.set_log_stream(zklog_fd)
self._zhandle = zookeeper.init(host, init_watcher, recv_timeout, ident)
pc.wait(timeout)
def test_zookeeper_logging_default(self):
"""By default zookeeper logging is turned off, unless in verbose
mode.
"""
log_file = self.makeFile()
def reset_zk_log():
zookeeper.set_debug_level(0)
zookeeper.set_log_stream(sys.stdout)
self.addCleanup(reset_zk_log)
finished = self.setup_cli_reactor()
self.setup_exit(0)
# Do this as late as possible to prevent zk background logging
# from causing problems.
zookeeper.set_debug_level(zookeeper.LOG_LEVEL_INFO)
zookeeper.set_log_stream(open(log_file, "w"))
self.mocker.replay()
main(["remove-unit", "mysql/3"])
yield finished
output = open(log_file).read()
self.assertEqual(output, "")
def test_zookeeper_logging_default(self):
"""By default zookeeper logging is turned off, unless in verbose
mode.
"""
log_file = self.makeFile()
def reset_zk_log():
zookeeper.set_debug_level(0)
zookeeper.set_log_stream(sys.stdout)
self.addCleanup(reset_zk_log)
finished = self.setup_cli_reactor()
self.setup_exit(0)
# Do this as late as possible to prevent zk background logging
# from causing problems.
zookeeper.set_debug_level(zookeeper.LOG_LEVEL_INFO)
zookeeper.set_log_stream(open(log_file, "w"))
self.mocker.replay()
main(["remove-unit", "mysql/3"])
yield finished
output = open(log_file).read()
self.assertEqual(output, "")
def __init__(self, hostname, port):
self.connected = False
zookeeper.set_log_stream(open("/dev/null"))
self.cv = threading.Condition()
def watcher(handle,type,state,path):
print "Connected"
self.cv.acquire()
self.connected = True
self.cv.notify()
self.cv.release()
self.cv.acquire()
connection = "%s:%d" % (hostname, port)
self.handle = zookeeper.init(connection, watcher, 10000)
self.cv.wait(10.0)
if not self.connected:
print "Connection to ZooKeeper cluster timed out - is a server running on localhost:%d?" % port
sys.exit()
self.cv.release()
if not self.connected:
print "Connection to ZooKeeper cluster timed out - is a server running on %s?" % connection
sys.exit()
def __init__(self, host, port, queuename):
self.connected = False
self.queuename = "/" + queuename
self.cv = threading.Condition()
zookeeper.set_log_stream(open("/dev/null"))
def watcher(handle, type, state, path):
# print "Connected"
self.cv.acquire()
self.connected = True
self.cv.notify()
self.cv.release()
self.cv.acquire()
host_s = host + ":" + str(port)
self.handle = zookeeper.init(host_s, watcher, 2000000)
self.cv.wait(10.0)
if not self.connected:
print(
"Connection to ZooKeeper cluster timed out "
"- is a server running on %s?" % host_s)
sys.exit()
self.cv.release()
def __init__(self, **config):
logger_name = config.get('logger_name')
self.logger = logging.getLogger(logger_name) if logger_name else Null()
self.zkhandle = None
self.auth = None
self.cv = threading.Condition()
try:
auth_config = config.get("auth")
if auth_config is not None:
auth_scheme = auth_config["scheme"]
auth_data = auth_config["data"]
self.auth = (auth_scheme, auth_data)
zklogfile_path, zklog_level = config.get("ZookeeperLog",
("/dev/stderr", "WARN"))
self.connection_timeout = config['timeout']
self.zkhosts = ','.join(config['host'])
except KeyError as err:
self.logger.exception("Missing configuration option: %s", err)
raise
except Exception as err:
self.logger.exception("Unknown configuration error: %s", err)
raise
try:
_f = open(zklogfile_path, 'a')
except IOError as err:
self.logger.error("Unable to open logfile %s %s",
zklogfile_path, err)
else:
zookeeper.set_log_stream(_f)
zookeeper.set_debug_level(LOG_LEVELS.get(zklog_level.upper(),
zookeeper.LOG_LEVEL_WARN))
self.connect()
if zookeeper.state(self.zkhandle) == zookeeper.CONNECTED_STATE:
self.logger.info('Connected to Zookeeper successfully')
else:
raise zookeeper.ZooKeeperException('Unable to connect '
'to Zookeeper')
def on_auth_callback(state, result):
with self.cv:
if result == zookeeper.AUTHFAILED:
self.logger.error(zookeeper.zerror(zookeeper.AUTHFAILED))
self.logger.info("on_auth: state %s, result %s",
state, result)
self.cv.notify()
if self.auth:
self.logger.info("Auth using %s", self.auth[0])
with self.cv:
res = zookeeper.add_auth(self.zkhandle, self.auth[0],
self.auth[1], on_auth_callback)
if res != zookeeper.OK:
self.logger.error("Invalid status %d",
zookeeper.zerror(res))
raise Exception("Invalid status")
self.cv.wait(self.connection_timeout)
if zookeeper.state(self.zkhandle) == zookeeper.AUTH_FAILED_STATE:
raise zookeeper.ZooKeeperException('authentication failed')
def main(argv):
usage = "usage: %prog [options]"
parser = OptionParser(usage=usage)
parser.add_option("-f",
"--force",
action="store_true",
dest="force",
help="Overwrite existing Sensei node info")
parser.add_option("-c",
"--reset",
action="store_true",
dest="reset",
help="Remove all registered nodes and then exit")
parser.add_option("-i",
"--init",
action="store_true",
dest="init",
help="Initialize nodes")
parser.add_option("-v",
"--verbose",
action="store_true",
dest="verbose",
help="Verbose mode")
(options, args) = parser.parse_args()
logging.basicConfig(format='[%(asctime)s] %(levelname)-8s"%(message)s"',
datefmt='%Y-%m-%d %a %H:%M:%S')
if options.verbose:
logger.setLevel(logging.NOTSET)
initialize()
zookeeper.set_log_stream(open("/dev/null"))
global cluster_client
cluster_client = SinClusterClient(settings.SIN_SERVICE_NAME,
settings.ZOOKEEPER_URL,
settings.ZOOKEEPER_TIMEOUT)
cluster_client.logger.setLevel(logging.INFO)
cluster_client.logger.addHandler(logging.StreamHandler())
cluster_client.add_listener(SinClusterListener())
if options.force or options.reset:
cluster_client.reset()
Node.objects.all().delete()
logger.info("Removed all registered nodes from the system.")
logger.info("You may want to shut down all the agents.")
if options.reset:
return
for node in settings.SIN_NODES["nodes"]:
n, created = Node.objects.get_or_create(id=node["node_id"],
defaults={
"id": node["node_id"],
"host": node["host"],
"agent_port": node["port"],
"online": False,
"group": Group(pk=1),
})
if not created:
if n.host != node["host"] or n.agent_port != node["port"]:
n.host = node["host"]
n.agent_port = node["port"]
n.save()
cluster_client.remove_node(node["node_id"])
cluster_client.register_node(node["node_id"],
node["host"],
port=node["port"])
if options.init:
return
# Reset online status. Some node(s) might have gone offline while Sin
# server was down, therefore the server did not get notified and still
# keeps the old "online" status for the node(s). If the node(s) are
# still online, we will send the start-store commands to them anyway.
# If a store is already running on a node, the start-store command
# will simply become a no-op.
Node.objects.filter(online=True).update(online=False)
static_files = static.File(os.path.join(os.path.join(SIN_HOME, 'admin')))
WSGI = wsgi.WSGIResource(reactor, pool, WSGIHandler())
root = Root(WSGI)
root.putChild('static', static_files)
log.startLogging(sys.stdout)
site = server.Site(root)
reactor.listenTCP(settings.SIN_LISTEN, site)
pool.start()
def post_initialization():
cluster_client.notify_all()
reactor.callInThread(post_initialization)
reactor.callInThread(monitoring)
reactor.run(installSignalHandlers=False)
from django.conf import settings
from datetime import datetime
import threading
import zookeeper
PERM_READ = 1
PERM_WRITE = 2
PERM_CREATE = 4
PERM_DELETE = 8
PERM_ADMIN = 16
PERM_ALL = PERM_READ | PERM_WRITE | PERM_CREATE | PERM_DELETE | PERM_ADMIN
zookeeper.set_log_stream(open(settings.ZOOKEEPER_CLI_LOG_PATH, "w"))
TIMEOUT = 10.0
class ZKClient(object):
def __init__(self, servers, timeout):
self.connected = False
self.conn_cv = threading.Condition( )
self.handle = -1
self.conn_cv.acquire()
print("Connecting to %s" % (servers))
self.handle = zookeeper.init(servers, self.connection_watcher, 30000)
self.conn_cv.wait(timeout)
self.conn_cv.release()
if not self.connected:
raise Exception("Unable to connect to %s" % (servers))
self.logger.info("Shutting down zookeeper session: %d" % self.handle)
zookeeper.close(self.handle)
if __name__ == '__main__':
usage = "usage: %prog [options]"
parser = OptionParser(usage=usage)
parser.add_option("", "--connect-string", dest="servers",
default="localhost:2181", help="comma separated list of host:port (default localhost:2181)")
parser.add_option("", "--timeout", dest="timeout", type="int",
default=5000, help="session timeout in milliseconds (default 5000)")
parser.add_option("-t", "--test", dest="test_node",type="int",
default=-1, help="testing node number (default -1)")
(options, args) = parser.parse_args()
zookeeper.set_log_stream(open("/dev/null"))
cc = SinClusterClient("sin", options.servers, options.timeout)
cc.add_listener(SinClusterListener())
cc.logger.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(asctime)s %(filename)s:%(lineno)d - %(message)s")
stream_handler = logging.StreamHandler()
stream_handler.setFormatter(formatter)
cc.logger.addHandler(stream_handler)
if (options.test_node >= 0):
cc.register_node(options.test_node)
cc.mark_node_available(options.test_node)
time.sleep(10)
sys.exit()
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os, sys
import zookeeper
import threading
import functools
import time
zookeeper.set_log_stream(sys.stdout)
ZOO_OPEN_ACL_UNSAFE = {"perms":0x1f, "scheme":"world", "id" :"anyone"}
def retry_on(*excepts):
""" Retry function execution if some known exception types are raised """
def _decorator(fn):
@functools.wraps(fn)
def wrapper(*args, **kwargs):
while True:
try:
return fn(*args, **kwargs)
except Exception, e:
if not any([isinstance(e, _) for _ in excepts]):
raise
# else: retry forever
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
# com.cat.core.http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import zookeeper, time, threading
f = open("out.log","w")
zookeeper.set_log_stream(f)
connected = False
conn_cv = threading.Condition( )
def my_connection_watcher(handle,type,state,path):
global connected, conn_cv
print("Connected, handle is ", handle)
conn_cv.acquire()
connected = True
conn_cv.notifyAll()
conn_cv.release()
conn_cv.acquire()
print("Connecting to localhost:2181 -- ")
handle = zookeeper.init("localhost:2181", my_connection_watcher, 10000, 0)
from django.conf import settings
from datetime import datetime
import threading
import zookeeper
from zookeeper_dashboard.zkadmin.models import Server
PERM_READ = 1
PERM_WRITE = 2
PERM_CREATE = 4
PERM_DELETE = 8
PERM_ADMIN = 16
PERM_ALL = PERM_READ | PERM_WRITE | PERM_CREATE | PERM_DELETE | PERM_ADMIN
zookeeper.set_log_stream(open("cli_log.txt","w"))
TIMEOUT = 10.0
class ZKClient(object):
def __init__(self, servers, timeout):
self.connected = False
self.conn_cv = threading.Condition( )
self.handle = -1
self.conn_cv.acquire()
print("Connecting to %s" % (servers))
self.handle = zookeeper.init(servers, self.connection_watcher, 30000)
self.conn_cv.wait(timeout)
self.conn_cv.release()
def disable_zookeeper_log():
import zookeeper
zookeeper.set_log_stream(open('/dev/null'))
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os, sys
import zookeeper
import threading
import functools
import time
zookeeper.set_log_stream(sys.stdout)
ZOO_OPEN_ACL_UNSAFE = {"perms": 0x1f, "scheme": "world", "id": "anyone"}
def retry_on(*excepts):
""" Retry function execution if some known exception types are raised """
def _decorator(fn):
@functools.wraps(fn)
def wrapper(*args, **kwargs):
while True:
try:
return fn(*args, **kwargs)
except Exception, e:
if not any([isinstance(e, _) for _ in excepts]):
raise
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import with_statement
from functools import partial
import logging
import threading
import zookeeper
ZK_ACL = {"perms": 0x1f,
"scheme": "world",
"id": "anyone"}
zookeeper.set_log_stream(open('/dev/null', 'w'))
DEFAULT_ERRNO = -9999
# JFYI
LOG_LEVELS = {"DEBUG": zookeeper.LOG_LEVEL_DEBUG,
"INFO": zookeeper.LOG_LEVEL_INFO,
"WARN": zookeeper.LOG_LEVEL_WARN,
"ERROR": zookeeper.LOG_LEVEL_ERROR}
class Null(object):
"""This class does nothing as logger"""
def __init__(self, *args, **kwargs):
pass
def reset_zk_log():
zookeeper.set_debug_level(0)
zookeeper.set_log_stream(sys.stdout)
def reset_zk_log():
zookeeper.set_debug_level(0)
zookeeper.set_log_stream(sys.stdout)
from kazoo.recipe.party import Party
from kazoo.recipe.party import ShallowParty
from kazoo.recipe.election import Election
from kazoo.retry import KazooRetry
from kazoo.handlers.util import thread
log = logging.getLogger(__name__)
ZK_OPEN_ACL_UNSAFE = {"perms": zookeeper.PERM_ALL, "scheme": "world",
"id": "anyone"}
## Zookeeper Logging Setup
# Setup Zookeeper logging thread
_logging_pipe = os.pipe()
zookeeper.set_log_stream(os.fdopen(_logging_pipe[1], 'w'))
@thread
def _loggingthread():
"""Zookeeper logging redirect
Zookeeper by default logs directly out. This thread handles reading
off the pipe that the above `set_log_stream` call designates so
that the Zookeeper logging output can be turned into Python logging
statements under the `Zookeeper` name.
"""
r, w = _logging_pipe
log = logging.getLogger('ZooKeeper').log
f = os.fdopen(r)
from datetime import datetime
import threading
import zookeeper
PERM_READ = 1
PERM_WRITE = 2
PERM_CREATE = 4
PERM_DELETE = 8
PERM_ADMIN = 16
PERM_ALL = PERM_READ | PERM_WRITE | PERM_CREATE | PERM_DELETE | PERM_ADMIN
zookeeper.set_log_stream(open("cli_log.txt","w"))
TIMEOUT = 10.0
class ZKClient(object):
def __init__(self, servers, timeout):
self.connected = False
self.conn_cv = threading.Condition( )
self.handle = -1
self.conn_cv.acquire()
self.handle = zookeeper.init(servers, self.connection_watcher, 30000)
self.conn_cv.wait(timeout)
self.conn_cv.release()
if not self.connected:
raise Exception("Unable to connect to %s" % (servers))
def connection_watcher(self, h, type, state, path):
self.handle = h
def disable_zookeeper_log():
import zookeeper
zookeeper.set_log_stream(open('/dev/null'))