def testhandlereuse(self):
        """
        Test a) multiple concurrent connections b) reuse of closed handles
        """
        cv = threading.Condition()
        self.connected = False
        def connection_watcher(handle, type, state, path):
            cv.acquire()
            self.connected = True
            self.assertEqual(zookeeper.CONNECTED_STATE, state)
            self.handle = handle
            cv.notify()
            cv.release()

        cv.acquire()
        handles = [ zookeeper.init(self.host) for i in xrange(10) ]
        ret = zookeeper.init(self.host, connection_watcher)
        cv.wait(15.0)
        cv.release()
        self.assertEqual(self.connected, True, "Connection timed out to " + self.host)
        self.assertEqual(True, all( [ zookeeper.state(handle) == zookeeper.CONNECTED_STATE for handle in handles ] ),
                         "Not all connections succeeded")
        oldhandle = handles[3]
        zookeeper.close(oldhandle)
        newhandle = zookeeper.init(self.host)

        # This assertion tests *internal* behaviour; i.e. that the module
        # correctly reuses closed handles. This is therefore implementation
        # dependent.
        self.assertEqual(newhandle, oldhandle, "Didn't get reused handle")
Exemple #2
0
  def _connect(self):
    """Creates a connection to a zookeeper instance."""
    s = []
    for host, port in self._servers:
      try:
        _, _, ips = socket.gethostbyname_ex(host)
        for ip in ips:
          s.append('%s:%s' % (ip, port))
      except socket.gaierror:
        logging.error('Hostname not known: %s', host)
      except socket.herror:
        logging.error('Unable to resolve %s', host)

    if not s:
      logging.error('No IPs found to connect to.. trying again in 1 second.')
      t = threading.Timer(1.0, self._connect)
      t.daemon = True
      t.start()
      return

    if self._clientid is not None:
      # Existing connections get registered with the same clientid that was
      # used before.
      self._zookeeper = zookeeper.init(
          ','.join(s), self._global_watch, None, clientid)
    else:
      self._zookeeper = zookeeper.init(','.join(s), self._global_watch)
    def testmanyhandles(self):
        """
        Test the ability of the module to support many handles.
        """
        # We'd like to do more, but currently the C client doesn't
        # work with > 83 handles (fails to create a pipe) on MacOS 10.5.8
        handles = [ zookeeper.init(self.host) for i in xrange(63) ]

        cv = threading.Condition()
        self.connected = False
        def connection_watcher(handle, type, state, path):
            cv.acquire()
            self.connected = True
            self.assertEqual(zookeeper.CONNECTED_STATE, state)
            self.handle = handle
            cv.notify()
            cv.release()

        cv.acquire()
        ret = zookeeper.init(self.host, connection_watcher)
        cv.wait(15.0)
        cv.release()
        self.assertEqual(self.connected, True, "Connection timed out to " + self.host)

        for i,h in enumerate(handles):
            path = "/zkpython-test-handles-%s" % str(i)
            self.assertEqual(path, zookeeper.create(h, path, "", [ZOO_OPEN_ACL_UNSAFE], zookeeper.EPHEMERAL))

        self.assertEqual(True, all( zookeeper.close(h) == zookeeper.OK for h in handles ))
Exemple #4
0
    def __init__(self, connection_string="127.0.0.1:2181", session_timeout=None,
                 wait=False):
        self.watches = WatchManager()
        self.ephemeral = {}
        self.handle = None

        connected = self.connected = threading.Event()
        def watch_session(handle, event_type, state, path):
            assert event_type == zookeeper.SESSION_EVENT
            assert not path
            if state == zookeeper.CONNECTED_STATE:
                if self.handle is None:
                    self.handle = handle
                    for watch in self.watches.clear():
                        self._watch(watch)
                    for path, data in list(self.ephemeral.items()):
                        zookeeper.create(self.handle, path, data['data'],
                                         data['acl'], data['flags'])
                else:
                    assert handle == self.handle
                connected.set()
                logger.info('connected %s', handle)
            elif state == zookeeper.CONNECTING_STATE:
                connected.clear()
            elif state == zookeeper.EXPIRED_SESSION_STATE:
                connected.clear()
                if self.handle is not None:
                    zookeeper.close(self.handle)
                self.handle = None
                init()
            else:
                logger.critical('unexpected session event %s %s', handle, state)

        if session_timeout:
            init = (lambda : zookeeper.init(connection_string, watch_session,
                                            session_timeout)
                    )
        else:
            init = lambda : zookeeper.init(connection_string, watch_session)

        handle = init()
        connected.wait(self.initial_connection_wait)
        if not connected.is_set():
            if wait:
                while not connected.is_set():
                    logger.critical("Can't connect to ZooKeeper at %r",
                                    connection_string)
                    connected.wait(1)
            else:
                zookeeper.close(handle)
                raise FailedConnect(connection_string)
Exemple #5
0
    def connect(self, servers=None, timeout=10, client_id=None):
        """
        Establish a connection to the given zookeeper server(s).

        @param servers: A string specifying the servers and their ports to
                        connect to. Multiple servers can be specified in
                        comma separated fashion.
        @param timeout: How many seconds to wait on a connection to the
                        zookeeper servers.

        @param session_id:
        @returns A deferred that's fired when the connection is established.
        """
        d = defer.Deferred()

        if self.connected:
            return defer.fail(
                zookeeper.ZooKeeperException("Already Connected"))

        # Use a scheduled function to ensure a timeout.
        def _check_timeout():
            # Close the handle
            try:
                if self.handle is not None:
                    zookeeper.close(self.handle)
            except zookeeper.ZooKeeperException:
                pass
            d.errback(
                ConnectionTimeoutException("could not connect before timeout"))

        scheduled_timeout = reactor.callLater(timeout, _check_timeout)

        # Assemble an on connect callback with closure variable access.
        callback = partial(self._cb_connected, scheduled_timeout, d)
        callback = self._zk_thread_callback(callback)

        if self._session_timeout is None:
            self._session_timeout = DEFAULT_SESSION_TIMEOUT

        if servers is not None:
            self._servers = servers

        # Use client id if specified.
        if client_id:
            self.handle = zookeeper.init(
                self._servers, callback, self._session_timeout, client_id)
        else:
            self.handle = zookeeper.init(
                self._servers, callback, self._session_timeout)
        return d
Exemple #6
0
    def connect_async(self):
        """Asynchronously initiate connection to ZK

        @return: AsyncResult object set on connection success
        @rtype AsyncResult
        """

        cb = self._wrap_session_callback(self._session_callback)
        if self._provided_client_id:
            self._handle = zookeeper.init(self._hosts, cb, self._timeout,
                self._provided_client_id)
        else:
            self._handle = zookeeper.init(self._hosts, cb, self._timeout)

        return self._connected_async_result
Exemple #7
0
    def connect(self, timeout = None):
        '''Connects to the zookeeper server'''
        # if no timeout provided, thake the configured one
        if timeout is None:
            timeout = self._timeout

        # check, if we're already connected
        if self._handle is not None and not self.is_connected():
            raise RuntimeError('Already connected')
            return

        condition = threading.Condition()
        def connection_watch(handle, type, state, path):
            condition.acquire()
            condition.notify()
            condition.release()

        # try to connect
        condition.acquire()
        self._handle = zookeeper.init(
			','.join(self._servers),
            connection_watch,
            self._timeout * 1000)
        condition.wait(self._timeout)
        condition.release()

        if zookeeper.state(self._handle) != zookeeper.CONNECTED_STATE:
            zookeeper.close(self._handle)
            raise RuntimeError(
                'unable to connect to %s ' % (' or '.join(self._servers)))
        zookeeper.set_watcher(self._handle, self.__global_watch)
Exemple #8
0
def main():
	from optparse import OptionParser as OptionParser

	parser = OptionParser()
	parser.add_option('-v', '--verbose', action="store_true", dest="flag_verbose",
		help='Triggers debugging traces')

	(options, args) = parser.parse_args(sys.argv)

	# Logging configuration
	if options.flag_verbose:
		logging.basicConfig(
			format='%(asctime)s %(message)s',
			datefmt='%m/%d/%Y %I:%M:%S',
			level=logging.DEBUG)
	else:
		logging.basicConfig(
			format='%(asctime)s %(message)s',
			datefmt='%m/%d/%Y %I:%M:%S',
			level=logging.INFO)

	if len(args) < 2:
		raise ValueError("not enough CLI arguments")

	ns = args[1]
	cnxstr = load_config().get(ns, 'zookeeper')

	zookeeper.set_debug_level(zookeeper.LOG_LEVEL_INFO)
	zh = zookeeper.init(cnxstr)
	zookeeper.create(zh, PREFIX, '', acl_openbar, 0)
	create_tree(zh, boot_tree())
	init_namespace(zh, ns)
	zookeeper.close(zh)
  def __init__(self, host=DEFAULT_HOST, start_gc=True):
    """ Creates a new ZKTransaction, which will communicate with Zookeeper
    on the given host.

    Args:
      host: A str that indicates which machine runs the Zookeeper service.
      start_gc: A bool that indicates if we should start the garbage collector
        for timed out transactions.
    """
    logging.basicConfig(format='%(asctime)s %(levelname)s %(filename)s:' \
      '%(lineno)s %(message)s ', level=logging.INFO)
    logging.debug("Started logging")

    # Connection instance variables.
    self.connect_cv = threading.Condition()
    self.connected = False
    self.host = host
    self.handle = zookeeper.init(host, self.receive_and_notify)

    # for blacklist cache
    self.blacklist_cv = threading.Condition()
    self.blacklist_cache = {}

    # for gc
    self.gc_running = False
    self.gc_cv = threading.Condition()
    if start_gc:
      self.start_gc()
Exemple #10
0
  def __init__(self, service_name, connect_string, timeout=DEFAULT_TIMEOUT, default_port=6664):
    self.SERVICE_NODE = "/" + service_name
    self.AVAILABILITY_NODE = self.SERVICE_NODE + "/available"
    self.MEMBERSHIP_NODE = self.SERVICE_NODE + "/members"
    self.connected = False
    self.timeout = timeout
    self.default_port = default_port
    self.conn_cv = threading.Condition()
    self.conn_cv.acquire()
    self.handle = zookeeper.init(connect_string, self.connection_watcher, timeout)
    self.conn_cv.wait(timeout / 1000)
    self.conn_cv.release()
    self.watcher_lock = threading.Lock()
    self.logger = logging.getLogger("sincc")

    if not self.connected:
      raise SinClusterClientError("Unable to connect to %s" % connect_string)

    for path in [self.SERVICE_NODE, self.AVAILABILITY_NODE, self.MEMBERSHIP_NODE]:
      if not zookeeper.exists(self.handle, path):
        zookeeper.create(self.handle, path, "", [ZOO_OPEN_ACL_UNSAFE], 0)
    self.listeners = []
    # Start to watch both /members and /available
    zookeeper.get_children(self.handle, self.MEMBERSHIP_NODE, self.watcher)
    available = zookeeper.get_children(self.handle, self.AVAILABILITY_NODE, self.watcher)
    self.available_nodes = {}
    for node_id in available:
      self.available_nodes[int(node_id)] = Node(int(node_id),
                                                zookeeper.get(self.handle, self.AVAILABILITY_NODE + "/" + node_id)[0])
Exemple #11
0
def connect(host=None):
    global conn_cv, connected, handle
    global reconnect_host

    if host is None:
        host = "localhost:2181"

    reconnect_host = host

    conn_cv.acquire()
    handle = zookeeper.init(host, my_connection_watcher, 10000)
    while not connected:
        conn_cv.wait()
    conn_cv.release()

    connected = False
    while not connected:
        try:
            zookeeper.create(handle, ROOT, "1", [ZOO_OPEN_ACL_UNSAFE], 0)
            connected = True
        except zookeeper.NodeExistsException as e:
            # No worries
            connected = True
        except zookeeper.ConnectionLossException:
            continue
        except:
            raise
 def __init__(self,queuename):
     self.connected = False
     self.queuename = "/" + queuename
     self.cv = threading.Condition()
     zookeeper.set_log_stream(open("/dev/null"))        
     def watcher(handle,type,state,path):
         print "Connected"
         self.cv.acquire()
         self.connected = True
         self.cv.notify()
         self.cv.release()
         
     self.cv.acquire()            
     self.handle = zookeeper.init("localhost:2181", watcher, 10000, 0)
     self.cv.wait(10.0)
     if not self.connected:
         print "Connection to ZooKeeper cluster timed out - is a server running on localhost:2181?"
         sys.exit()
     self.cv.release()
     try:
         zookeeper.create(self.handle,self.queuename,"queue top level", [ZOO_OPEN_ACL_UNSAFE],0)
     except IOError, e:
         if e.message == zookeeper.zerror(zookeeper.NODEEXISTS):
             print "Queue already exists"    
         else:
             raise e
Exemple #13
0
    def post(self):
	request_dict = self.request.arguments
	node_key = (request_dict['node_tree'])[0]

        cluster_name  = (request_dict['cluster_name'])[0]
        zk=zookeeper.init(self.zk_connect(cluster_name))

        def get_node(node_key):
    	    """获取子节点生成快照存到MySQL"""
            if node_key == "/":
                for node in zookeeper.get_children(zk,node_key):
                     key =  "/" + node
                     if (zookeeper.get(zk,key)[1])['numChildren'] > 0:
                          get_node(key)
                     else:
            	          value = (zookeeper.get(zk,key))[0]
    		          create_time = time.time()
    		          table = ZdSnapshot(cluster_name= cluster_name ,path=key , data=value ,create_time=self.GetNowTime())
    		          table.save()
            else:
                for node in zookeeper.get_children(zk,node_key):
                     key =  node_key + "/" + node
                     if (zookeeper.get(zk,key)[1])['numChildren'] > 0:
                          get_node(key)
                     else:
            	          value = (zookeeper.get(zk,key))[0]
    		          create_time = time.time()
    		          table = ZdSnapshot(cluster_name= cluster_name,path=key , data=value ,create_time=self.GetNowTime())
    		          table.save()
	get_node(node_key)
	self.write("生成快照成功!!!!!")
        zookeeper.close(zk)
Exemple #14
0
    def post(self):
	request_dict = self.request.arguments
	node_key = (request_dict['node_key'])[0]
        cluster_name  = (request_dict['cluster_name'])[0]
        zk=zookeeper.init(self.zk_connect(cluster_name))


        data = []
        def get_node_tree(node_key):
            if node_key == "/":
                for node in zookeeper.get_children(zk,node_key):
                     key =  "/" + node
                     if (zookeeper.get(zk,key)[1])['numChildren'] > 0:
                          get_node_tree(key)
                          print key
                     else:
                          print key
            else:
                for node in zookeeper.get_children(zk,node_key):
                     key =  node_key + "/" + node
                     if (zookeeper.get(zk,key)[1])['numChildren'] > 0:
                          get_node_tree(key)
                          data.append(key)
                     else:
                          data.append(key)
        
            return data

	get_node_tree(node_key)
	data.append(node_key)

	for items in data:
	    zookeeper.delete(zk,items)
        zookeeper.close(zk)                                                                                                                          
        self.write("删除成功")
Exemple #15
0
    def __init__(self, setting):
        #Connect to ZK servers
        print "Connecting to ZooKeeper ... "
        self.connected = False
        self.log = open("zookeeper.log", 'a')
        self.log.write("\n\n=============\nZOOKEEPER LOG\n=============\n")
        self.log.write(datetime.now().__str__())
        zookeeper.set_log_stream(self.log)
        self.cv = threading.Condition()
        self.cv2 = threading.Condition()
        def watcher (handle, type, state, path):
            self.cv.acquire()
            if state == zookeeper.CONNECTED_STATE:
                print "Connected!"
                self.connected = True
            else:
                print "Disconnected from ZooKeeper: ",
                print zookeeper.zerror(state)
            self.cv.notify()
            self.cv.release()
        
        self.cv.acquire()
        self.zh = zookeeper.init(setting.zookeeperloc, watcher, 10000)
        self.cv.wait(10.0)
        if not self.connected:
            print "Cannot connect to ZooKeeper. ",
            print "Check that server(s) are on " + setting.zookeeperloc
            sys.exit()
        self.cv.release()

        self.root = setting.zookeeperroot
        self.createRootNode(self.root)
        self.zpath = '/'
Exemple #16
0
    def test_logging(self):
        handler = zope.testing.loggingsupport.InstalledHandler(
            'ZooKeeper')
        try:
            handle = zookeeper.init('zookeeper.example.com:2181')
            zookeeper.close(handle)
        except:
            pass

        wait(lambda : [r for r in handler.records
                       if 'environment' in r.getMessage()]
             )
        handler.clear()

        # Test that the filter for the "Exceeded deadline by" noise works.
        # cheat and bypass zk by writing to the pipe directly.
        os.write(zc.zk._logging_pipe[1],
                 '2012-01-06 16:45:44,572:43673(0x1004f6000):ZOO_WARN@'
                 'zookeeper_interest@1461: Exceeded deadline by 27747ms\n')
        wait(lambda : [r for r in handler.records
                       if ('Exceeded deadline by' in r.getMessage()
                           and r.levelno == logging.DEBUG)
                       ]
             )

        self.assert_(not [r for r in handler.records
                          if ('Exceeded deadline by' in r.getMessage()
                              and r.levelno == logging.WARNING)
                          ])

        handler.uninstall()
Exemple #17
0
  def __init__(self,queuename, port, is_producer=False):
    self.connected = False
    self.queuename = "/" + queuename
    self.cv = threading.Condition()
    zookeeper.set_log_stream(open("/dev/null"))
    def watcher(handle,type,state,path):
      print "Connected"
      self.cv.acquire()
      self.connected = True
      self.cv.notify()
      self.cv.release()

    self.cv.acquire()
    self.handle = zookeeper.init("localhost:%d" % port, watcher, 10000)
    self.cv.wait(10.0)
    if not self.connected:
      print "Connection to ZooKeeper cluster timed out - is a server running on localhost:%d?" % port
      sys.exit()
    self.cv.release()
    if is_producer:
      while True:
        try:
          zookeeper.create(self.handle,self.queuename,"queue top level", [ZOO_OPEN_ACL_UNSAFE],0)
          print "Fila criada, OK"
          return
        except zookeeper.NodeExistsException:
          print "Tratorando filas existentes"
          while True:
            children = sorted(zookeeper.get_children(self.handle, self.queuename,None))
            if len(children) == 0:
              (data,stat) = zookeeper.get(self.handle, self.queuename, None)
              zookeeper.delete(self.handle, self.queuename, stat["version"])
              break
            for child in children:
              data = self.get_and_delete(self.queuename + "/" + child)
 def __init__(self, config_server_="http://xxx.xxx.com/"):
     self.config_server = config_server_
     self.cache_config = []  
     self.zk_path = "/config"
     self.zklist = self.getZkServer()
     zookeeper.set_debug_level(zookeeper.LOG_LEVEL_ERROR)
     self.zookeeper = zookeeper.init(self.zklist)
    def __init__(self, servers,port=2181, timeout=TIMEOUT):
        """
        create connection to zookeeper
        @param server e.g. "localhost"
        @param port (std 2181)
        """
        self.connected = False
        self.conn_cv = threading.Condition( )
        self.handle = -1
        #@todo make cross platform
        zookeeper.set_log_stream(open("/dev/null"))   

        self.conn_cv.acquire()

        q.logger.log("Connecting to %s" % (servers))
        start = time.time()
        
        self.handle = zookeeper.init(servers, self._connection_watcher, 30000)
        self.conn_cv.wait(timeout)
        self.conn_cv.release()

        if not self.connected:
            raise RuntimeError("Unable to connect to %s" % (servers))

        q.logger.log("Connected in %d ms, handle is %d" % (int((time.time() - start) * 1000), self.handle))
Exemple #20
0
    def __init__(self, barriername, number_of_workers):
	self.cv = threading.Condition()
	self.connected = False
	self.barrier = "/" + barriername
	self.workers = number_of_workers
	zookeeper.set_log_stream(open('/dev/null'))
	def watcher(handle, type, state, path):
	    self.cv.acquire()
	    self.connected = True
	    self.cv.notify()
	    self.cv.release()

	self.cv.acquire()
	self.handle = zookeeper.init("localhost:2181", watcher, 10000)
	self.cv.wait(10.0)
	if not self.connected:
            print "Connection to ZooKeeper cluster timed out - is a server running on localhost:2181?"
            sys.exit()
	self.cv.release()
	try:
	    zookeeper.create(self.handle, self.barrier, '\x00', [ZOO_OPEN_ACL_UNSAFE], 0)
	except zookeeper.NodeExistsException:
	    pass
	except Exception, ex:
	    print ex
	    raise ex
Exemple #21
0
    def __init__(self, servers, acl, timeout=DEFAULT_TIMEOUT):
        """
        zookeeper 操作类的封装
        """
        self.epoch = 0
        self.acl = acl
        self.timeout = timeout
        self.connected = False
        #线程同步对象
        self.conn_cv = threading.Condition()
        self.handle = -1
        #线程尝试获取锁,如果拿到就执行下面的代码,否则等待其他线程通知
        self.conn_cv.acquire()

        self.handle = zookeeper.init(servers, self.conn_watcher, timeout)
        #wait方法会释放锁,然后阻塞,等待其他线程执行notify或者notifyAll方法,如果指定的时间没有得到通知,线程重新获取锁,如果获取到锁,继续执行下面的代码
        self.conn_cv.wait(timeout / 10000)
        #释放锁
        self.conn_cv.release()
        #检查连接状态
        if not self.connected:
            raise ZKClientError("Unable to connect to %s" % (servers))

        if not self.add_auth(scheme=acl["scheme"], id=acl["id"]):
            raise ZKClientError("add_auth to zookeeper fail")
Exemple #22
0
  def __init__(self,queuename):
    self.connected = False
    self.queuename = "/" + queuename
    self.cv = threading.Condition()
    #zookeeper.set_log_stream(sys.stdout)
    zookeeper.set_log_stream(open("/dev/null"))
    def watcher(handle,type,state,path):
      print "Connected"
      self.cv.acquire()
      self.connected = True
      self.cv.notify()
      self.cv.release()

    self.cv.acquire()
    self.handle = zookeeper.init("localhost:2181,localhost:2182,"\
        "localhost:2183,localhost:2184,localhost:2185", watcher, 10000)
    self.cv.wait(10.0)
    if not self.connected:
      print "Connection to ZooKeeper cluster timed out - is a server running on localhost:2181?"
      sys.exit()
    self.cv.release()
    try:
      retry_on_loss(zookeeper.create)\
        (self.handle,self.queuename,"queue top level", [ZOO_OPEN_ACL_UNSAFE],0)
    except zookeeper.NodeExistsException:
      print "Queue already exists"
Exemple #23
0
    def __init__(self, zkhosts, root=NODE_HQ_ROOT, alivenode='alive'):
        """zkhosts: a string or a list. list will be ','.join-ed into a string.
        root: root node path (any parents must exist, if any)
        """
        
        if not isinstance(zkhosts, basestring):
            zkhosts = ','.join(zkhosts)
        self.zkhosts = zkhosts
        self.ROOT = root
        self.alivenode = alivenode
        self.nodename = os.uname()[1]

        self.NODE_SERVERS = self.ROOT + '/servers'
        self.NODE_ME = self.NODE_SERVERS+'/'+self.nodename

        self.zh = zk.init(self.zkhosts, self.__watcher)

        self.jobs = {}

        self.initialize_node_structure()

        if not zk.exists(self.zh, self.NODE_ME):
            zk.create(self.zh, self.NODE_ME, '', PERM_WORLD)

        # setup notification
        zk.get_children(self.zh, self.NODE_SERVERS, self.__servers_watcher)
        #zk.get_children(self.zh, self.NODE_ME, self.__watcher)

        self.NODE_JOBS = self.NODE_ME+'/jobs'
        zk.acreate(self.zh, self.NODE_JOBS, '', PERM_WORLD)
Exemple #24
0
    def __init__(self, server, root_node, host='localhost', port=2181):
        self.root = root_node
        self.host = host
        self.port = port
        self.counter = random.randint(0,2**30)
        self.server = server
        self.zcv = Condition() 

        def watcher(handle,type,state,path):
            print "Z connected (%s:%s)" % (self.host, self.port)
            self.zcv.acquire()
            self.connected = True
            self.zcv.notify()
            self.zcv.release()

        self.zcv.acquire()
        self.connected = False

        self.handle = z.init("%s:%s"%(self.host,self.port), watcher, 10000, 0)
        self.zcv.wait(10.0)

        if not self.connected:
            print "Connection to Z cluster timed out - is a server running on %s:%s?" % (self.host,self.port)
            self.connected = False
            self.zcv.release()
            return
        self.zcv.release()

        # make sure root node exists
        try:
            self.create(self.root,"RACS root node", [ZOO_OPEN_ACL_UNSAFE])
        except IOError, e:
            if e.args[0] != z.zerror(z.NODEEXISTS):
                raise e
Exemple #25
0
def connect(servers):
    cond = threading.Condition()
    connected = [False]

    # We attempt a connection for 10 seconds here. This is a long timeout
    # for servicing a web request, so hopefully it is successful.
    def connect_watcher(zh, event, state, path):
        logging.debug("CONNECT WATCHER: event=%s, state=%s, path=%s" % (event, state, path))
        try:
            cond.acquire()
            if state == zookeeper.CONNECTED_STATE:
                # We only want to notify the main thread once the state has been
                # connected. We store the connected variable in an odd way because
                # of the way variables are bound in the local scope for functions.
                connected[0] = True
                cond.notify()
        finally:
            cond.release()

    cond.acquire()
    try:
        # We default to port 2181 if no port is provided as part of the host specification.
        server_list = ",".join(map(lambda x: (x.find(":") > 0 and x) or "%s:2181" % x, servers))
        handle = zookeeper.init(server_list, connect_watcher, 10000)
        cond.wait(60.0)
    finally:
        # Save whether we were successful or not.
        is_connected = connected[0]
        cond.release()

    if not(is_connected):
        raise ZookeeperException("Unable to connect.")

    return handle
    def testconnection(self):
        cv = threading.Condition()
        self.connected = False
        def connection_watcher(handle, type, state, path):
            cv.acquire()
            self.connected = True
            self.assertEqual(zookeeper.CONNECTED_STATE, state)
            self.handle = handle
            cv.notify()
            cv.release()

        cv.acquire()
        ret = zookeeper.init(self.host, connection_watcher)
        cv.wait(15.0)
        cv.release()
        self.assertEqual(self.connected, True, "Connection timed out to " + self.host)
        self.assertEqual(zookeeper.CONNECTED_STATE, zookeeper.state(self.handle))

        self.assertEqual(zookeeper.close(self.handle), zookeeper.OK)
        # Trying to close the same handle twice is an error, and the C library will segfault on it
        # so make sure this is caught at the Python module layer
        self.assertRaises(zookeeper.ZooKeeperException,
                          zookeeper.close,
                          self.handle)

        self.assertRaises(zookeeper.ZooKeeperException,
                          zookeeper.get,
                          self.handle,
                          "/")
Exemple #27
0
 def start(self):
     self.handle = zookeeper.init(self.servers, self.connection_watcher, self.timeout * 1000)
     self.conn_cv.acquire()
     self.conn_cv.wait(self.timeout)
     self.conn_cv.release()
     if not self.connected:
         raise TimeoutException
Exemple #28
0
    def __init__(self, zkservers, lockname):
        self.connected = False
        self.lockname = "/" + lockname
        self.uuid = str(uuid.uuid4())
        self.cv = threading.Condition()
        self.log = logging.getLogger('zookeeper')

        def connection_watcher(_handle, _type, _state, _path):
            self.cv.acquire()
            self.connected = True
            self.cv.notify()
            self.cv.release()

        self.cv.acquire()
        try:
            self.handle = zookeeper.init(",".join(zkservers), connection_watcher, 4000)
        except:
            self.log.exception('Failed to connect Zookeeper cluster (%r)' % zkservers)
            self.cv.release()
            raise SyncError(SYNC_LOCK_CONNECTION_FAILURE, "Failed to connect to Zookeeper cluster")

        self.cv.wait(4.0)
        if not self.connected:
            self.log.error('Failed to connect to Zookeeper cluster (%r)' % zkservers)
            self.cv.release()
            raise SyncError(SYNC_LOCK_CONNECTION_FAILURE, "Failed to connect to Zookeeper cluster")
        self.cv.release()
Exemple #29
0
def initialize(zkservers):
	global connected
	global zh
	zh = zookeeper.init(zkservers,check_connected)
	while True:
		if connected:
			break
		time.sleep(0.5)
Exemple #30
0
def prepare_zookeeper():
    handle = zookeeper.init( host );
    host_ip = socket.gethostbyname(socket.gethostname());
    domain_node = "%s/%s"%(rootnode, domain)
    create_node_with_parent_node( handle, domain_node );

    nodename = "%s/%s/%s"%( rootnode, domain, host_ip )
    return { "handle": handle, "nodename":nodename }
def start_zk(args, zk_path):
    print zk_path + '/bin/zkServer.sh start'
    subprocess.Popen(zk_path + '/bin/zkServer.sh start', shell=True)
    hostname = socket.gethostname()
    ip = socket.gethostbyname(hostname)
    print ip + ':' + str(args.port)
    if args.myid == 1:
        while True:
            time.sleep(2)
            handler = zookeeper.init(ip + ':' + str(args.port))
            try:
                zookeeper.create(handler, "/tera", "", [{
                    "perms": 0x7f,
                    "scheme": "world",
                    "id": "anyone"
                }], 0)
                zookeeper.create(handler, "/tera/master-lock", "",
                                 [{
                                     "perms": 0x7f,
                                     "scheme": "world",
                                     "id": "anyone"
                                 }], 0)
                zookeeper.create(handler, "/tera/ts", "", [{
                    "perms": 0x7f,
                    "scheme": "world",
                    "id": "anyone"
                }], 0)
                zookeeper.create(handler, "/tera/kick", "", [{
                    "perms": 0x7f,
                    "scheme": "world",
                    "id": "anyone"
                }], 0)
                zookeeper.close(handler)
                break
            except:
                print '#######################\nkeep trying!!\n##########################\n'
                traceback.print_exc()
                continue
Exemple #32
0
    def __init__(self, host, port, queuename):
        self.connected = False
        self.queuename = "/" + queuename
        self.cv = threading.Condition()
        zookeeper.set_log_stream(open("/dev/null"))

        def watcher(handle, type, state, path):
            # print "Connected"
            self.cv.acquire()
            self.connected = True
            self.cv.notify()
            self.cv.release()

        self.cv.acquire()
        host_s = host + ":" + str(port)
        self.handle = zookeeper.init(host_s, watcher, 2000000)
        self.cv.wait(10.0)
        if not self.connected:
            print(
                "Connection to ZooKeeper cluster timed out "
                "- is a server running on %s?" % host_s)
            sys.exit()
        self.cv.release()
	def set_up(self):
		self.cv = threading.Condition()
		self.connected = False
		def connection_watcher(handle, type, state, path):
			self.cv.acquire()
			self.connected = True
			self.cv.notify()
			self.cv.release()
		self.cv.acquire()
		self.handle = zookeeper.init(self.host, connection_watcher)
		self.cv.wait(10.0)
		self.cv.release()
		if not self.connected:
			raise Exception("Couldn't connect to host -", self.host)

		# set up the NG2znode look-up table:
		if not zookeeper.exists(self.handle, '/look-up', None):
			d = { 'default-graph' : '/dg' }
			zookeeper.create(self.handle, '/look-up', json.dumps(d), [ZOO_OPEN_ACL_UNSAFE])
		else:
			(self.NG2znode, stat) = zookeeper.get(self.handle, '/look-up', None)
			self.NG2znode =  json.loads(str(self.NG2znode))
			_logger.debug('%s' %self.NG2znode['default-graph'])
Exemple #34
0
    def __init__(self, servers, acl, timeout=DEFAULT_TIMEOUT):
        """
        zookeeper 操作类的封装
        """
        self.epoch = 0
        self.acl = acl
        self.timeout = timeout
        self.connected = False
        #线程同步对象
        self.conn_cv = threading.Condition()
        self.handle = -1
        #线程尝试获取锁,如果拿到就执行下面的代码,否则等待其他线程通知
        self.conn_cv.acquire()
        self.handle = zookeeper.init(servers, self.conn_watcher, timeout)
        #wait方法会释放锁,然后阻塞,等待其他线程执行notify或者notifyAll方法,如果指定的时间没有得到通知,线程重新获取锁,如果获取到锁,继续执行下面的代码
        self.conn_cv.wait(timeout / 10000)
        #释放锁
        self.conn_cv.release()
        #检查连接状态
        if not self.connected:
            raise ZKClientError("Unable to connect to %s" % (servers))

        if not self.add_auth(scheme=acl["scheme"], id=acl["id"]):
            raise ZKClientError("add_auth to zookeeper fail")
Exemple #35
0
def main():
    from optparse import OptionParser as OptionParser

    parser = OptionParser()
    parser.add_option('-v',
                      '--verbose',
                      action="store_true",
                      dest="flag_verbose",
                      help='Triggers debugging traces')
    parser.add_option(
            '-s', '--smart',
            action="store_true", dest="SMART", default=False,
            help="Delete onle the members belong to services with multiple" \
                 " members")
    parser.add_option('-d',
                      '--dry-run',
                      action="store_true",
                      dest="DRY",
                      default=False,
                      help="Do not delete, just print")
    parser.add_option('-n',
                      '--min-services',
                      type=int,
                      action="store",
                      dest="NUM",
                      default=4,
                      help="Do not delete election if less the NUM")
    parser.add_option('-1',
                      '--alone',
                      action="store_true",
                      dest="ALONE",
                      default=False,
                      help="Also consider members alone in their group")

    (options, args) = parser.parse_args(sys.argv)

    # Logging configuration
    if options.flag_verbose:
        logging.basicConfig(format='%(asctime)s %(message)s',
                            datefmt='%m/%d/%Y %I:%M:%S',
                            level=logging.DEBUG)
    else:
        logging.basicConfig(format='%(asctime)s %(message)s',
                            datefmt='%m/%d/%Y %I:%M:%S',
                            level=logging.INFO)

    if len(args) < 2:
        raise ValueError("not enough CLI arguments: NS TYPE [TYPE...]")

    ns = args[1]
    cnxstr = load_namespace_conf(ns)['zookeeper']

    zookeeper.set_debug_level(zookeeper.LOG_LEVEL_INFO)
    zh = zookeeper.init(cnxstr)

    for srvtype in args[2:]:
        for group in namespace_tree(ns, srvtype):
            logging.debug(">DIR %s", group)
            for node in list_problematic_nodes(zh, group, options):
                delete_node(zh, node, options)

    zookeeper.close(zh)
Exemple #36
0
        len1 = len(runtimecontainer[i].split('='))
        #print len1
        if (len1 == 2):
            dic1 = runtimecontainer[i].split('=')[0]
            value1 = runtimecontainer[i].split('=')[1]
            items.append((str(dic1), str(value1)))
            #print items

    d = dict(items)
    return d


#函数结束

#初始化zookeeper连接
zk = zoo.init("10.245.0.96:2181")
zoo.set_debug_level(zoo.LOG_LEVEL_ERROR)
#获取所有的应用
path = "/UCMP/instances"
try:
    application = zoo.get_children(zk, path)
except:
    print
    print "1===没有相关节点信息,请确认程序名称或者版本信息"
    zoo.close(zk)
    os._exit(0)
applicationlist = []
for index in range(len(application)):
    #if ( list(application[index])[-1] != '_' and list(application[index])[0] != '&'):
    if (not (application[index].endswith('_')
             or application[index].startswith('&'))):
import os

import zookeeper
import pykeeper

host = os.environ['DOTCLOUD_ZOOKEEPER_CONNECTION_HOST']
port = os.environ['DOTCLOUD_ZOOKEEPER_CONNECTION_PORT']
connection_str = '%s:%s' % (host, port)
#client = pykeeper.ZooKeeper(connection_str)


def callback(*args, **kwargs):
    print args
    print kwargs


handle = zookeeper.init(connection_str, callback)

zookeeper.client_id(handle)
zookeeper.exists(handle, '/')
zookeeper.get(handle, '/')
zookeeper.get_children(handle, '/')
zookeeper.state(handle)
Exemple #38
0
 def _initialize_register(self):
     if self._register_handler is None:
         self._register_handler = zookeeper.init(self._hosts,
                                                 self._register_watcher)
Exemple #39
0
 def _initialize_discovery(self):
     if self._discovery_handler is None:
         self._discovery_handler = zookeeper.init(self._hosts,
                                                  self._discovery_watcher)
Exemple #40
0
import zookeeper
import time


def myWatch(handler, type, state, path):
    print "handler:" + str(handler) + ",type:" + str(type) + ",state:" + str(
        state) + ",path:" + path


#        print  'a'
#        print str(type)
#        print  'b'

path = '/haha'
handler = zookeeper.init("10.0.100.131:6181")
#zookeeper.get(handler,path,myWatch)
while True:
    data = zookeeper.get(handler, path, myWatch)
    #      print  data
    time.sleep(50)
Exemple #41
0
def main():
    from optparse import OptionParser as OptionParser

    parser = OptionParser()
    parser.add_option('-v',
                      '--verbose',
                      action="store_true",
                      dest="flag_verbose",
                      default=False,
                      help='Triggers debugging traces')
    parser.add_option('--lazy',
                      action="store_true",
                      dest="LAZY",
                      default=False,
                      help='Quickly check if things seem OK.')
    parser.add_option(
        '--slow',
        action="store_true",
        dest="SLOW",
        default=False,
        help='Send small batches to avoid timeouts on slow hosts.')
    parser.add_option('--avoid',
                      action="append",
                      type="string",
                      dest="AVOID_TYPES",
                      help='Avoid entries for the specified service types')

    (options, args) = parser.parse_args(sys.argv)

    # Logging configuration
    if options.flag_verbose:
        logging.basicConfig(format='%(asctime)s %(message)s',
                            datefmt='%m/%d/%Y %I:%M:%S',
                            level=logging.DEBUG)
    else:
        logging.basicConfig(format='%(asctime)s %(message)s',
                            datefmt='%m/%d/%Y %I:%M:%S',
                            level=logging.INFO)

    if len(args) < 2:
        raise ValueError("not enough CLI arguments")

    ns = args[1]
    cnxstr = load_namespace_conf(ns)['zookeeper']
    zookeeper.set_debug_level(zookeeper.LOG_LEVEL_INFO)
    for shard in cnxstr.split(";"):
        logging.info("ZK=%s", shard)
        zh = zookeeper.init(shard)

        # synchronous creation of the root
        try:
            zookeeper.create(zh, PREFIX, '', acl_openbar, 0)
        except zookeeper.NodeExistsException:
            pass

        missing = True
        if options.LAZY:
            _m = False
            for t, _, _ in SRVTYPES:
                try:
                    _, _ = zookeeper.get(zh, PREFIX_NS + '/' + ns + '/el/' + t)
                except Exception:
                    _m = True
            missing = _m

        if missing:
            create_tree(zh, namespace_tree(ns, options), options)
        zookeeper.close(zh)
Exemple #42
0
#/usr/bin/env python
#coding=utf8
"""
# Author: kellanfan
# Created Time : Tue 27 Nov 2018 01:52:20 PM CST

# File Name: myzktest.py
# Description:

"""

import zookeeper


def myWatch(zk, type, state, path):
    print("zk:", str(type), str(state), str(path))
    zookeeper.get(zk, path, myWatch)


zk = zookeeper.init("192.168.10.10:2181")
data = zookeeper.get(zk, "/zk-bbb", myWatch)
print(data)
import zookeeper
import time
import sys
import threading
#zk = zookeeper.init("10.10.1.201:6181,10.10.1.212:6181,10.10.1.213:6181")
zk = zookeeper.init("10.0.100.131:6181")

#a=zk.get_children(zk, "/", None)
#zookeeper.create(zk,"/test3","hehe",[{"perms":0x1f,"scheme":"world","id":"anyone"}],0)


def set(i):
    nodeinfo = '/node%d' % i
    datainfo = 'datajfajdfkjaslkjdfajsfiwerkjkjelkjkljafd%d' % i
    zookeeper.create(zk, nodeinfo, datainfo, [{
        "perms": 0x1f,
        "scheme": "world",
        "id": "anyone"
    }], 0)
    print 'insert  number %d is ok! ' % i


def get(i):
    nodeinfo = '/node%d' % i
    data = zookeeper.get(zk, nodeinfo)
    print data


start_time = time.time()
for i in range(1100, 2100):
    t = threading.Thread(target=get, args=(i, ))
Exemple #44
0
 def connect(self):
     self.handle = zookeeper.init(self.servers, self._global_watcher)
     self.on_state(self, self.state_name)
Exemple #45
0
def get_connected_handles(cnxstr):
    zookeeper.set_debug_level(zookeeper.LOG_LEVEL_INFO)
    for shard in cnxstr.split(";"):
        zh = zookeeper.init(shard)
        yield ZkHandle(zh)
Exemple #46
0
    def reconnect(self):
        """Attempt to reconnect to ZK."""
        if self._stopped.is_set():
            self._safe_close()
            return

        def safe_close(zh):
            try:
                zookeeper.close(zh)
            except:
                # TODO(wickman) When the SystemError bug is fixed in zkpython, narrow this except clause.
                pass

        def activate():
            self._authenticated.set()
            self._live.set()

        def on_authentication(zh, rc):
            if self._zh != zh:
                safe_close(zh)
                return
            if rc == zookeeper.OK:
                activate()

        def maybe_authenticate():
            if self._authenticated.is_set() or not self._credentials:
                activate()
                return
            try:
                scheme, credentials = self._credentials
                zookeeper.add_auth(self._zh, scheme, credentials,
                                   on_authentication)
            except zookeeper.ZooKeeperException as e:
                self._logger('Failed to authenticate: %s' % e)

        def connection_handler(handle, type, state, path):
            if self._zh != handle:
                safe_close(handle)
                return
            if self._stopped.is_set():
                return
            if self._watch:
                self._watch(self, type, state, path)
            if state == zookeeper.CONNECTED_STATE:
                self._logger('Connection started, setting live.')
                maybe_authenticate()
                self._clear_completions()
            elif state == zookeeper.EXPIRED_SESSION_STATE:
                self._logger('Session lost, clearing live state.')
                self._session_expirations.increment()
                self._live.clear()
                self._authenticated.clear()
                self._zh = None
                self._init_count = 0
                self.reconnect()
            else:
                self._logger('Connection lost, clearing live state.')
                self._connection_losses.increment()
                self._live.clear()

        # this closure is exposed for testing only -- in order to simulate session events.
        self._handler = connection_handler

        timeout_ms = int(self._timeout_secs * 1000)
        while True:
            self._safe_close()
            servers = self.expand_ensemble(self._servers)
            self._log('Connecting to ZK hosts at %s' % servers)
            self._zh = zookeeper.init(servers, connection_handler, timeout_ms)
            self._init_count += 1
            self._live.wait(self._timeout_secs + 1)
            if self._live.is_set():
                break
            elif self._max_reconnects > 0 and self._init_count >= self._max_reconnects:
                self._safe_close()
                raise ZooKeeper.ConnectionTimeout(
                    'Timed out waiting for ZK connection to %s' % servers)
        self._log('Successfully connected to ZK at %s' % servers)
Exemple #47
0
 def start(self):
     if self._handler is None:
         self._handler = zookeeper.init(self._hosts,
                                        self._connection_state_watcher)
         LOGGER.info("create zookeeper client successfully, "
                     "handler is: %s" % self._handler)
Exemple #48
0
zookeeper.set_log_stream(f)

connected = False
conn_cv = threading.Condition( )

def my_connection_watcher(handle,type,state,path):
    global connected, conn_cv
    print("Connected, handle is ", handle)
    conn_cv.acquire()
    connected = True
    conn_cv.notifyAll()
    conn_cv.release()
    
conn_cv.acquire()
print("Connecting to localhost:2181 -- ")
handle = zookeeper.init("localhost:2181", my_connection_watcher, 10000, 0)
while not connected:
    conn_cv.wait()
conn_cv.release()

def my_getc_watch( handle, type, state, path ):
    print("Watch fired -- ")
    print(type, state, path)

ZOO_OPEN_ACL_UNSAFE = {"perms":0x1f, "scheme":"world", "id" :"anyone"};

try:
    zookeeper.create(handle, "/zk-python", "data", [ZOO_OPEN_ACL_UNSAFE], 0)
    zookeeper.get_children(handle, "/zk-python", my_getc_watch)
    for i in xrange(5):
        print("Creating sequence node ", i, " ", zookeeper.create(handle, "/zk-python/sequencenode", "data", [ZOO_OPEN_ACL_UNSAFE], zookeeper.SEQUENCE ))
Exemple #49
0
 def _init_handle(self):
     if hasattr(self, '_zh'):
         zookeeper.close(self._zh)
     
     self._zh = zookeeper.init(self._hosts, self._global_watcher(), 1000)
Exemple #50
0
            _pos = -1
        else:
            item[1]['__sep__'] = sep + '├─'
            _pos = 0

        html = html + filter_node_treegrid(item[1], depth + 1, _pos, sep)

    return html


######################################################################
#                              开始代码执行                          #
######################################################################

# zookeeper连接
zk = zookeeper.init(config.zookeeper['server'], watcher)

# 初始化、配置app
app = Flask(__name__)
app.config.from_mapping(config.site)

# 注册模板函数到jinja2
app.jinja_env.filters['datetime'] = filter_datetime
app.jinja_env.filters['json_dumps'] = filter_json_dumps
app.jinja_env.filters['url_encode'] = filter_url_encode
app.jinja_env.filters['url_quote'] = filter_url_quote
app.jinja_env.filters['node_treegrid'] = filter_node_treegrid


# 在所有的页面请求前执行
@app.before_request
Exemple #51
0
        for k in path:
            p = path[k]

            _project = project.copy()
            _project.update({'key': k})

            if not os.path.exists(G.tmp_dir + p):
                os.makedirs(G.tmp_dir + p)

            if not G.project.has_key(p):
                G.project[p] = []

            G.project[p].append(_project)

    # zookeeper 连接
    G.zookeeper = zookeeper.init(G.zookeeper_server, watcher)

    # /service/<name>
    zookeeper_node_set(SERVIE_NODE / G.name, json.dumps(G.project),
                       zookeeper.PERM_ALL)
    # /online/<name>
    nodepath = ONLINE_NODE / G.name
    if zookeeper.exists(G.zookeeper, nodepath):
        raise BaseException('The same service has been run in other places')
    zookeeper_node_set(nodepath,
                       '',
                       zookeeper.PERM_READ,
                       ephemeral=True,
                       parent_perms=zookeeper.PERM_ALL)
except BaseException, e:
    print e.message
import zookeeper
handler = zookeeper.init('localhost:2181')
#0表示持久化的,1表示持久化+序号,2表示瞬时的,3表示瞬时加序号型的
zookeeper.create(handler, '/zkpython_create_node', 'mydata1')