Esempio n. 1
0
    def __init__(self, barriername, number_of_workers):
	self.cv = threading.Condition()
	self.connected = False
	self.barrier = "/" + barriername
	self.workers = number_of_workers
	zookeeper.set_log_stream(open('/dev/null'))
	def watcher(handle, type, state, path):
	    self.cv.acquire()
	    self.connected = True
	    self.cv.notify()
	    self.cv.release()

	self.cv.acquire()
	self.handle = zookeeper.init("localhost:2181", watcher, 10000)
	self.cv.wait(10.0)
	if not self.connected:
            print "Connection to ZooKeeper cluster timed out - is a server running on localhost:2181?"
            sys.exit()
	self.cv.release()
	try:
	    zookeeper.create(self.handle, self.barrier, '\x00', [ZOO_OPEN_ACL_UNSAFE], 0)
	except zookeeper.NodeExistsException:
	    pass
	except Exception, ex:
	    print ex
	    raise ex
Esempio n. 2
0
    def test_sync_get_large_datanode(self):
        """
        Test that we can retrieve datanode sizes up to
        1Mb with default parameters (depends on ZooKeeper server).
        """

        data = ''.join(["A" for x in range(1024 * 1023)])
        self.ensureDeleted("/zk-python-test-large-datanode")
        zookeeper.create(self.handle, "/zk-python-test-large-datanode", data,
                         [{
                             "perms": 0x1f,
                             "scheme": "world",
                             "id": "anyone"
                         }])
        (ret, stat) = zookeeper.get(self.handle,
                                    "/zk-python-test-large-datanode")
        self.assertEqual(
            len(ret), 1024 * 1023,
            "Should have got 1Mb returned, instead got %s" % len(ret))
        (ret, stat) = zookeeper.get(self.handle,
                                    "/zk-python-test-large-datanode", None,
                                    500)
        self.assertEqual(
            len(ret), 500,
            "Should have got 500 bytes returned, instead got %s" % len(ret))
Esempio n. 3
0
  def __init__(self, service_name, connect_string, timeout=DEFAULT_TIMEOUT, default_port=6664):
    self.SERVICE_NODE = "/" + service_name
    self.AVAILABILITY_NODE = self.SERVICE_NODE + "/available"
    self.MEMBERSHIP_NODE = self.SERVICE_NODE + "/members"
    self.connected = False
    self.timeout = timeout
    self.default_port = default_port
    self.conn_cv = threading.Condition()
    self.conn_cv.acquire()
    self.handle = zookeeper.init(connect_string, self.connection_watcher, timeout)
    self.conn_cv.wait(timeout / 1000)
    self.conn_cv.release()
    self.watcher_lock = threading.Lock()
    self.logger = logging.getLogger("sincc")

    if not self.connected:
      raise SinClusterClientError("Unable to connect to %s" % connect_string)

    for path in [self.SERVICE_NODE, self.AVAILABILITY_NODE, self.MEMBERSHIP_NODE]:
      if not zookeeper.exists(self.handle, path):
        zookeeper.create(self.handle, path, "", [ZOO_OPEN_ACL_UNSAFE], 0)
    self.listeners = []
    # Start to watch both /members and /available
    zookeeper.get_children(self.handle, self.MEMBERSHIP_NODE, self.watcher)
    available = zookeeper.get_children(self.handle, self.AVAILABILITY_NODE, self.watcher)
    self.available_nodes = {}
    for node_id in available:
      self.available_nodes[int(node_id)] = Node(int(node_id),
                                                zookeeper.get(self.handle, self.AVAILABILITY_NODE + "/" + node_id)[0])
Esempio n. 4
0
def connect(host=None):
    global conn_cv, connected, handle
    global reconnect_host

    if host is None:
        host = "localhost:2181"

    reconnect_host = host

    conn_cv.acquire()
    handle = zookeeper.init(host, my_connection_watcher, 10000)
    while not connected:
        conn_cv.wait()
    conn_cv.release()

    connected = False
    while not connected:
        try:
            zookeeper.create(handle, ROOT, "1", [ZOO_OPEN_ACL_UNSAFE], 0)
            connected = True
        except zookeeper.NodeExistsException as e:
            # No worries
            connected = True
        except zookeeper.ConnectionLossException:
            continue
        except:
            raise
Esempio n. 5
0
    def register_consumer(self, consumer_group, consumer_id, topic):
        """Creates the following permanent node, if it does not exist already:
            /consumers/{consumer_group}/ids

        The data written at this node is just the consumer_id so that we can 
        later track who created what.

        We then create the following emphemeral node:
            /consumers/{consumer_group}/ids/{consumer_id}
        
        The data written in this node is a dictionary of topic names (in 
        unicode) to the number of threads that this consumer has registered
        for this topic (in our case, always one).
        """
        self._create_path_if_needed(self.path_for_consumer_ids(consumer_group),
                                    consumer_id)
        # Create an emphermal node for this consumer
        consumer_id_path = self.path_for_consumer_id(consumer_group, consumer_id)
        log.info("Registering Consumer {0}, trying to create {1}"
                 .format(consumer_id, consumer_id_path))
        zookeeper.create(self._zk.handle, 
                         consumer_id_path,
                         json.dumps({topic : 1}), # topic : # of threads
                         ZKUtil.ACL,
                         zookeeper.EPHEMERAL)
Esempio n. 6
0
    def __init__(self, queuename):
        self.connected = False
        self.queuename = "/" + queuename
        self.cv = threading.Condition()
        zookeeper.set_log_stream(open("/dev/null"))

        def watcher(handle, type, state, path):
            print "Connected"
            self.cv.acquire()
            self.connected = True
            self.cv.notify()
            self.cv.release()

        self.cv.acquire()
        self.handle = zookeeper.init("localhost:2181", watcher, 10000)
        self.cv.wait(10.0)
        if not self.connected:
            print "Connection to ZooKeeper cluster timed out - is a server running on localhost:2181?"
            sys.exit()
        self.cv.release()
        try:
            zookeeper.create(self.handle, self.queuename, "queue top level",
                             [ZOO_OPEN_ACL_UNSAFE], 0)
        except zookeeper.NodeExistsException:
            print "Queue already exists"
Esempio n. 7
0
  def __tryGC(self, app_id, app_path):
    try:
#      print "try to obtrain app %s lock" % app_id
      val = zookeeper.get(self.handle, PATH_SEPARATOR.join([app_path, GC_TIME_PATH]), None)[0]
      lasttime = float(val)
    except zookeeper.NoNodeException:
      lasttime = 0
    if lasttime + GC_INTERVAL < time.time():
      # try to gc this app.
      gc_path = PATH_SEPARATOR.join([app_path, GC_LOCK_PATH])
      try:
        now = str(time.time())
        zookeeper.create(self.handle, gc_path, now, ZOO_ACL_OPEN, zookeeper.EPHEMERAL)
        # succeed to obtain lock.
        # TODO: should we handle the timeout of gc also?
        try:
          self.__executeGC(app_id, app_path)
          # update lasttime when gc was succeeded.
          now = str(time.time())
          self.__updateNode(PATH_SEPARATOR.join([app_path, GC_TIME_PATH]), now)
        except Exception, e:
          print "warning: gc error %s" % e
          traceback.print_exc()
        zookeeper.delete(self.handle, gc_path, -1)
      except zookeeper.NodeExistsException:
        # fail to obtain lock. try next time.
        pass
Esempio n. 8
0
 def __init__(self,queuename):
     self.connected = False
     self.queuename = "/" + queuename
     self.cv = threading.Condition()
     zookeeper.set_log_stream(open("/dev/null"))        
     def watcher(handle,type,state,path):
         print "Connected"
         self.cv.acquire()
         self.connected = True
         self.cv.notify()
         self.cv.release()
         
     self.cv.acquire()            
     self.handle = zookeeper.init("localhost:2181", watcher, 10000, 0)
     self.cv.wait(10.0)
     if not self.connected:
         print "Connection to ZooKeeper cluster timed out - is a server running on localhost:2181?"
         sys.exit()
     self.cv.release()
     try:
         zookeeper.create(self.handle,self.queuename,"queue top level", [ZOO_OPEN_ACL_UNSAFE],0)
     except IOError, e:
         if e.message == zookeeper.zerror(zookeeper.NODEEXISTS):
             print "Queue already exists"    
         else:
             raise e
Esempio n. 9
0
def create_node(zh, options, path):
    try:
        zookeeper.create(zh, path, options.VALUE, acl_openbar,
                         zookeeper.EPHEMERAL)
        logging.info("OK create %s", path)
    except Exception as e:
        logging.info("ERROR create %s: %s", path, e)
Esempio n. 10
0
    def set(self, path, data='', previous_data=None):
        path = '%s%s' % (self.prefix, path)
        try:
            current_data, current_meta = zookeeper.get(self.handle, path, None)
        except zookeeper.NoNodeException:
            if not previous_data:
                try:
                    zookeeper.create(self.handle, path, data, [ZOO_OPEN_ACL_UNSAFE])
                    return True
                except:
                    self.log.exception('Failed to create a missing key %s', path)
                    return False
            else:
                return False
        except:
            self.log.exception('Failed to set key %s', path)
            return False

        version = None
        if previous_data:
            if current_data != previous_data:
                self.log.error('Previous data constraint failed')
                return False
            version = current_meta['version']

        try:
            if version is None:
                zookeeper.set(self.handle, path, data)
            else:
                zookeeper.set(self.handle, path, data, version)
        except:
            self.log.exception('Set failed')
            return False

        return True
Esempio n. 11
0
def connect(host=None):
    global conn_cv, connected, handle
    global reconnect_host

    if host is None:
        host = "localhost:2181"

    reconnect_host = host

    conn_cv.acquire()
    handle = zookeeper.init(host, my_connection_watcher, 10000)
    while not connected:
        conn_cv.wait()
    conn_cv.release()

    connected = False
    while not connected:
        try:
            zookeeper.create(handle, ROOT, "1", [ZOO_OPEN_ACL_UNSAFE], 0)
            connected = True
        except zookeeper.NodeExistsException as e:
            # No worries
            connected = True
        except zookeeper.ConnectionLossException:
            continue
        except:
            raise
Esempio n. 12
0
 def setUp( self ):
     zktestbase.TestBase.setUp(self)
     try:
         zookeeper.create(self.handle, "/zk-python-existstest","existstest", [ZOO_OPEN_ACL_UNSAFE],zookeeper.EPHEMERAL)
         zookeeper.create(self.handle, "/zk-python-aexiststest","existstest",[ZOO_OPEN_ACL_UNSAFE],zookeeper.EPHEMERAL)
     except:
         pass
Esempio n. 13
0
 def watch_session(handle, event_type, state, path):
     assert event_type == zookeeper.SESSION_EVENT
     assert not path
     if state == zookeeper.CONNECTED_STATE:
         if self.handle is None:
             self.handle = handle
             for watch in self.watches.clear():
                 self._watch(watch)
             for path, data in list(self.ephemeral.items()):
                 zookeeper.create(self.handle, path, data['data'],
                                  data['acl'], data['flags'])
         else:
             assert handle == self.handle
         connected.set()
         logger.info('connected %s', handle)
     elif state == zookeeper.CONNECTING_STATE:
         connected.clear()
     elif state == zookeeper.EXPIRED_SESSION_STATE:
         connected.clear()
         if self.handle is not None:
             zookeeper.close(self.handle)
         self.handle = None
         init()
     else:
         logger.critical('unexpected session event %s %s', handle, state)
Esempio n. 14
0
def zookeeper_node_set(nodepath, nodevalue, perms, **option):
    ephemeral = 0
    if option.has_key('ephemeral') and option['ephemeral']:
        ephemeral = zookeeper.EPHEMERAL

    parent_perms = perms
    if option.has_key('parent_perms'):
        parent_perms = option['parent_perms']

    p = FilePath('/')

    for v in nodepath.split('/'):
        p = p / v

        if not zookeeper.exists(G.zookeeper, p):
            if not option.has_key('nocreate') or not option['nocreate']:
                if p == nodepath:
                    print zookeeper.create(G.zookeeper, p, nodevalue,
                                           [{
                                               "perms": perms,
                                               "scheme": "world",
                                               "id": "anyone"
                                           }], ephemeral)
                    return True
                else:
                    zookeeper.create(G.zookeeper, p, '', [{
                        "perms": parent_perms,
                        "scheme": "world",
                        "id": "anyone"
                    }], 0)
        elif p == nodepath:
            print zookeeper.set(G.zookeeper, p, nodevalue)
            return True

    return False
Esempio n. 15
0
    def create(self, path, data="", flags=0, makepath=False, force=False):
        """
        创建节点
        """
        acl2 = {"perms": 0x1f, "scheme": "world", "id": "anyone"}
        if self.acl["scheme"] == "digest":
            user, _ = self.acl["id"].split(':')
            id2 = "%s:%s" % (user,
                             base64.b64encode(
                                 hashlib.new("sha1", self.acl["id"]).digest()))
            acl2 = {"perms": self.acl["perms"], "scheme": "digest", "id": id2}
        log.info("zk create [%s]" % path)
        if makepath:
            if not self.exists(os.path.dirname(path)):
                self.create(path=os.path.dirname(path),
                            data="",
                            flags=flags,
                            makepath=True)
        log.info(path)
        if force and self.exists(path):
            self.delete(path)

        zookeeper.create(self.handle, path, data, [acl2], flags)
        if self.exists(path):
            return 1
        else:
            return 0
Esempio n. 16
0
    def register_consumer(self, consumer_group, consumer_id, topic):
        """Creates the following permanent node, if it does not exist already:
            /consumers/{consumer_group}/ids

        The data written at this node is just the consumer_id so that we can 
        later track who created what.

        We then create the following emphemeral node:
            /consumers/{consumer_group}/ids/{consumer_id}
        
        The data written in this node is a dictionary of topic names (in 
        unicode) to the number of threads that this consumer has registered
        for this topic (in our case, always one).
        """
        self._create_path_if_needed(self.path_for_consumer_ids(consumer_group),
                                    consumer_id)
        # Create an emphermal node for this consumer
        consumer_id_path = self.path_for_consumer_id(consumer_group,
                                                     consumer_id)
        log.info("Registering Consumer {0}, trying to create {1}".format(
            consumer_id, consumer_id_path))
        zookeeper.create(
            self._zk.handle,
            consumer_id_path,
            json.dumps({topic: 1}),  # topic : # of threads
            ZKUtil.ACL,
            zookeeper.EPHEMERAL)
Esempio n. 17
0
    def _create_path_if_needed(self, path, data=None):
        """Creates permanent nodes for all elements in the path if they don't
        already exist. Places data for each node created. (You'll probably want
        to use the consumer_id for this, just for accounting purposes -- it's 
        not used as part of the balancing algorithm).

        Our zc.zk.ZooKeeper object doesn't know how to create nodes, so we
        have to dig into the zookeeper base library. Note that we can't create
        all of it in one go -- ZooKeeper only allows atomic operations, and
        so we're creating these one at a time.
        """
        if not path.startswith("/"):
            raise ValueError("Paths must be fully qualified (begin with '/').")

        def _build_path(nodes):
            return "/" + "/".join(nodes)

        nodes_to_create = path[1:].split("/")  # remove beginning "/"
        created_so_far = []
        for node in nodes_to_create:
            created_path = _build_path(created_so_far)
            if node and node not in self._zk.children(created_path).data:
                node_to_create = _build_path(created_so_far + [node])
                # If data is a string, we'll initialize the node with it...
                if isinstance(data, basestring):
                    init_data = data
                else:
                    init_data = json.dumps(data)
                zookeeper.create(self._zk.handle, node_to_create, init_data,
                                 ZKUtil.ACL)
            created_so_far.append(node)
Esempio n. 18
0
    def _real_register(self):
        for znode, data in self._register_entry_set.iter_entry():
            if self._base_znode == "/":
                znode = "/%s" % znode
            else:
                znode = "%s/%s" % (self._base_znode, znode)

            while True:
                try:
                    zookeeper.create(
                        self._register_handler,
                        znode,
                        data,
                        # 权限
                        [{
                            "perms": 0x1f,
                            "scheme": "world",
                            "id": "anyone"
                        }],
                        # 节点类型
                        zookeeper.EPHEMERAL)
                    LOGGER.info("create znode: %s successfully" % znode)
                    break
                except zookeeper.NodeExistsException:
                    if self._delete_if_exists:
                        LOGGER.info("%s already exists, delete it anyway" %
                                    znode)
                        try:
                            zookeeper.delete(self._register_handler, znode)
                        except zookeeper.NoNodeException:
                            pass
                        continue
                    raise
Esempio n. 19
0
  def __tryGC(self, app_id, app_path):
    try:
#      print "try to obtrain app %s lock" % app_id
      val = zookeeper.get(self.handle, PATH_SEPARATOR.join([app_path, GC_TIME_PATH]), None)[0]
      lasttime = float(val)
    except zookeeper.NoNodeException:
      lasttime = 0
    if lasttime + GC_INTERVAL < time.time():
      # try to gc this app.
      gc_path = PATH_SEPARATOR.join([app_path, GC_LOCK_PATH])
      try:
        now = str(time.time())
        zookeeper.create(self.handle, gc_path, now, ZOO_ACL_OPEN, zookeeper.EPHEMERAL)
        # succeed to obtain lock.
        # TODO: should we handle the timeout of gc also?
        try:
          self.__executeGC(app_id, app_path)
          # update lasttime when gc was succeeded.
          now = str(time.time())
          self.__updateNode(PATH_SEPARATOR.join([app_path, GC_TIME_PATH]), now)
        except Exception, e:
          print "warning: gc error %s" % e
          traceback.print_exc()
        zookeeper.delete(self.handle, gc_path, -1)
      except zookeeper.NodeExistsException:
        # fail to obtain lock. try next time.
        pass
Esempio n. 20
0
  def __init__(self,queuename, port, is_producer=False):
    self.connected = False
    self.queuename = "/" + queuename
    self.cv = threading.Condition()
    zookeeper.set_log_stream(open("/dev/null"))
    def watcher(handle,type,state,path):
      print "Connected"
      self.cv.acquire()
      self.connected = True
      self.cv.notify()
      self.cv.release()

    self.cv.acquire()
    self.handle = zookeeper.init("localhost:%d" % port, watcher, 10000)
    self.cv.wait(10.0)
    if not self.connected:
      print "Connection to ZooKeeper cluster timed out - is a server running on localhost:%d?" % port
      sys.exit()
    self.cv.release()
    if is_producer:
      while True:
        try:
          zookeeper.create(self.handle,self.queuename,"queue top level", [ZOO_OPEN_ACL_UNSAFE],0)
          print "Fila criada, OK"
          return
        except zookeeper.NodeExistsException:
          print "Tratorando filas existentes"
          while True:
            children = sorted(zookeeper.get_children(self.handle, self.queuename,None))
            if len(children) == 0:
              (data,stat) = zookeeper.get(self.handle, self.queuename, None)
              zookeeper.delete(self.handle, self.queuename, stat["version"])
              break
            for child in children:
              data = self.get_and_delete(self.queuename + "/" + child)
Esempio n. 21
0
    def _create_path_if_needed(self, path, data=None):
        """Creates permanent nodes for all elements in the path if they don't
        already exist. Places data for each node created. (You'll probably want
        to use the consumer_id for this, just for accounting purposes -- it's 
        not used as part of the balancing algorithm).

        Our zc.zk.ZooKeeper object doesn't know how to create nodes, so we
        have to dig into the zookeeper base library. Note that we can't create
        all of it in one go -- ZooKeeper only allows atomic operations, and
        so we're creating these one at a time.
        """
        if not path.startswith("/"):
            raise ValueError("Paths must be fully qualified (begin with '/').")

        def _build_path(nodes):
            return "/" + "/".join(nodes)

        nodes_to_create = path[1:].split("/") # remove beginning "/"
        created_so_far = []
        for node in nodes_to_create:
            created_path = _build_path(created_so_far)
            if node and node not in self._zk.children(created_path).data:
                node_to_create = _build_path(created_so_far + [node])
                # If data is a string, we'll initialize the node with it...
                if isinstance(data, basestring):
                    init_data = data 
                else:
                    init_data = json.dumps(data)
                zookeeper.create(self._zk.handle, node_to_create, init_data, ZKUtil.ACL)
            created_so_far.append(node)
Esempio n. 22
0
def main():
	from optparse import OptionParser as OptionParser

	parser = OptionParser()
	parser.add_option('-v', '--verbose', action="store_true", dest="flag_verbose",
		help='Triggers debugging traces')

	(options, args) = parser.parse_args(sys.argv)

	# Logging configuration
	if options.flag_verbose:
		logging.basicConfig(
			format='%(asctime)s %(message)s',
			datefmt='%m/%d/%Y %I:%M:%S',
			level=logging.DEBUG)
	else:
		logging.basicConfig(
			format='%(asctime)s %(message)s',
			datefmt='%m/%d/%Y %I:%M:%S',
			level=logging.INFO)

	if len(args) < 2:
		raise ValueError("not enough CLI arguments")

	ns = args[1]
	cnxstr = load_config().get(ns, 'zookeeper')

	zookeeper.set_debug_level(zookeeper.LOG_LEVEL_INFO)
	zh = zookeeper.init(cnxstr)
	zookeeper.create(zh, PREFIX, '', acl_openbar, 0)
	create_tree(zh, boot_tree())
	init_namespace(zh, ns)
	zookeeper.close(zh)
Esempio n. 23
0
    def __init__(self, zkhosts, root=NODE_HQ_ROOT, alivenode='alive'):
        """zkhosts: a string or a list. list will be ','.join-ed into a string.
        root: root node path (any parents must exist, if any)
        """
        
        if not isinstance(zkhosts, basestring):
            zkhosts = ','.join(zkhosts)
        self.zkhosts = zkhosts
        self.ROOT = root
        self.alivenode = alivenode
        self.nodename = os.uname()[1]

        self.NODE_SERVERS = self.ROOT + '/servers'
        self.NODE_ME = self.NODE_SERVERS+'/'+self.nodename

        self.zh = zk.init(self.zkhosts, self.__watcher)

        self.jobs = {}

        self.initialize_node_structure()

        if not zk.exists(self.zh, self.NODE_ME):
            zk.create(self.zh, self.NODE_ME, '', PERM_WORLD)

        # setup notification
        zk.get_children(self.zh, self.NODE_SERVERS, self.__servers_watcher)
        #zk.get_children(self.zh, self.NODE_ME, self.__watcher)

        self.NODE_JOBS = self.NODE_ME+'/jobs'
        zk.acreate(self.zh, self.NODE_JOBS, '', PERM_WORLD)
Esempio n. 24
0
    def try_lock(self):
        while True:
            try:
                zookeeper.create(self.handle, self.lockname, self.uuid,
                                 [ZOO_OPEN_ACL_UNSAFE], zookeeper.EPHEMERAL)
            except:
                self.log.debug('Failed to acquire lock (%r)' % self.lockname)
                return False

            try:
                (data, _) = zookeeper.get(self.handle, self.lockname, None)
                break
            except:
                self.log.exception(
                    'try_lock: create succeeded but get failed? (%r)' %
                    self.lockname)
                continue

        if data == self.uuid:
            self.log.debug('Lock acquired (%r)' % self.lockname)
            return True
        else:
            self.log.error(
                'try_lock: create succeeded but data is wrong? (%r)' %
                self.lockname)
            print "failed to acquire lock"
            return False
Esempio n. 25
0
 def register_machine(self, machine, data):
     if not isinstance(data, odin_pb2.Machine):
         raise TypeError('data must be a protobuffer Machine')
     zookeeper.create(self._z,
                      machine,
                      data.SerializeToString(),
                      [ZOO_OPEN_ACL_UNSAFE],
                      zookeeper.EPHEMERAL)
Esempio n. 26
0
 def create(self, path="", value=""):
     try:
         zookeeper.create(self.zk, path, value, self.acl, self.flags)
     except zookeeper.NodeExistsException:
         pass
     except zookeeper.NoNodeException:
         self.create(path=path.rsplit("/", 1)[0], refactor=False)
         self.create(path=path, value=value, refactor=False)
Esempio n. 27
0
  def mark_node_available(self, node_id, data=""):
    """Mark a node available."""

    path = self.AVAILABILITY_NODE + "/" + str(node_id)
    try:
      zookeeper.create(self.handle, path, data, [ZOO_OPEN_ACL_UNSAFE], zookeeper.EPHEMERAL)
      self.logger.info("Node %d: %s is now available" % (node_id, data))
    except zookeeper.NodeExistsException:
      self.logger.warn("%s already exists" % path)
Esempio n. 28
0
def set(i):
    nodeinfo = '/node%d' % i
    datainfo = 'datajfajdfkjaslkjdfajsfiwerkjkjelkjkljafd%d' % i
    zookeeper.create(zk, nodeinfo, datainfo, [{
        "perms": 0x1f,
        "scheme": "world",
        "id": "anyone"
    }], 0)
    print 'insert  number %d is ok! ' % i
Esempio n. 29
0
    def write(self, path, contents, ephemeral=False, exclusive=False):
        """ 
        Writes the contents to the path in zookeeper. It will create the path in
        zookeeper if it does not already exist.

        This method will return True if the value is written, False otherwise.
        (The value will not be written if the exclusive is True and the node
        already exists.)
        """
        partial_path = ''

        # We start from the second element because we do not want to inclued
        # the initial empty string before the first "/" because all paths begin
        # with "/". We also don't want to include the final node because that
        # is dealt with later.

        for path_part in path.split("/")[1:-1]:
            partial_path = partial_path + "/" + path_part
            if not(zookeeper.exists(self.handle, partial_path)):
                try:
                    zookeeper.create(self.handle, partial_path, '', [self.acl], 0)
                except zookeeper.NodeExistsException:
                    pass

        exists = zookeeper.exists(self.handle, path)

        # Don't create it if we're exclusive.
        if exists and exclusive:
            return False

        # We make sure that we have the creation flags for ephemeral nodes,
        # otherwise they could be associated with previous connections that
        # have not yet timed out.
        if ephemeral and exists:
            try:
                zookeeper.delete(self.handle, path)
            except zookeeper.NoNodeException:
                pass
            exists = False

        if exists:
            zookeeper.set(self.handle, path, contents)
            return True
        else:
            flags = (ephemeral and zookeeper.EPHEMERAL or 0)
            try:
                zookeeper.create(self.handle, path, contents, [self.acl], flags)
                return True
            except zookeeper.NodeExistsException:
                if not(exclusive):
                    # Woah, something happened between the top and here.
                    # We just restart and retry the whole routine.
                    self.write(path, contents, ephemeral=ephemeral)
                    return True
                else:
                    return False
Esempio n. 30
0
  def __init__(self, semaphorename, hostname, port, initial_value = 0):
    ZooKeeperBase.__init__(self, hostname, port)

    self.semaphorename = "/" + semaphorename
    try:
      zookeeper.create(self.handle,self.semaphorename,"semaphore top level", [ZOO_OPEN_ACL_UNSAFE],0)
      self.signal(initial_value)
      print "Created new semaphore, OK"
    except zookeeper.NodeExistsException:
      print "Semaphore Already Exists"
Esempio n. 31
0
  def __init__(self,queuename, hostname, port, is_producer=False):
    ZooKeeperBase.__init__(self, hostname, port)
    self.queuename = "/" + queuename

    if is_producer:
      try:
        zookeeper.create(self.handle,self.queuename,"queue top level", [ZOO_OPEN_ACL_UNSAFE],0)
        print "Created new Queue, OK"
      except zookeeper.NodeExistsException:
        print "Queue Already Exists"
Esempio n. 32
0
  def register_node(self, node_id, host, port=0):
    """Register a node to the cluster."""

    path = self.MEMBERSHIP_NODE + "/" + str(node_id)
    if port <= 0:
      port = self.default_port
    try:
      data = host + ":" + str(port)
      zookeeper.create(self.handle, path, data, [ZOO_OPEN_ACL_UNSAFE], 0)
      self.logger.info("Node %d: %s is registered" % (node_id, data))
    except zookeeper.NodeExistsException:
      self.logger.warn("%s already exists" % path)
Esempio n. 33
0
File: zk.py Progetto: ialzuru/ncfs
 def createRootNode(self, root):
     #Create root node recursively from back to front to back
     if root == '/': return
     while True:
         try:
             root = root[:-1]
             zookeeper.create(self.zh, root, '', [ZOO_OPEN_ACL_UNSAFE], 0)
             return
         except zookeeper.NodeExistsException:
             return
         except zookeeper.NoNodeException:
             self.createRootNode(root.rsplit('/', 1)[0] + '/')
Esempio n. 34
0
def create_zookeeper_without_delete_children(zk, path, data):
    try:
        if zookeeper.exists(zk, path):
            return
        zookeeper.create(zk, path, data, [{
            "perms": 15,
            "scheme": "world",
            "id": "anyone"
        }], 0)
    except zookeeper.NodeExistsException, e:
        err_str = "path %s exist\n" % path
        handle_output(False, err_str)
Esempio n. 35
0
 def createRootNode(self, root):
     #Create root node recursively from back to front to back
     if root == '/': return
     while True:
         try:
             root = root[:-1]
             zookeeper.create(self.zh, root, '', [ZOO_OPEN_ACL_UNSAFE], 0)
             return
         except zookeeper.NodeExistsException:
             return
         except zookeeper.NoNodeException:
             self.createRootNode(root.rsplit('/', 1)[0] + '/')
Esempio n. 36
0
 def _action_generator(self, parsed_args):
     for zh, group in self.iterate_groups(parsed_args):
         for key, first, last in self._list_elections(zh, group):
             tail = str(1+int(last)).rjust(10, '0')
             suffix = key + '-' + tail
             path = group + '/' + suffix
             try:
                 zookeeper.create(zh, path, parsed_args.value,
                                  _acl_openbar, 0)
                 yield group, suffix, "OK"
             except Exception as ex:
                 yield group, suffix, str(ex)
Esempio n. 37
0
 def test_empty_node(self):
     """
     Test for a bug when instead of empty string we can get
     random data from buffer malloc'ed to hold node contents.
     See ZOOKEEPER-1906 for details
     """
     NODE_PATH = "/zk-python-test-empty-node"
     self.ensureDeleted(NODE_PATH)
     zookeeper.create(self.handle, NODE_PATH, "",
                      [{"perms":0x1f, "scheme":"world", "id" :"anyone"}])
     (data,stat) = zookeeper.get(self.handle, NODE_PATH, None)
     self.assertEqual(data, "", "Data is not empty as expected: " + data)
Esempio n. 38
0
def node_add():
    basepath = request.args.get('path', '')

    if request.method == 'POST':
        path = request.form.get('path')
        data = request.form.get('data')
        print request.form
        print request.values
        verify = True

        if len(path) == 0:
            verify = False
            flash('"Node Path" not allow empty')
        elif path[0] == '/':
            verify = False
            flash('"Node Path" can not start with "/"')
        else:
            path = CONFIG_NODE / basepath / path
            if zookeeper.exists(zk, path):
                verify = False
                flash('Path already exists:"%s"' % (path))

        if verify:
            nodepath = FilePath('/')
            try:
                for node in path.split('/'):
                    nodepath = nodepath / node

                    if not zookeeper.exists(zk, nodepath):
                        if nodepath != path:
                            zookeeper.create(zk, nodepath, '',
                                             [{
                                                 "perms": zookeeper.PERM_ALL,
                                                 "scheme": 'world',
                                                 "id": 'anyone'
                                             }], 0)
                        else:
                            zookeeper.create(zk, nodepath, data,
                                             [{
                                                 "perms": zookeeper.PERM_ALL,
                                                 "scheme": 'world',
                                                 "id": 'anyone'
                                             }], 0)

                return redirect('/node')
            except BaseException, e:
                flash(e.message)

        formdata = {
            'path': request.form.get('path') if path != '' else '',
            'data': data
        }
Esempio n. 39
0
    def _write(self, path, contents, ephemeral, exclusive, sequential,
               mustexist):
        # We start from the second element because we do not want to inclued
        # the initial empty string before the first "/" because all paths begin
        # with "/". We also don't want to include the final node because that
        # is dealt with later.
        partial_path = ''
        for path_part in path.split("/")[1:-1]:
            partial_path = partial_path + "/" + path_part
            if not (zookeeper.exists(self.handle, partial_path)):
                try:
                    zookeeper.create(self.handle, partial_path, '', [self.acl],
                                     0)
                except zookeeper.NodeExistsException:
                    pass

        if sequential:
            exists = False
        else:
            exists = zookeeper.exists(self.handle, path)

        # Don't create it if we're exclusive.
        if exists and exclusive:
            return False

        # Check if we require the node to exist.
        if not exists and mustexist:
            return False

        # We make sure that we have the creation flags for ephemeral nodes,
        # otherwise they could be associated with previous connections that
        # have not yet timed out.
        if ephemeral and exists:
            try:
                zookeeper.delete(self.handle, path)
            except zookeeper.NoNodeException:
                pass
            exists = False

        if exists:
            zookeeper.set(self.handle, path, contents)
            return path
        else:
            flags = 0
            if ephemeral:
                flags = flags | zookeeper.EPHEMERAL
            if sequential:
                flags = flags | zookeeper.SEQUENCE

            # NOTE: We return the final path created.
            return zookeeper.create(self.handle, path, contents, [self.acl],
                                    flags)
Esempio n. 40
0
def before_request():
    if not zookeeper.exists(zk, ROOT_NODE):
        zookeeper.create(zk, ROOT_NODE, '', [{
            "perms": zookeeper.PERM_ALL,
            "scheme": "world",
            "id": "anyone"
        }], 0)
    elif not zookeeper.exists(zk, CONFIG_NODE):
        zookeeper.create(zk, CONFIG_NODE, '', [{
            "perms": zookeeper.PERM_ALL,
            "scheme": "world",
            "id": "anyone"
        }], 0)
Esempio n. 41
0
    def post(self):
	request_dict = self.request.arguments
        cluster_name  = (request_dict['cluster_name'])[0]
        zk=zookeeper.init(self.zk_connect(cluster_name))
	new_node = (request_dict['New_post_node'])[0]
	new_value = (request_dict['new_node_value'])[0]
	if zookeeper.exists(zk,new_node):
	    zookeeper.close(zk)
	    self.write("此节点存在")
	else:
	    zookeeper.create(zk,new_node,new_value,[{"perms":0x1f,"scheme":"world","id":"anyone"}],0)	
	    zookeeper.close(zk)
	    self.write("增加成功")
Esempio n. 42
0
def main():
    from optparse import OptionParser as OptionParser

    parser = OptionParser()
    parser.add_option(
        '-v',
        '--verbose',
        action="store_true",
        dest="flag_verbose",
        help='Triggers debugging traces')
    parser.add_option(
        '--slow',
        action="store_true",
        dest="SLOW",
        help='Only play with small batches to avoid timeouts on slow hosts.')
    parser.add_option(
        '--avoid',
        action="append",
        type="string",
        dest="AVOID_TYPES",
        help='Do not populate entries for the specified service types')

    (options, args) = parser.parse_args(sys.argv)

    # Logging configuration
    if options.flag_verbose:
        logging.basicConfig(
            format='%(asctime)s %(message)s',
            datefmt='%m/%d/%Y %I:%M:%S',
            level=logging.DEBUG)
    else:
        logging.basicConfig(
            format='%(asctime)s %(message)s',
            datefmt='%m/%d/%Y %I:%M:%S',
            level=logging.INFO)

    if len(args) < 2:
        raise ValueError("not enough CLI arguments")

    ns = args[1]
    cnxstr = load_namespace_conf(ns)['zookeeper']
    zookeeper.set_debug_level(zookeeper.LOG_LEVEL_INFO)
    zh = zookeeper.init(cnxstr)

    # synchronous creation of the root
    try:
        zookeeper.create(zh, PREFIX, '', acl_openbar, 0)
    except zookeeper.NodeExistsException:
        pass
    create_tree(zh, namespace_tree(ns, options), options)
    zookeeper.close(zh)
Esempio n. 43
0
    def _write(self, path, contents, ephemeral, exclusive, sequential, mustexist):
        # We start from the second element because we do not want to inclued
        # the initial empty string before the first "/" because all paths begin
        # with "/". We also don't want to include the final node because that
        # is dealt with later.
        partial_path = ''
        for path_part in path.split("/")[1:-1]:
            partial_path = partial_path + "/" + path_part
            if not(zookeeper.exists(self.handle, partial_path)):
                try:
                    zookeeper.create(self.handle, partial_path, '', [self.acl], 0)
                except zookeeper.NodeExistsException:
                    pass

        if sequential:
            exists = False
        else:
            exists = zookeeper.exists(self.handle, path)

        # Don't create it if we're exclusive.
        if exists and exclusive:
            return False

        # Check if we require the node to exist.
        if not exists and mustexist:
            return False

        # We make sure that we have the creation flags for ephemeral nodes,
        # otherwise they could be associated with previous connections that
        # have not yet timed out.
        if ephemeral and exists:
            try:
                zookeeper.delete(self.handle, path)
            except zookeeper.NoNodeException:
                pass
            exists = False

        if exists:
            zookeeper.set(self.handle, path, contents)
            return path
        else:
            flags = 0
            if ephemeral:
                flags = flags | zookeeper.EPHEMERAL
            if sequential:
                flags = flags | zookeeper.SEQUENCE

            # NOTE: We return the final path created.
            return zookeeper.create(self.handle, path, contents, [self.acl], flags)
Esempio n. 44
0
    def post(self):
        request_dict = self.request.arguments
        node_json_list = (eval(request_dict['node_json'][0])).values()
	cluster_name = (request_dict['cluster_name'])[0]
	current_node = (request_dict['current_node'])[0]
	if current_node == "/" :
	    current_node = ""
        zk=zookeeper.init(self.zk_connect(cluster_name))
	for item in node_json_list:
	    key = current_node + "/" + item[0]
	    value = item[1]
	    if not zookeeper.exists(zk, key):
	        zookeeper.create(zk, key , value , [{"perms":0x1f,"scheme":"world","id":"anyone"}],0)
	self.write("写入成功")
        zookeeper.close(zk)
Esempio n. 45
0
 def electLeader(self, handle, nodename, candidateName):
     '''
     This method elects the new leader from the list of candidates.
     The candidate with the lowest sequence Id gets to be the leader.
     '''
     candidates = zookeeper.get_children(handle, nodename)
     myName = candidateName.rsplit('/',1)[1]
     leader = sorted(candidates)[0]
     if ( myName == leader):
         try:
             zookeeper.create(handle, nodename+"/leader",myName,[ZOO_OPEN_ACL_UNSAFE],zookeeper.EPHEMERAL)
             log.info(" I won the elections, I am leader now :)")
             self.isLeader = True
         except zookeeper.NodeExistsException,e:
             log.info(e)
Esempio n. 46
0
 def _recursivelyCreatePath(self, path, intermediate_data=''):
     '''
     Should do a check to see if this dir already exists.
     '''
     current_path = ''
     for part in path.split('/'):
         if len(part) > 0:
             try:
                 current_path += '/' + part
                 zookeeper.create(self._z,
                                  current_path,
                                  intermediate_data,
                                  [ZOO_OPEN_ACL_UNSAFE])
             except zookeeper.NodeExistsException:
                 pass
Esempio n. 47
0
    def __init__(self, name):
        self.name = ROOT + '/' + name
        self.cv = threading.Condition()
        created = False
        while not created:
            try:
                zookeeper.create(handle, self.name, "0", [ZOO_OPEN_ACL_UNSAFE], 0)
                created=True
            except zookeeper.NodeExistsException:
                # S'ok.
                created = True
            except zookeeper.ConnectionLossException:
                continue

        self.znode = None
Esempio n. 48
0
    def test_sync_get_large_datanode(self):
        """
        Test that we can retrieve datanode sizes up to
        1Mb with default parameters (depends on ZooKeeper server).
        """

        data = "".join(["A" for x in xrange(1024 * 1023)])
        self.ensureDeleted("/zk-python-test-large-datanode")
        zookeeper.create(
            self.handle, "/zk-python-test-large-datanode", data, [{"perms": 0x1F, "scheme": "world", "id": "anyone"}]
        )
        (ret, stat) = zookeeper.get(self.handle, "/zk-python-test-large-datanode")
        self.assertEqual(len(ret), 1024 * 1023, "Should have got 1Mb returned, instead got %s" % len(ret))
        (ret, stat) = zookeeper.get(self.handle, "/zk-python-test-large-datanode", None, 500)
        self.assertEqual(len(ret), 500, "Should have got 500 bytes returned, instead got %s" % len(ret))
Esempio n. 49
0
 def test_empty_node(self):
     """
     Test for a bug when instead of empty string we can get
     random data from buffer malloc'ed to hold node contents.
     See ZOOKEEPER-1906 for details
     """
     NODE_PATH = "/zk-python-test-empty-node"
     self.ensureDeleted(NODE_PATH)
     zookeeper.create(self.handle, NODE_PATH, "", [{
         "perms": 0x1f,
         "scheme": "world",
         "id": "anyone"
     }])
     (data, stat) = zookeeper.get(self.handle, NODE_PATH, None)
     self.assertEqual(data, "", "Data is not empty as expected: " + data)
Esempio n. 50
0
def create_namespace_tree(zh, ns, logger, batch_size=2048, precheck=False):
    if precheck and _probe(zh, ns, logger):
        return 0, 0

    # Synchronous creation of the root, helps detecting a lot of
    # problems with the connection
    try:
        zookeeper.create(zh, _PREFIX, '', _acl_openbar, 0)
        logger.info("Created %s", _PREFIX)
    except zookeeper.NodeExistsException:
        logger.info("Already %s", _PREFIX)
        pass

    nodes = generate_namespace_tree(ns, [t for t, _, _ in _srvtypes])
    return _create_tree(zh, nodes, logger, int(batch_size))
Esempio n. 51
0
def main():
    from optparse import OptionParser as OptionParser

    parser = OptionParser()
    parser.add_option('-v',
                      '--verbose',
                      action="store_true",
                      dest="flag_verbose",
                      help='Triggers debugging traces')
    parser.add_option(
        '--slow',
        action="store_true",
        dest="SLOW",
        help='Only play with small batches to avoid timeouts on slow hosts.')
    parser.add_option(
        '--avoid',
        action="append",
        type="string",
        dest="AVOID_TYPES",
        help='Do not populate entries for the specified service types')

    (options, args) = parser.parse_args(sys.argv)

    # Logging configuration
    if options.flag_verbose:
        logging.basicConfig(format='%(asctime)s %(message)s',
                            datefmt='%m/%d/%Y %I:%M:%S',
                            level=logging.DEBUG)
    else:
        logging.basicConfig(format='%(asctime)s %(message)s',
                            datefmt='%m/%d/%Y %I:%M:%S',
                            level=logging.INFO)

    if len(args) < 2:
        raise ValueError("not enough CLI arguments")

    ns = args[1]
    cnxstr = load_namespace_conf(ns)['zookeeper']
    zookeeper.set_debug_level(zookeeper.LOG_LEVEL_INFO)
    zh = zookeeper.init(cnxstr)

    # synchronous creation of the root
    try:
        zookeeper.create(zh, PREFIX, '', acl_openbar, 0)
    except zookeeper.NodeExistsException:
        pass
    create_tree(zh, namespace_tree(ns, options), options)
    zookeeper.close(zh)
Esempio n. 52
0
    def testmanyhandles(self):
        """
        Test the ability of the module to support many handles.
        """
        # We'd like to do more, but currently the C client doesn't
        # work with > 83 handles (fails to create a pipe) on MacOS 10.5.8
        handles = [zookeeper.init(self.host) for i in xrange(63)]

        cv = threading.Condition()
        self.connected = False

        def connection_watcher(handle, type, state, path):
            cv.acquire()
            self.connected = True
            self.assertEqual(zookeeper.CONNECTED_STATE, state)
            self.handle = handle
            cv.notify()
            cv.release()

        cv.acquire()
        ret = zookeeper.init(self.host, connection_watcher)
        cv.wait(15.0)
        cv.release()
        self.assertEqual(self.connected, True,
                         "Connection timed out to " + self.host)

        for i, h in enumerate(handles):
            path = "/zkpython-test-handles-%s" % str(i)
            self.assertEqual(
                path,
                zookeeper.create(h, path, "", [ZOO_OPEN_ACL_UNSAFE],
                                 zookeeper.EPHEMERAL))

        self.assertEqual(
            True, all(zookeeper.close(h) == zookeeper.OK for h in handles))
Esempio n. 53
0
 def create(self, path, data="", flags=0, acl=[ZOO_OPEN_ACL_UNSAFE]):
     start = time.time()
     result = zookeeper.create(self.handle, path, data, acl, flags)
     if VERBOSE:
         print("Node %s created in %d ms"
               % (path, int((time.time() - start) * 1000)))
     return result
Esempio n. 54
0
 def create(self, path, data="", flags=0, acl=[ZOO_OPEN_ACL_UNSAFE]):
     start = time.time()
     result = zookeeper.create(self.handle, path, data, acl, flags)
     if verbose:
         print("Node %s created in %d ms" %
               (path, int((time.time() - start) * 1000)))
     return result
Esempio n. 55
0
    def test_async_delete(self):
        ZOO_OPEN_ACL_UNSAFE = {
            "perms": 0x1f,
            "scheme": "world",
            "id": "anyone"
        }
        self.assertEqual(self.connected, True)
        ret = zookeeper.create(self.handle, "/zk-python-adeletetest",
                               "nodecontents", [ZOO_OPEN_ACL_UNSAFE],
                               zookeeper.EPHEMERAL)
        self.assertEqual(ret, "/zk-python-adeletetest")

        self.cv = threading.Condition()
        self.callback_flag = False
        self.rc = -1

        def callback(handle, rc):
            self.cv.acquire()
            self.callback_flag = True
            self.cv.notify()
            self.rc = rc  # don't assert this here, as if the assertion fails, the test will block
            self.cv.release()

        self.cv.acquire()
        ret = zookeeper.adelete(self.handle, "/zk-python-adeletetest", -1,
                                callback)
        self.assertEqual(ret, zookeeper.OK, "adelete failed")
        while not self.callback_flag:
            self.cv.wait(15)
        self.cv.release()

        self.assertEqual(self.callback_flag, True, "adelete timed out")
        self.assertEqual(self.rc, zookeeper.OK)