Example #1
0
    def test_sync_get_large_datanode(self):
        """
        Test that we can retrieve datanode sizes up to
        1Mb with default parameters (depends on ZooKeeper server).
        """

        data = ''.join(["A" for x in range(1024 * 1023)])
        self.ensureDeleted("/zk-python-test-large-datanode")
        zookeeper.create(self.handle, "/zk-python-test-large-datanode", data,
                         [{
                             "perms": 0x1f,
                             "scheme": "world",
                             "id": "anyone"
                         }])
        (ret, stat) = zookeeper.get(self.handle,
                                    "/zk-python-test-large-datanode")
        self.assertEqual(
            len(ret), 1024 * 1023,
            "Should have got 1Mb returned, instead got %s" % len(ret))
        (ret, stat) = zookeeper.get(self.handle,
                                    "/zk-python-test-large-datanode", None,
                                    500)
        self.assertEqual(
            len(ret), 500,
            "Should have got 500 bytes returned, instead got %s" % len(ret))
Example #2
0
    def test_multiple_watchers(self):
        """
        Test whether multiple watchers are correctly called
        """
        cv1, cv2 = threading.Condition(), threading.Condition()

        def watcher1(*args, **kwargs):
            cv1.acquire()
            self.watcher1 = True
            cv1.notify()
            cv1.release()

        def watcher2(*args, **kwargs):
            cv2.acquire()
            self.watcher2 = True
            cv2.notify()
            cv2.release()

        nodename = "/zk-python-multiple-watcher-test"
        self.ensureCreated(nodename, "test")
        cv1.acquire()
        cv2.acquire()
        zookeeper.get(self.handle, nodename, watcher1)
        zookeeper.get(self.handle, nodename, watcher2)
        zookeeper.set(self.handle, nodename, "test")
        cv1.wait(15)
        cv2.wait(15)
        self.assertTrue(self.watcher1 and self.watcher2,
                        "One or more watchers failed to fire")
Example #3
0
 def test_sync_getset(self):
     self.assertEqual(self.connected, True, "Not connected!")
     (data,stat) = zookeeper.get(self.handle, "/zk-python-getsettest", None)
     self.assertEqual(data, "on", "Data is not 'on' as expected: " + data)
     ret = zookeeper.set(self.handle, "/zk-python-getsettest",
                         "off", stat["version"])
     (data,stat) = zookeeper.get(self.handle, "/zk-python-getsettest", None)
     self.assertEqual(data, "off", "Data is not 'off' as expected: " + data)        
Example #4
0
  def notifyFailedTransaction(self, app_id, txid):
    """ Notify failed transaction id.

    This method will add the transaction id into black list.
    After this call, the transaction becomes invalid.
    """
    self.__waitForConnect()
    self.checkTransaction(app_id, txid)
    print "notify failed transaction app:%s, txid:%d" % (app_id, txid)
    txpath = self.__getTransactionPath(app_id, txid)
    lockpath = None
    try:
      lockpath = zookeeper.get(self.handle, PATH_SEPARATOR.join([txpath, TX_LOCK_PATH]), None)[0]
    except zookeeper.NoNodeException:
      # there is no lock. it means there is no need to rollback.
      pass

    if lockpath:
      # add transacion id to black list.
      now = str(time.time())
      broot = self.__getBlacklistRootPath(app_id)
      if not zookeeper.exists(self.handle, broot):
        self.__forceCreatePath(broot)
      zookeeper.acreate(self.handle, PATH_SEPARATOR.join([broot, str(txid)]), now, ZOO_ACL_OPEN)
      # update local cache before notification
      if app_id in self.blacklistcache:
        with self.blacklistcv:
          self.blacklistcache[app_id].add(str(txid))
      # copy valid transaction id for each updated key into valid list.
      for child in zookeeper.get_children(self.handle, txpath):
        if re.match("^" + TX_UPDATEDKEY_PREFIX, child):
          value = zookeeper.get(self.handle, PATH_SEPARATOR.join([txpath, child]), None)[0]
          valuelist = value.split(PATH_SEPARATOR)
          key = urllib.unquote_plus(valuelist[0])
          vid = valuelist[1]
          vtxroot = self.__getValidTransactionRootPath(app_id)
          if not zookeeper.exists(self.handle, vtxroot):
            self.__forceCreatePath(vtxroot)
          vtxpath = self.__getValidTransactionPath(app_id, key)
          zookeeper.acreate(self.handle, vtxpath, vid, ZOO_ACL_OPEN)
      # release the lock
      try:
        zookeeper.adelete(self.handle, lockpath)
      except zookeeper.NoNodeException:
        # this should be retry.
        pass

    # just remove transaction node
    try:
      for item in zookeeper.get_children(self.handle, txpath):
        zookeeper.adelete(self.handle, PATH_SEPARATOR.join([txpath, item]))
      zookeeper.adelete(self.handle, txpath)
    except zookeeper.NoNodeException:
      # something wrong. next GC will take care of it.
      return False

    return True
Example #5
0
  def notifyFailedTransaction(self, app_id, txid):
    """ Notify failed transaction id.

    This method will add the transaction id into black list.
    After this call, the transaction becomes invalid.
    """
    self.__waitForConnect()
    self.checkTransaction(app_id, txid)
    print "notify failed transaction app:%s, txid:%d" % (app_id, txid)
    txpath = self.__getTransactionPath(app_id, txid)
    lockpath = None
    try:
      lockpath = zookeeper.get(self.handle, PATH_SEPARATOR.join([txpath, TX_LOCK_PATH]), None)[0]
    except zookeeper.NoNodeException:
      # there is no lock. it means there is no need to rollback.
      pass

    if lockpath:
      # add transacion id to black list.
      now = str(time.time())
      broot = self.__getBlacklistRootPath(app_id)
      if not zookeeper.exists(self.handle, broot):
        self.__forceCreatePath(broot)
      zookeeper.acreate(self.handle, PATH_SEPARATOR.join([broot, str(txid)]), now, ZOO_ACL_OPEN)
      # update local cache before notification
      if app_id in self.blacklistcache:
        with self.blacklistcv:
          self.blacklistcache[app_id].add(str(txid))
      # copy valid transaction id for each updated key into valid list.
      for child in zookeeper.get_children(self.handle, txpath):
        if re.match("^" + TX_UPDATEDKEY_PREFIX, child):
          value = zookeeper.get(self.handle, PATH_SEPARATOR.join([txpath, child]), None)[0]
          valuelist = value.split(PATH_SEPARATOR)
          key = urllib.unquote_plus(valuelist[0])
          vid = valuelist[1]
          vtxroot = self.__getValidTransactionRootPath(app_id)
          if not zookeeper.exists(self.handle, vtxroot):
            self.__forceCreatePath(vtxroot)
          vtxpath = self.__getValidTransactionPath(app_id, key)
          zookeeper.acreate(self.handle, vtxpath, vid, ZOO_ACL_OPEN)
      # release the lock
      try:
        zookeeper.adelete(self.handle, lockpath)
      except zookeeper.NoNodeException:
        # this should be retry.
        pass

    # just remove transaction node
    try:
      for item in zookeeper.get_children(self.handle, txpath):
        zookeeper.adelete(self.handle, PATH_SEPARATOR.join([txpath, item]))
      zookeeper.adelete(self.handle, txpath)
    except zookeeper.NoNodeException:
      # something wrong. next GC will take care of it.
      return False

    return True
Example #6
0
 def test_none_callback(self):
     """
     Test that no errors are raised when None is passed as a callback.
     """
     self.ensureCreated("/zk-python-none-callback-test", "test")
     # To do this we need to issue two operations, waiting on the second
     # to ensure that the first completes
     zookeeper.get(self.handle, "/zk-python-none-callback-test", None)
     (d, s) = zookeeper.get(self.handle, "/zk-python-none-callback-test")
     self.assertEqual(d, "test")
Example #7
0
def _probe(zh, ns, logger):
    logger.info("Probing for an existing namespace [%s]", ns)
    try:
        for t, _, _ in _srvtypes:
            _, _ = zookeeper.get(zh, _PREFIX_NS + '/' + ns + '/el/' + t)
        for path in get_meta0_paths(zh, ns):
            _, _ = zookeeper.get(zh, path)
        return True
    except Exception:
        return False
Example #8
0
    def _real_start(self, handler, type_, state, path):
        znode = zookeeper.get(handler, path, self._real_start)
        main_version = znode[1]["version"]
        LOGGER.info("main version is: %s" % main_version)

        tmp_cache = {}
        children = zookeeper.get_children(handler, path)
        for child in children:
            znode = zookeeper.get(handler, path + "/" + child)
            tmp_cache[child] = znode[0]
            LOGGER.info("key=%s, value=%s, version=%s" %
                        (child, znode[0], znode[1]["version"]))
        self._local_cache = tmp_cache
        self._main_version = main_version
Example #9
0
    def test_sync_get_large_datanode(self):
        """
        Test that we can retrieve datanode sizes up to
        1Mb with default parameters (depends on ZooKeeper server).
        """

        data = "".join(["A" for x in xrange(1024 * 1023)])
        self.ensureDeleted("/zk-python-test-large-datanode")
        zookeeper.create(
            self.handle, "/zk-python-test-large-datanode", data, [{"perms": 0x1F, "scheme": "world", "id": "anyone"}]
        )
        (ret, stat) = zookeeper.get(self.handle, "/zk-python-test-large-datanode")
        self.assertEqual(len(ret), 1024 * 1023, "Should have got 1Mb returned, instead got %s" % len(ret))
        (ret, stat) = zookeeper.get(self.handle, "/zk-python-test-large-datanode", None, 500)
        self.assertEqual(len(ret), 500, "Should have got 500 bytes returned, instead got %s" % len(ret))
Example #10
0
  def get_MR(self,id1):

    #EME: Suggested change (Requires zookeeper to be set up properly)
    (data, stat) = zookeeper.get(self.handle, "/root/MR/" + str(id1), True) 
    return data

    #EME: Old
    (data, stat) = zookeeper.get(self.handle, "/root/segment/" + str(id1), True)
    children = zookeeper.get_children(self.handle, "/root/segment", True)
    ask_these = ""
    for child in children:
      if(child != str(id1)):                      
        ask_these = ask_these + str(child) + ","
    ask_these = ask_these[0:len(ask_these)-1]     
    return [stat["mtime"], 1, str(ask_these)]     # EME MO should not be hardcoded, take from zookeeper instead
Example #11
0
  def __tryGC(self, app_id, app_path):
    try:
#      print "try to obtrain app %s lock" % app_id
      val = zookeeper.get(self.handle, PATH_SEPARATOR.join([app_path, GC_TIME_PATH]), None)[0]
      lasttime = float(val)
    except zookeeper.NoNodeException:
      lasttime = 0
    if lasttime + GC_INTERVAL < time.time():
      # try to gc this app.
      gc_path = PATH_SEPARATOR.join([app_path, GC_LOCK_PATH])
      try:
        now = str(time.time())
        zookeeper.create(self.handle, gc_path, now, ZOO_ACL_OPEN, zookeeper.EPHEMERAL)
        # succeed to obtain lock.
        # TODO: should we handle the timeout of gc also?
        try:
          self.__executeGC(app_id, app_path)
          # update lasttime when gc was succeeded.
          now = str(time.time())
          self.__updateNode(PATH_SEPARATOR.join([app_path, GC_TIME_PATH]), now)
        except Exception, e:
          print "warning: gc error %s" % e
          traceback.print_exc()
        zookeeper.delete(self.handle, gc_path, -1)
      except zookeeper.NodeExistsException:
        # fail to obtain lock. try next time.
        pass
Example #12
0
  def releaseLock(self, app_id, txid, key = None):
    """ Release acquired lock.

    You must call acquireLock() first.
    if the transaction is not valid or it is expired, this raises Exception.
    After the release lock, you could not use transaction ID again.
    If there is no lock, this method returns False.
    """
    self.__waitForConnect()
    self.checkTransaction(app_id, txid)
    txpath = self.__getTransactionPath(app_id, txid)

    has_lock = False
    try:
      lockpath = zookeeper.get(self.handle, PATH_SEPARATOR.join([txpath, TX_LOCK_PATH]), None)[0]
      if key:
        lockroot = self.__getLockRootPath(app_id, key)
        if not lockroot == lockpath:
          raise ZKTransactionException(ZKTransactionException.TYPE_DIFFERENT_ROOTKEY, "You can not specify different root entity for release.")
      zookeeper.adelete(self.handle, lockpath)
      has_lock = True
    except zookeeper.NoNodeException:
      # there is no lock.
      pass
    # If the transaction doesn't have active lock or not, we should delete it.
    # delete transaction node
    for child in zookeeper.get_children(self.handle, txpath):
      zookeeper.adelete(self.handle, PATH_SEPARATOR.join([txpath, child]))
    zookeeper.adelete(self.handle, txpath)

    return has_lock
Example #13
0
    def getReadLock(self, metadata):
        #Acquire read lock on file through ZooKeeper
        fn = metadata.filename
        print "Trying to acquire read lock for " + self.root + fn
        self.zpath = self.createLockNode(fn, 'r')
        myLock = self.zpath.rsplit('/', 1)[1]
        while True:
            children = sorted(zookeeper.get_children(self.zh, self.root + fn))
            lastLock = ''
            for child in children:
                try:
                    if child == myLock:
                        break
                    elif zookeeper.get(self.zh,
                                       self.root + fn + '/' + child)[0] == 'w':
                        lastLock = child
                except:
                    pass
            if lastLock != '':

                def watcher(handle, type, state, path):
                    self.cv2.acquire()
                    self.cv2.notify()
                    self.cv2.release()

                self.cv2.acquire()
                if zookeeper.exists(self.zh, self.root + fn + '/' + lastLock,
                                    watcher):
                    self.cv2.wait()
                self.cv2.release()
            else:
                break
        print "Acquired read lock for " + self.root + fn
Example #14
0
File: zk.py Project: ialzuru/ncfs
    def getReadLock(self, metadata):
        #Acquire read lock on file through ZooKeeper
        fn = metadata.filename
        print "Trying to acquire read lock for " + self.root + fn
        self.zpath = self.createLockNode(fn, 'r')
        myLock = self.zpath.rsplit('/',1)[1]
        while True:
            children = sorted(zookeeper.get_children(self.zh, self.root + fn))
            lastLock = ''
            for child in children:
                try:
                    if child == myLock:
                        break
                    elif zookeeper.get(self.zh, self.root+fn+'/'+child)[0] == 'w':
                        lastLock = child
                except:
                    pass
            if lastLock != '':
                def watcher (handle, type, state, path):
                    self.cv2.acquire()
                    self.cv2.notify()
                    self.cv2.release()

                self.cv2.acquire()
                if zookeeper.exists(self.zh, self.root+fn+'/'+lastLock, watcher):
                    self.cv2.wait()
                self.cv2.release()
            else:
                break
        print "Acquired read lock for " + self.root + fn
Example #15
0
  def __init__(self, service_name, connect_string, timeout=DEFAULT_TIMEOUT, default_port=6664):
    self.SERVICE_NODE = "/" + service_name
    self.AVAILABILITY_NODE = self.SERVICE_NODE + "/available"
    self.MEMBERSHIP_NODE = self.SERVICE_NODE + "/members"
    self.connected = False
    self.timeout = timeout
    self.default_port = default_port
    self.conn_cv = threading.Condition()
    self.conn_cv.acquire()
    self.handle = zookeeper.init(connect_string, self.connection_watcher, timeout)
    self.conn_cv.wait(timeout / 1000)
    self.conn_cv.release()
    self.watcher_lock = threading.Lock()
    self.logger = logging.getLogger("sincc")

    if not self.connected:
      raise SinClusterClientError("Unable to connect to %s" % connect_string)

    for path in [self.SERVICE_NODE, self.AVAILABILITY_NODE, self.MEMBERSHIP_NODE]:
      if not zookeeper.exists(self.handle, path):
        zookeeper.create(self.handle, path, "", [ZOO_OPEN_ACL_UNSAFE], 0)
    self.listeners = []
    # Start to watch both /members and /available
    zookeeper.get_children(self.handle, self.MEMBERSHIP_NODE, self.watcher)
    available = zookeeper.get_children(self.handle, self.AVAILABILITY_NODE, self.watcher)
    self.available_nodes = {}
    for node_id in available:
      self.available_nodes[int(node_id)] = Node(int(node_id),
                                                zookeeper.get(self.handle, self.AVAILABILITY_NODE + "/" + node_id)[0])
Example #16
0
 def handle_availability_changed(self):
   available = zookeeper.get_children(self.handle, self.AVAILABILITY_NODE, self.watcher)
   self.available_nodes.clear()
   for node_id in available:
     self.available_nodes[int(node_id)] = Node(int(node_id),
                                               zookeeper.get(self.handle, self.AVAILABILITY_NODE + "/" + node_id)[0])
   self.notify_all()
Example #17
0
 def queue_callback(self, zh, rc, data):
   if zookeeper.OK == rc:
     try:
       for child in sorted(data):
         action = self.actionNode + '/' + child
         workLog = self.actionNode + '/' + child + '/worklog'
         statusLog = self.statusNode + '/status-'
         """ Launch the task if the task has not been executed """
         if zookeeper.exists(zh, workLog, None) == None:
           self.launch(zh, workLog, action, statusLog)
         else:
           """ If task has been previous launched, check for partial execution """
           buffer = zookeeper.get(zh, workLog, 0)
           state = simplejson.loads(buffer[0])
           """ If task is incompleted in execution, launch again """
           if 'status' in state and state['status'] == 'STARTING':
             logger.info('Relaunch '+child)
             self.launch(zh, workLog, action, statusLog)
           else:
             """ If the task has been launched, and completed, update status queue """
             if zookeeper.exists(zh, statusLog, None) == None:
               logger.info('Update status.')
               self.update(zh, statusLog, state)
     except NoNodeException, err:
       """ Skip no node exception """
     except Exception, err:
       logger.exception(err)
Example #18
0
def list_problematic_nodes (zh, path, options):
    path = path.replace('//', '/')
    try:
        children = list(zookeeper.get_children(zh, path))
        if len(children) <= 0:
            return

        if options.ALONE and 1 == len(children):
            yield path + '/' + children[0]
            return

        if options.NUM > len(children):
            logging.info("SKIP only %d nodes in %s", len(children), path)
            return

        if options.SMART:
            children.sort()
            # check for services registered several times
            group = {}
            for n in children:
                n = path + '/' + n
                data, meta = tuple(zookeeper.get(zh, n))
                print repr(data), repr(meta)
                if data in group:
                    yield group[data]
                group[data] = n;
        else:
            # systematical removal
            for n in children:
                yield path + '/' + n

    except Exception as e:
        logging.warn("ERROR list %s: %s", path, e)
Example #19
0
 def unlock(self):
     try:
         (data, _) = zookeeper.get(self.handle, self.lockname, None)
         if data == self.uuid:
             zookeeper.delete(self.handle, self.lockname)
     except:
         self.log.exception('unlock')
Example #20
0
    def set(self, path, data='', previous_data=None):
        path = '%s%s' % (self.prefix, path)
        try:
            current_data, current_meta = zookeeper.get(self.handle, path, None)
        except zookeeper.NoNodeException:
            if not previous_data:
                try:
                    zookeeper.create(self.handle, path, data, [ZOO_OPEN_ACL_UNSAFE])
                    return True
                except:
                    self.log.exception('Failed to create a missing key %s', path)
                    return False
            else:
                return False
        except:
            self.log.exception('Failed to set key %s', path)
            return False

        version = None
        if previous_data:
            if current_data != previous_data:
                self.log.error('Previous data constraint failed')
                return False
            version = current_meta['version']

        try:
            if version is None:
                zookeeper.set(self.handle, path, data)
            else:
                zookeeper.set(self.handle, path, data, version)
        except:
            self.log.exception('Set failed')
            return False

        return True
Example #21
0
  def __init__(self,queuename, port, is_producer=False):
    self.connected = False
    self.queuename = "/" + queuename
    self.cv = threading.Condition()
    zookeeper.set_log_stream(open("/dev/null"))
    def watcher(handle,type,state,path):
      print "Connected"
      self.cv.acquire()
      self.connected = True
      self.cv.notify()
      self.cv.release()

    self.cv.acquire()
    self.handle = zookeeper.init("localhost:%d" % port, watcher, 10000)
    self.cv.wait(10.0)
    if not self.connected:
      print "Connection to ZooKeeper cluster timed out - is a server running on localhost:%d?" % port
      sys.exit()
    self.cv.release()
    if is_producer:
      while True:
        try:
          zookeeper.create(self.handle,self.queuename,"queue top level", [ZOO_OPEN_ACL_UNSAFE],0)
          print "Fila criada, OK"
          return
        except zookeeper.NodeExistsException:
          print "Tratorando filas existentes"
          while True:
            children = sorted(zookeeper.get_children(self.handle, self.queuename,None))
            if len(children) == 0:
              (data,stat) = zookeeper.get(self.handle, self.queuename, None)
              zookeeper.delete(self.handle, self.queuename, stat["version"])
              break
            for child in children:
              data = self.get_and_delete(self.queuename + "/" + child)
Example #22
0
 def _action_generator(self, parsed_args):
     for zh, group in self.iterate_groups(parsed_args):
         children = list(self._list_nodes(zh, group))
         for child in children:
             n = group + '/' + child
             value, meta = tuple(zookeeper.get(zh, n))
             yield group, child, meta['dataLength'], repr(value)
Example #23
0
    def try_lock(self):
        while True:
            try:
                zookeeper.create(self.handle, self.lockname, self.uuid,
                                 [ZOO_OPEN_ACL_UNSAFE], zookeeper.EPHEMERAL)
            except:
                self.log.debug('Failed to acquire lock (%r)' % self.lockname)
                return False

            try:
                (data, _) = zookeeper.get(self.handle, self.lockname, None)
                break
            except:
                self.log.exception(
                    'try_lock: create succeeded but get failed? (%r)' %
                    self.lockname)
                continue

        if data == self.uuid:
            self.log.debug('Lock acquired (%r)' % self.lockname)
            return True
        else:
            self.log.error(
                'try_lock: create succeeded but data is wrong? (%r)' %
                self.lockname)
            print "failed to acquire lock"
            return False
Example #24
0
    def _take_action(self, parsed_args):
        import zookeeper
        from oio.zk.client import get_meta0_paths, get_connected_handles
        self.logger.debug("Checking meta0 services")

        # TODO: tcp touch to the meta0 services

        # check they are registered in the ZK
        for zh in get_connected_handles(self.zookeeper()):
            for p in get_meta0_paths(zh, self.app.options.ns):
                try:
                    registered = set()
                    for n in zookeeper.get_children(zh.get(), p):
                        v, m = zookeeper.get(zh.get(), p + '/' + n)
                        registered.add(v)
                    known = set()
                    for t, i, p, s in self.filter_services(
                            self.catalog, 'meta0'):
                        known.add('%s:%d' % (i, p))
                    self.logger.info("meta0 known=%d zk_registered=%d",
                                     len(known), len(registered))
                    assert registered == known
                except Exception as ex:
                    self.logger.exception(
                        "Failed to list the meta0 services "
                        "from zookeeper: %s", ex)
                finally:
                    zh.close()

        yield ('OK', None)
Example #25
0
    def zookeeper_watch(self, zh, event, state, path):
        self.cond.acquire()
        try:
            if event == zookeeper.CHILD_EVENT:
                fns = self.child_watches.get(path, None)
            elif event == zookeeper.CHANGED_EVENT:
                fns = self.content_watches.get(path, None)
            else:
                fns = None
        finally:
            self.cond.release()

        result = None
        try:
            if fns and event == zookeeper.CHILD_EVENT:
                result = zookeeper.get_children(self.handle, path,
                                                self.zookeeper_watch)
            elif fns and event == zookeeper.CHANGED_EVENT:
                result, _ = zookeeper.get(self.handle, path,
                                          self.zookeeper_watch)
        except zookeeper.NoNodeException:
            pass

        if result != None and fns != None:
            for fn in fns:
                # Don't allow an individual watch firing an exception to
                # prevent all other watches from being fired. Just log an
                # error message and moved on to the next callback.
                try:
                    fn(result)
                except Exception:
                    logging.exception("Error executing watch for %s.", path)
Example #26
0
  def launch(self, zh, workLogNode, actionNode, statusNode):
    state = {}
    data = zookeeper.get(zh, actionNode, 0)
    jsonp = simplejson.loads(data[0])
    state['cmdPath'] = jsonp['cmdPath']
    state['actionPath'] = actionNode
    state['actionId'] = jsonp['actionId']
    state['host'] = self.znode
    state['status']='STARTING'
    self.update(zh, workLogNode, state)

    logger.info("Launch: "+simplejson.dumps(jsonp))
    dispatcher = Runner()
    try:
      result = dispatcher.run(jsonp)
      logger.info("Result: "+simplejson.dumps(result))
      if "exit_code" in result and result['exit_code']==0:
        state['status']='SUCCEEDED'
      else:
        state['status']='FAILED'
    except:
      logger.exception('Execution error: '+actionNode)
      state['status']='FAILED'
    self.update(zh, workLogNode, state)
    self.enqueue(zh, statusNode, state)
Example #27
0
    def zookeeper_watch(self, zh, event, state, path):
        self.cond.acquire()
        try:
            result = None
            if event == zookeeper.CHILD_EVENT:
                fns = self.child_watches.get(path, None)
                if fns:
                    try:
                        result = zookeeper.get_children(self.handle, path, self.zookeeper_watch)
                    except zookeeper.NoNodeException:
                        result = None
            elif event == zookeeper.CHANGED_EVENT:
                fns = self.content_watches.get(path, None)
                if fns:
                    try:
                        result, _ = zookeeper.get(self.handle, path, self.zookeeper_watch)
                    except zookeeper.NoNodeException:
                        result = None
            else:
                return

            if result != None and fns != None:
                for fn in fns:
                    # Don't allow an individual watch firing an exception to
                    # prevent all other watches from being fired. Just log an
                    # error message and moved on to the next callback.
                    try:
                        fn(result)
                    except Exception:
                        logging.exception("Error executing watch for %s.", path)
        finally:
            self.cond.release()
Example #28
0
 def unlock(self):
     try:
         (data, _) = zookeeper.get(self.handle, self.lockname, None)
         if data == self.uuid:
             zookeeper.delete(self.handle, self.lockname)
     except:
         self.log.exception('unlock')
Example #29
0
  def notify_failed_transaction(self, app_id, txid):
    """ Marks the given transaction as failed, invalidating its use by future
    callers.

    Args:
      app_id: The application ID whose transaction we wish to invalidate.
      txid: An int representing the transaction ID we wish to invalidate.
    Returns:
      True if the transaction was invalidated, False otherwise.
    """
    logging.warning("Notify failed transaction app: {0}, txid: {1}"\
      .format(app_id, str(txid)))

    self.wait_for_connect()
    lockpath = None
    lock_list = []

    txpath = self.get_transaction_path(app_id, txid)

    try:
      lockpath = zookeeper.get(self.handle, 
          PATH_SEPARATOR.join([txpath, TX_LOCK_PATH]), None)[0]
      lock_list = lockpath.split(LOCK_LIST_SEPARATOR)
    except zookeeper.NoNodeException:
      # There is no need to rollback because there is no lock.
      pass
    except zookeeper.ZooKeeperException, zoo_exception:
      logging.error("Exception seen when notifying a failed transaction {0}"\
        .format(str(zoo_exception)))
      return
Example #30
0
 def queue_callback(self, zh, rc, data):
   if zookeeper.OK == rc:
     try:
       for child in sorted(data):
         action = self.actionNode + '/' + child
         workLog = self.actionNode + '/' + child + '/worklog'
         statusLog = self.statusNode + '/status-'
         """ Launch the task if the task has not been executed """
         if zookeeper.exists(zh, workLog, None) == None:
           self.launch(zh, workLog, action, statusLog)
         else:
           """ If task has been previous launched, check for partial execution """
           buffer = zookeeper.get(zh, workLog, 0)
           state = simplejson.loads(buffer[0])
           """ If task is incompleted in execution, launch again """
           if 'status' in state and state['status'] == 'STARTING':
             logger.info('Relaunch '+child)
             self.launch(zh, workLog, action, statusLog)
           else:
             """ If the task has been launched, and completed, update status queue """
             if zookeeper.exists(zh, statusLog, None) == None:
               logger.info('Update status.')
               self.update(zh, statusLog, state)
     except NoNodeException, err:
       """ Skip no node exception """
     except Exception, err:
       logger.exception(err)
Example #31
0
  def launch(self, zh, workLogNode, actionNode, statusNode):
    state = {}
    data = zookeeper.get(zh, actionNode, 0)
    jsonp = simplejson.loads(data[0])
    state['cmdPath'] = jsonp['cmdPath']
    state['actionPath'] = actionNode
    state['actionId'] = jsonp['actionId']
    state['host'] = self.znode
    state['status']='STARTING'
    self.update(zh, workLogNode, state)

    logger.info("Launch: "+simplejson.dumps(jsonp))
    dispatcher = Runner()
    try:
      result = dispatcher.run(jsonp)
      logger.info("Result: "+simplejson.dumps(result))
      if "exit_code" in result and result['exit_code']==0:
        state['status']='SUCCEEDED'
      else:
        state['status']='FAILED'
    except:
      logger.exception('Execution error: '+actionNode)
      state['status']='FAILED'
    self.update(zh, workLogNode, state)
    self.enqueue(zh, statusNode, state)
Example #32
0
    def watcher(self, h_zk, w_type, w_stat, w_path):
        '''
        监听 agent/slaver 发送消息格式为 {agent:[state,value]} value只有当stat为change时才会有
        watch回调
        h_zk:就是我们创建连接之后的返回值,我试了下,发现好像指的是连接的一个索引值,以0开始
        w_type:事件类型,-1表示没有事件发生,1表示创建节点,2表示节点删除,3表示节点数据被改变,4表示子节点发生变化
        w_state:客户端的状态,0:断开连接状态,3:正常连接的状态,4认证失败,-112:过期啦
        w_path:这个状态就不用解释了,znode的路径
        '''

        log = "%s watche node change:t:%d s:%d p:%s" % (self.server_node_name,
                                                        w_type, w_stat, w_path)
        self.loger.info(log)
        if w_type == 3:
            new_vaule = zookeeper.get(h_zk, w_path, self.watcher)
            self.agent_stat = new_vaule
            msg_txt = '{"%s":["change","%s"]}' % (self.agent_name, new_vaule)
            cur_role = getLocalhostRole()
            log = 'cur role %s' % cur_role
            self.loger.info(log)
            if cur_role and cur_role != 'follower':
                self.msg_queue.put(msg_txt)  # 交由上层AgentMgr.process处理
            else:
                self.loger.info(msg_txt)

        elif w_type == 2:
            msg_txt = '{"%s":["delete"]}' % (self.agent_name)
            cur_role = getLocalhostRole()
            log = 'cur role %s' % cur_role
            self.loger.info(log)
            if cur_role and cur_role != 'follower':
                self.msg_queue.put(msg_txt)  # 交由上层AgentMgr.process处理
            else:
                self.loger.info(msg_txt)
Example #33
0
  def __tryGC(self, app_id, app_path):
    try:
#      print "try to obtrain app %s lock" % app_id
      val = zookeeper.get(self.handle, PATH_SEPARATOR.join([app_path, GC_TIME_PATH]), None)[0]
      lasttime = float(val)
    except zookeeper.NoNodeException:
      lasttime = 0
    if lasttime + GC_INTERVAL < time.time():
      # try to gc this app.
      gc_path = PATH_SEPARATOR.join([app_path, GC_LOCK_PATH])
      try:
        now = str(time.time())
        zookeeper.create(self.handle, gc_path, now, ZOO_ACL_OPEN, zookeeper.EPHEMERAL)
        # succeed to obtain lock.
        # TODO: should we handle the timeout of gc also?
        try:
          self.__executeGC(app_id, app_path)
          # update lasttime when gc was succeeded.
          now = str(time.time())
          self.__updateNode(PATH_SEPARATOR.join([app_path, GC_TIME_PATH]), now)
        except Exception, e:
          print "warning: gc error %s" % e
          traceback.print_exc()
        zookeeper.delete(self.handle, gc_path, -1)
      except zookeeper.NodeExistsException:
        # fail to obtain lock. try next time.
        pass
Example #34
0
def watcher(zk, type, state, nodepath):
    log('watcher', {
        'type': type,
        'state': state,
        'nodepath': nodepath
    }, 'DEBUG')

    if type == zookeeper.SESSION_EVENT:
        zookeeper.set_watcher(zk, watcher)
        if state == zookeeper.CONNECTED_STATE:
            for k in G.project:
                if zookeeper.exists(zk, k, watcher):
                    # 启动时马上就通知一次,防止在断线过程中出现了数据更改,而服务又不知道
                    on_zookeeper_node_change(zookeeper.get(zk, k), k)

            if zookeeper.exists(zk, ROOT_NODE, watcher):
                config = zookeeper.get(zk, ROOT_NODE)[0]
                if config != '':
                    G.config = json.loads(config)
                    if not isinstance(G.config, dict):
                        raise TypeError()
                else:
                    G.config = {}

                log('config', G.config, 'DEBUG')
    elif type == zookeeper.CREATED_EVENT or type == zookeeper.CHANGED_EVENT:
        nodevalue = zookeeper.get(zk, nodepath, watcher)
        if nodepath == ROOT_NODE:
            if nodevalue[0] != '':
                G.config = json.loads(nodevalue[0])
                if not isinstance(G.config, dict):
                    raise TypeError()
            else:
                G.config = {}

            log('config', G.config, 'DEBUG')

            # restart process
            if G.config.has_key('restart') and G.config['restart'] in [
                    True, G.name
            ]:
                G.queue.put((0, {'type': 'restart'}))
        else:
            on_zookeeper_node_change(nodevalue, nodepath)
    elif type == zookeeper.DELETED_EVENT:
        zookeeper.exists(zk, nodepath, watcher)  # 期待再次创建节点
Example #35
0
def gets(path):
    init_client()
    global handle
    try:
        (data, stat) = zookeeper.get(handle, path, None)
        return (data, stat)
    except zookeeper.NoNodeException:
        return None
Example #36
0
 def get(self, path):
     path = '%s%s' % (self.prefix, path)
     try:
         current_data, _current_meta = zookeeper.get(self.handle, path, None)
     except:
         self.log.exception('Failed to fetch value')
         return None
     return current_data
Example #37
0
def get(path):
    init_client()
    global handle
    try:
        (data,stat) = zookeeper.get(handle,path,None)
        return data
    except zookeeper.NoNodeException:
        return None
Example #38
0
def print_recursive(handle, path):
  try:
    children = zookeeper.get_children(handle, path)
    for child in children:
      print_recursive(handle, PATH_SEPARATOR.join([path, child]))
    value = zookeeper.get(handle, path)[0]
    print "{0} = {1}".format(path, value)
  except zookeeper.NoNodeException:
    pass
Example #39
0
    def read(self, path, default=None):
        """
        Returns the conents in the path. default is returned if the path does not exists.
        """
        value = default
        if zookeeper.exists(self.handle, path):
            value, timeinfo = zookeeper.get(self.handle, path)

        return value
Example #40
0
 def get_and_delete(self,node):
     try:
         (data,stat) = zookeeper.get(self.handle, node, None)
         zookeeper.delete(self.handle, node, stat["version"])
         return data
     except IOError, e:
         if e.message == zookeeper.zerror(zookeeper.NONODE):
             return None 
         raise e
Example #41
0
def get_znode(zk, path):
    for i in range(3):
        try:
            result = zookeeper.get(zk,path)
            return result
        except zookeeper.NoNodeException:
            return False
        except zookeeper.ConnectionLossException:
            continue
Example #42
0
    def post(self):
	request_dict = self.request.arguments
	node_id = (request_dict['choose_node'])[0]
        cluster_name  = (request_dict['cluster_name'])[0]
        zk=zookeeper.init(self.zk_connect(cluster_name))
	_value = (zookeeper.get(zk,node_id))[0]
	zookeeper.close(zk)
	self.write(_value)
	logging.info('%s查看了集群%s的节点:%s'%(self.get_current_user(),cluster_name,node_id))
Example #43
0
	def put_ng(self, ng, val):
		# create the new znode that holds the graph data
		payload = { 'graph' : ng , 'data' : val }
		znode_id = zookeeper.create(self.handle, '/ng', json.dumps(payload), [ZOO_OPEN_ACL_UNSAFE], zookeeper.SEQUENCE)
		# now update mNG2znode look-up table:
		(self.NG2znode, stat) = zookeeper.get(self.handle, '/look-up', None)
		self.NG2znode =  json.loads(str(self.NG2znode))
		self.NG2znode[ng] = znode_id
		zookeeper.set(self.handle, '/look-up', json.dumps(self.NG2znode))
Example #44
0
 def _get_provider_data(self, name):
     provider_path = '{}/providers/{}'.format(self._get_service_path(), name)
     try:
         data, stat = zookeeper.get(self._zh, provider_path, self._get_provider_watcher(name))
     except zookeeper.ZooKeeperException as e:
         self._logger.error("get data failed, exception - {}".format(e))
         return {}, {}
     
     return json.loads(data), stat
Example #45
0
 def get_and_delete(self, node):
     try:
         (data, stat) = zookeeper.get(self.handle, node, None)
         zookeeper.delete(self.handle, node, stat["version"])
         return data
     except IOError, e:
         if e.message == zookeeper.zerror(zookeeper.NONODE):
             return None
         raise e
Example #46
0
File: sincc.py Project: alanma/sin
 def handle_membership_changed(self):
   members = zookeeper.get_children(self.handle, self.MEMBERSHIP_NODE, self.watcher)
   # No need to watch /available here.
   available = zookeeper.get_children(self.handle, self.AVAILABILITY_NODE)
   self.available_nodes.clear()
   for node_id in members:
     if node_id in available:
       self.available_nodes[int(node_id)] = Node(int(node_id),
                                                 zookeeper.get(self.handle, self.AVAILABILITY_NODE + "/" + node_id)[0])
   self.notify_all()
Example #47
0
 def dumpTree(self, path):
   self.__waitForConnect()
   try:
     value = zookeeper.get(self.handle, path, None)[0]
     print "%s = \"%s\"" % (path, value)
     children = zookeeper.get_children(self.handle, path)
     for child in children:
       self.dumpTree(PATH_SEPARATOR.join([path, child]))
   except zookeeper.NoNodeException:
     print "%s is not exist." % path
Example #48
0
 def dumpTree(self, path):
   self.__waitForConnect()
   try:
     value = zookeeper.get(self.handle, path, None)[0]
     print "%s = \"%s\"" % (path, value)
     children = zookeeper.get_children(self.handle, path)
     for child in children:
       self.dumpTree(PATH_SEPARATOR.join([path, child]))
   except zookeeper.NoNodeException:
     print "%s is not exist." % path
Example #49
0
 def get_node_tree(node_key):
     if node_key == "/":
         for node in zookeeper.get_children(zk,node_key):
              key =  "/" + node
              if (zookeeper.get(zk,key)[1])['numChildren'] > 0:
                   get_node_tree(key)
                   print key
              else:
                   print key
     else:
         for node in zookeeper.get_children(zk,node_key):
              key =  node_key + "/" + node
              if (zookeeper.get(zk,key)[1])['numChildren'] > 0:
                   get_node_tree(key)
                   data.append(key)
              else:
                   data.append(key)
 
     return data
Example #50
0
    def getARset(self):

        children = zookeeper.get_children(self.handle, "/root/segment", True)
        #for i in range(1, self.nbr_agents + 1):
        for child in children:
            (data, stat) = zookeeper.get(self.handle,
                                         "/root/segment/" + str(child), True)

            #zookeeper keeps things as string, convert to a list
            self.agent_registry[child] = eval(data)
Example #51
0
  def mark_node_unavailable(self, node_id):
    """Mark a node unavailable."""

    path = self.AVAILABILITY_NODE + "/" + str(node_id)
    try:
      data = zookeeper.get(self.handle, path)[0]
      zookeeper.delete(self.handle, path)
      self.logger.info("Node %d: %s is now unavailable" % (node_id, data))
    except zookeeper.NoNodeException:
      self.logger.warn("Tried to mark node %s unavailable, but it did not exist" % path)
Example #52
0
    def post(self):
	request_dict = self.request.arguments
	node_tree = (request_dict['node_tree'])[0]
        cluster_name  = (request_dict['cluster_name'])[0]
        zk=zookeeper.init(self.zk_connect(cluster_name))
        _value = (zookeeper.get(zk,node_tree))[0]
	create_time = time.time()
	table = ZdSnapshot(cluster_name= cluster_name ,path=node_tree , data=_value ,create_time = self.GetNowTime())
	table.save()
        zookeeper.close(zk)
	self.write("生成快照成功!!!!!")
Example #53
0
 def get_status_of(self, server=None):
     server = server or self.nodename
     status = dict(name=server)
     try:
         node = zk.get(self.zh, self.NODE_SERVERS+'/'+server+'/alive')
         status['alive'] = node[1]
     except zk.NoNodeException:
         pass
     try:
         jobspath = self.NODE_SERVERS+'/'+server+'/jobs'
         jobs = zk.get_children(self.zh, jobspath)
         status['jobs'] = []
         for j in jobs:
             jobpath = jobspath+'/'+j
             nodevals = zk.get(self.zh, jobpath)
             mtime = nodevals[1]['mtime']
             status['jobs'].append(dict(name=j, ts=mtime/1000.0))
     except zk.NoNodeException:
         status['jobs'] = null;
     return status
Example #54
0
 def test_sync_getset(self):
     self.assertEqual(self.connected, True, "Not connected!")
     (data,stat) = zookeeper.get(self.handle, "/zk-python-getsettest", None)
     self.assertEqual(data, "on", "Data is not 'on' as expected: " + data)
     ret = zookeeper.set(self.handle, "/zk-python-getsettest",
                         "off", stat["version"])
     (data,stat) = zookeeper.get(self.handle, "/zk-python-getsettest", None)
     self.assertEqual(data, "off", "Data is not 'off' as expected: " + data)
     self.assertRaises(zookeeper.BadVersionException,
                       zookeeper.set,
                       self.handle,
                       "/zk-python-getsettest",
                       "test",
                       stat["version"]+1)
     stat2 = zookeeper.set2(self.handle, "/zk-python-getsettest",
                            "set2", stat["version"])
     self.assertNotEqual(stat2, None, "set2 call failed, return should not be None")
     self.assertEqual(stat2["numChildren"], 0,
                      "set2 call failed, numChildren not 0 in set2 call")
     (data,stat) = zookeeper.get(self.handle, "/zk-python-getsettest", None)
     self.assertEqual(data, "set2", "Data is not 'set2' as expected: " + data)