示例#1
0
    def get_vBuckets_info(master):
        """
        return state and count items for all vbuckets for each node
        format: dict: {u'1node_ip1': {'vb_79': ['replica', '0'], 'vb_78': ['active', '0']..}, u'1node_ip1':....}
        """
        rest = RestConnection(master)
        port = rest.get_nodes_self().memcached
        nodes = rest.node_statuses()
        _nodes_stats= {}
        for node in nodes:
            stat={}
            buckets = []
            _server = {"ip": node.ip, "port": node.port, "username": master.rest_username,
                           "password": master.rest_password}
            try:
                buckets = rest.get_buckets()
                mc = MemcachedClient(node.ip, port)
                stat_hash = mc.stats("hash")
            except Exception:
                if not buckets:
                    log.error("There are not any buckets in {0}:{1} node".format(node.ip, node.port))
                else:
                    log.error("Impossible to get vBucket's information for {0}:{1} node".format(node.ip, node.port))
                    _nodes_stats[node.ip+":"+str(node.port)]
                continue
            mc.close()
            vb_names=[key[:key.index(":")] for key in stat_hash.keys()]

            for name in vb_names:
                stat[name]=[stat_hash[name + ":state"], stat_hash[name+":counted"]]
            _nodes_stats[node.ip+":"+str(port)] = stat
        log.info(_nodes_stats)
        return _nodes_stats
示例#2
0
    def direct_client(server, bucket, timeout=30):
        log = logger.Logger.get_logger()
        rest = RestConnection(server)
        node = None
        try:
            node = rest.get_nodes_self()
        except ValueError as e:
            log.info("could not connect to server {0}, will try scanning all nodes".format(server))
        if not node:
            nodes = rest.get_nodes()
            for n in nodes:
                if n.ip == server.ip and n.port == server.port:
                    node = n

        if isinstance(server, dict):
            log.info("dict:{0}".format(server))
            log.info("creating direct client {0}:{1} {2}".format(server["ip"], node.memcached, bucket))
        else:
            log.info("creating direct client {0}:{1} {2}".format(server.ip, node.memcached, bucket))
        RestHelper(rest).vbucket_map_ready(bucket, 60)
        vBuckets = RestConnection(server).get_vbuckets(bucket)
        if isinstance(server, dict):
            client = MemcachedClient(server["ip"], node.memcached, timeout=timeout)
        else:
            client = MemcachedClient(server.ip, node.memcached, timeout=timeout)
        client.vbucket_count = len(vBuckets)
        bucket_info = rest.get_bucket(bucket)
        # todo raise exception for not bucket_info
        client.sasl_auth_plain(bucket_info.name.encode("ascii"), bucket_info.saslPassword.encode("ascii"))
        return client
示例#3
0
    def get_vBuckets_info(master):
        """
        return state and count items for all vbuckets for each node
        format: dict: {u'1node_ip1': {'vb_79': ['replica', '0'], 'vb_78': ['active', '0']..}, u'1node_ip1':....}
        """
        rest = RestConnection(master)
        port = rest.get_nodes_self().memcached
        nodes = rest.node_statuses()
        _nodes_stats = {}
        for node in nodes:
            stat = {}
            buckets = []
            _server = {"ip": node.ip, "port": node.port, "username": master.rest_username,
                           "password": master.rest_password}
            try:
                buckets = rest.get_buckets()
                mc = MemcachedClient(node.ip, port)
                stat_hash = mc.stats("hash")
            except Exception:
                if not buckets:
                    log.error("There are not any buckets in {0}:{1} node".format(node.ip, node.port))
                else:
                    log.error("Impossible to get vBucket's information for {0}:{1} node".format(node.ip, node.port))
                    _nodes_stats[node.ip + ":" + str(node.port)]
                continue
            mc.close()
            vb_names = [key[:key.index(":")] for key in list(stat_hash.keys())]

            for name in vb_names:
                stat[name] = [stat_hash[name + ":state"], stat_hash[name + ":counted"]]
            _nodes_stats[node.ip + ":" + str(port)] = stat
        log.info(_nodes_stats)
        return _nodes_stats
示例#4
0
 def wait_for_mc_stats(master,
                       bucket,
                       stat_key,
                       stat_value,
                       timeout_in_seconds=120,
                       verbose=True):
     log = logger.get("infra")
     log.info("Waiting for bucket {0} stat: {1} to match {2} on {3}".format(
         bucket, stat_key, stat_value, master.ip))
     start = time.time()
     verified = False
     while (time.time() - start) <= timeout_in_seconds:
         c = MemcachedClient(master.ip, constants.memcached_port)
         stats = c.stats()
         c.close()
         if stats and stat_key in stats \
                 and str(stats[stat_key]) == str(stat_value):
             log.info("{0} : {1}".format(stat_key, stats[stat_key]))
             verified = True
             break
         else:
             if stats and stat_key in stats:
                 if verbose:
                     log.info("{0} : {1}".format(stat_key, stats[stat_key]))
             sleep_time = 2
             if not verbose:
                 sleep_time = 0.1
             sleep(sleep_time, log_type="infra")
     return verified
示例#5
0
 def wait_for_mc_stats(master,
                       bucket,
                       stat_key,
                       stat_value,
                       timeout_in_seconds=120,
                       verbose=True):
     log.info("waiting for bucket {0} stat : {1} to match {2} on {3}".format(bucket, stat_key, \
                                                                             stat_value, master.ip))
     start = time.time()
     verified = False
     while (time.time() - start) <= timeout_in_seconds:
         c = MemcachedClient(master.ip, 11210)
         stats = c.stats()
         c.close()
         if stats and stat_key in stats and str(
                 stats[stat_key]) == str(stat_value):
             log.info("{0} : {1}".format(stat_key, stats[stat_key]))
             verified = True
             break
         else:
             if stats and stat_key in stats:
                 if verbose:
                     log.info("{0} : {1}".format(stat_key, stats[stat_key]))
             if not verbose:
                 time.sleep(0.1)
             else:
                 time.sleep(2)
     return verified
示例#6
0
 def get_mb_stats(servers, key):
     for server in servers:
         c = MemcachedClient(server.ip, 11210)
         log.info("Get flush param on server {0}, {1}".format(server, key))
         value = c.stats().get(key, None)
         log.info("Get flush param on server {0}, {1}".format(server, value))
         c.close()
    def test_vbucket_id_option(self):
        bucket = RestConnection(self.server_origin).get_bucket(self.buckets[0])
        self.num_items = self.num_items - (self.num_items % len(bucket.vbuckets))
        num_items_per_vb = self.num_items / len(bucket.vbuckets)
        template = '{{ "mutated" : 0, "age": {0}, "first_name": "{1}" }}'
        gen_load = DocumentGenerator('cbtransfer', template, range(5), ['james', 'john'], start=0, end=self.num_items)
        client = MemcachedClient(self.server_origin.ip,
                                 int(bucket.vbuckets[0].master.split(':')[1]))
        kv_value_dict = {}
        vb_id_to_check = bucket.vbuckets[-1].id
        for vb_id in xrange(len(bucket.vbuckets)):
            cur_items_per_vb = 0
            while cur_items_per_vb < num_items_per_vb:
                key, value = gen_load.next()

                client.set(key, 0, 0, value, vb_id)
                if vb_id_to_check == vb_id:
                    kv_value_dict[key] = value
                cur_items_per_vb += 1

        transfer_source = 'http://%s:%s' % (self.server_origin.ip, self.server_origin.port)
        transfer_destination = 'http://%s:%s' % (self.server_recovery.ip, self.server_recovery.port)
        output = self.shell.execute_cbtransfer(transfer_source, transfer_destination,
                                      "-b %s -B %s -i %s" % (bucket.name, bucket.name, vb_id_to_check))
        client = MemcachedClient(self.server_recovery.ip,
                                 int(bucket.vbuckets[0].master.split(':')[1]))
        for key, value in kv_value_dict.iteritems():
            _, _, d = client.get(key, vbucket=vb_id_to_check)
            self.assertEquals(d, value, 'Key: %s expected. Value expected %s. Value actual %s' % (
                                        key, value, d))
示例#8
0
 def get_mb_stats(servers, key):
     log = logger.get("infra")
     for server in servers:
         c = MemcachedClient(server.ip, constants.memcached_port)
         log.info("Get flush param on server {0}, {1}".format(server, key))
         val = c.stats().get(key, None)
         log.info("Get flush param on server {0}, {1}".format(server, val))
         c.close()
示例#9
0
 def get_mb_stats(servers, key):
     log = logger.Logger.get_logger()
     for server in servers:
         c = MemcachedClient(server.ip, 11210)
         log.info("Get flush param on server {0}, {1}".format(server, key))
         value = c.stats().get(key, None)
         log.info("Get flush param on server {0}, {1}".format(server, value))
         c.close()
示例#10
0
 def flushctl_set(servers, key, val):
     log = logger.Logger.get_logger()
     for server in servers:
         c = MemcachedClient(server.ip, 11210)
         log.info("Setting flush param on server {0}, {1} to {2}".format(server, key, val))
         rv = c.set_flush_param(key, val)
         log.info("Setting flush param on server {0}, {1} to {2}, result: {3}".format(server, key, val, rv))
         c.close()
示例#11
0
 def test_list_mechs(self):
     nodes = RestConnection(self.master).get_nodes()
     for n in nodes:
         if n.ip == self.master.ip and n.port == self.master.port:
             node = n
     client = MemcachedClient(self.master.ip, node.memcached)
     mechs = list(client.sasl_mechanisms())
     assert "CRAM-MD5" in mechs
     assert "PLAIN" in mechs
     assert len(list(mechs)) == 2
示例#12
0
 def test_list_mechs(self):
     nodes = RestConnection(self.master).get_nodes()
     for n in nodes:
         if n.ip == self.master.ip and n.port == self.master.port:
             node = n
     client = MemcachedClient(self.master.ip, node.memcached)
     mechs = list(client.sasl_mechanisms())
     assert "CRAM-MD5" in mechs
     assert "PLAIN" in mechs
     assert len(list(mechs)) == 2
示例#13
0
 def connection(self, client_ip, bucket_name, user,password, port=11210):
     log.info("Bucket name for connection is ---- {0}, username -- {1}, ----- password -- {2}".format(bucket_name,user, \
                                                                                                      password))
     try:
         mc = MemcachedClient(host=client_ip, port=port)
         mc.sasl_auth_plain(user,password)
         mc.bucket_select(bucket_name)
         return mc, True
     except Exception as e:
         log.info( "Exception is from connection function {0}".format(e))
         return False, False
示例#14
0
 def load_one_mutation_into_source_vb0(self, vb0_active_src_node):
     key = self.vb0_keys[self.key_counter]
     memc_client = MemcachedClient(vb0_active_src_node.ip, 11210)
     try:
         memc_client.set(key, exp=0, flags=0, val="dummy val")
         self.key_counter += 1
         self.keys_loaded.append(key)
         self.log.info("Loaded key {} onto vb0 in {}".format(key, vb0_active_src_node.ip))
         self.log.info ("deleted, flags, exp, rev_id, cas for key {} = {}".format(key, memc_client.getMeta(key)))
     except MemcachedError as e:
         self.log.error(e)
示例#15
0
    def __init__(self, host=cfg.COUCHBASE_IP, port=11210):
        self.host = host
        self.port = port
        self.client = None

        try:
            self.client = MemcachedClient(self.host, self.port)
        except MemcachedError as ex:
            msg = "error connecting to host %s for gathering postconditions"\
                   %  self.host
            logger.error(msg)
            pass
示例#16
0
 def test_list_mechs(self):
     nodes = RestConnection(self.master).get_nodes()
     for n in nodes:
         if n.ip == self.master.ip and n.port == self.master.port:
             node = n
     client = MemcachedClient(self.master.ip, node.memcached)
     mechs = list(client.sasl_mechanisms())
     self.log.info("Start check mech types")
     assert "SCRAM-SHA1" in mechs
     assert "SCRAM-SHA256" in mechs
     assert "SCRAM-SHA512" in mechs
     assert "PLAIN" in mechs
     assert len(list(mechs)) == 4
示例#17
0
 def test_list_mechs(self):
     nodes = RestConnection(self.master).get_nodes()
     for n in nodes:
         if n.ip == self.master.ip and n.port == self.master.port:
             node = n
     client = MemcachedClient(self.master.ip, node.memcached)
     mechs = list(client.sasl_mechanisms())
     self.log.info("Start check mech types")
     assert b"SCRAM-SHA1" in mechs
     assert b"SCRAM-SHA256" in mechs
     assert b"SCRAM-SHA512" in mechs
     assert b"PLAIN" in mechs
     assert len(list(mechs)) == 4
 def load_one_mutation_into_source_vb0(self, vb0_active_src_node):
     key = self.vb0_keys[self.key_counter]
     memc_client = MemcachedClient(vb0_active_src_node.ip, 11210)
     try:
         memc_client.set(key, exp=0, flags=0, val="dummy val")
         self.key_counter += 1
         self.keys_loaded.append(key)
         self.log.info("Loaded key {0} onto vb0 in {1}".format(
             key, vb0_active_src_node.ip))
         self.log.info(
             "deleted, flags, exp, rev_id, cas for key {0} = {1}".format(
                 key, memc_client.getMeta(key)))
     except MemcachedError as e:
         self.log.error(e)
示例#19
0
 def wait_for_warmup(self, host, port):
     if self.bucket_type == 'ephemeral': return
     while True:
         client = McdClient(host, port)
         try:
             client.bucket_select("default")
             response = client.stats()
             # check the old style or new style (as of 4.5) results
             mode = response.get('ep_degraded_mode')
             if mode is not None:
                 if mode == '0' or mode == 'false':
                     break
         except Exception as ex:
             pass
         self.sleep(1)
示例#20
0
class EPStatChecker(StatChecker):
    def __init__(self, host=cfg.COUCHBASE_IP, port=11210):
        self.host = host
        self.port = port
        self.client = None

        try:
            self.client = MemcachedClient(self.host, self.port)
        except MemcachedError as ex:
            msg = "error connecting to host %s for gathering postconditions"\
                   %  self.host
            logger.error(msg)
            pass

    @property
    def stat_keys(self):
        keys = []

        stats = self.get_stats()
        if stats:
            keys = list(stats.keys())
        return keys

    def get_stats(self):
        stats = None

        if self.client:
            try:
                stats = self.client.stats()
            except Exception:
                logger.info("unable to get stats from host: %s:%s" %\
                    (self.host, self.port))

        return stats
示例#21
0
 def _get_clients(self, bucket):
     for vbucket in bucket.vbuckets:
         if vbucket.master not in self.clients:
             ip, port = vbucket.master.split(':')
             if bucket.port:
                 port = bucket.port
             self.clients[vbucket.master] = MemcachedClient(ip, int(port))
示例#22
0
 def do_auth(self, bucket, password):
     """ default self.auth_mech is 'PLAIN' """
     self.log.info("Authenticate with {0} to {1}:{2}".format(
         self.auth_mech, bucket, password))
     ret = None
     nodes = RestConnection(self.master).get_nodes()
     for n in nodes:
         if n.ip == self.master.ip and n.port == self.master.port:
             node = n
     client = MemcachedClient(self.master.ip, node.memcached)
     try:
         if self.auth_mech == "PLAIN":
             ret = client.sasl_auth_plain(bucket, password)[2]
         else:
             self.fail("Invalid auth mechanism {0}".format(self.auth_mech))
     except MemcachedError, e:
         ret = e[0].split(' for vbucket')[0]
示例#23
0
    def test_vbucket_id_option(self):
        bucket = RestConnection(self.server_origin).get_bucket(self.buckets[0])
        self.num_items = self.num_items - (self.num_items % len(bucket.vbuckets))
        num_items_per_vb = self.num_items // len(bucket.vbuckets)
        template = '{{ "mutated" : 0, "age": {0}, "first_name": "{1}" }}'
        gen_load = DocumentGenerator('cbtransfer', template, list(range(5)), ['james', 'john'], start=0, end=self.num_items)
        client = MemcachedClient(self.server_origin.ip,
                                 int(bucket.vbuckets[0].master.split(':')[1]))
        kv_value_dict = {}
        vb_id_to_check = bucket.vbuckets[-1].id
        for vb_id in range(len(bucket.vbuckets)):
            cur_items_per_vb = 0
            while cur_items_per_vb < num_items_per_vb:
                key, value = next(gen_load)

                client.set(key, 0, 0, value, vb_id)
                if vb_id_to_check == vb_id:
                    kv_value_dict[key] = value
                cur_items_per_vb += 1

        transfer_source = 'http://%s:%s' % (self.server_origin.ip, self.server_origin.port)
        transfer_destination = 'http://%s:%s' % (self.server_recovery.ip, self.server_recovery.port)
        output = self.shell.execute_cbtransfer(transfer_source, transfer_destination,
                                      "-b %s -B %s -i %s" % (bucket.name, bucket.name, vb_id_to_check))
        client = MemcachedClient(self.server_recovery.ip,
                                 int(bucket.vbuckets[0].master.split(':')[1]))
        for key, value in kv_value_dict.items():
            _, _, d = client.get(key, vbucket=vb_id_to_check)
            self.assertEqual(d, value, 'Key: %s expected. Value expected %s. Value actual %s' % (
                                        key, value, d))
示例#24
0
 def addServer(self, ipaddr):
     try:
         mc = MemcachedClient(host=ipaddr, port=11211)
         self.servers.append(mc)
         return True
     except Exception, e:
         log("Unable to add server %s (%s)" % (ipaddr, str(e)))
         sys.exit(1)
         return False
示例#25
0
 def do_auth(self, bucket, password):
     """ default self.auth_mech is 'PLAIN' """
     self.log.info("Authenticate with {0} to {1}:{2}".format(self.auth_mech,
                                                             bucket,
                                                             password))
     ret = None
     nodes = RestConnection(self.master).get_nodes()
     for n in nodes:
         if n.ip == self.master.ip and n.port == self.master.port:
             node = n
     client = MemcachedClient(self.master.ip, node.memcached)
     try:
         if self.auth_mech == "PLAIN":
             ret = client.sasl_auth_plain(bucket, password)[2]
         else:
             self.fail("Invalid auth mechanism {0}".format(self.auth_mech))
     except MemcachedError, e:
         ret = e[0].split(' for vbucket')[0]
示例#26
0
    def vbMcdClient(self, vb):
        server = self.serverVb(vb)
        if server in self.mcdClientMap:
            client = self.mcdClientMap[server]
        else:
            ip, port = server.split(':')
            client = MemcachedClient(ip, int(port))
            self.mcdClientMap[server] = client

        return client
示例#27
0
 def direct_client(server, bucket, timeout=30):
     rest = RestConnection(server)
     node = rest.get_nodes_self()
     log = logger.Logger.get_logger()
     if isinstance(server, dict):
         log.info("dict:{0}".format(server))
         log.info("creating direct client {0}:{1} {2}".format(server["ip"], node.memcached, bucket))
     else:
         log.info("creating direct client {0}:{1} {2}".format(server.ip, node.memcached, bucket))
     RestHelper(rest).vbucket_map_ready(bucket, 60)
     vBuckets = RestConnection(server).get_vbuckets(bucket)
     if isinstance(server, dict):
         client = MemcachedClient(server["ip"], node.memcached, timeout=timeout)
     else:
         client = MemcachedClient(server.ip, node.memcached, timeout=timeout)
     client.vbucket_count = len(vBuckets)
     bucket_info = rest.get_bucket(bucket)
     #todo raise exception for not bucket_info
     client.sasl_auth_plain(bucket_info.name.encode('ascii'),
                            bucket_info.saslPassword.encode('ascii'))
     return client
 def verify_revid(self):
     missing_keys = False
     src_node = self.get_active_vb0_node(self.src_master)
     dest_node = self.get_active_vb0_node(self.dest_master)
     src_client = MemcachedClient(src_node.ip, 11210)
     dest_client = MemcachedClient(dest_node.ip, 11210)
     for key in self.keys_loaded:
         try:
             src_meta = src_client.getMeta(key)
             dest_meta = dest_client.getMeta(key)
             self.log.info(
                 "deleted, flags, exp, rev_id, cas for key from Source({0}) {1} = {2}"
                 .format(src_node.ip, key, src_meta))
             self.log.info(
                 "deleted, flags, exp, rev_id, cas for key from Destination({0}) {1} = {2}"
                 .format(dest_node.ip, key, dest_meta))
             if src_meta == dest_meta:
                 self.log.info(
                     "RevID verification successful for key {0}".format(
                         key))
             else:
                 self.fail(
                     "RevID verification failed for key {0}".format(key))
         except MemcachedError as e:
             self.log.error("Key {0} threw {1} on getMeta()".format(key, e))
             missing_keys = True
     if missing_keys:
         self.fail("Some keys are missing at destination")
示例#29
0
 def wait_for_mc_stats(master, bucket, stat_key, stat_value, timeout_in_seconds=120, verbose=True):
     log.info("waiting for bucket {0} stat : {1} to match {2} on {3}".format(bucket, stat_key, \
                                                                             stat_value, master.ip))
     start = time.time()
     verified = False
     while (time.time() - start) <= timeout_in_seconds:
         c = MemcachedClient(master.ip, 11210)
         stats = c.stats()
         c.close()
         if stats and stat_key in stats and str(stats[stat_key]) == str(stat_value):
             log.info("{0} : {1}".format(stat_key, stats[stat_key]))
             verified = True
             break
         else:
             if stats and stat_key in stats:
                 if verbose:
                     log.info("{0} : {1}".format(stat_key, stats[stat_key]))
             if not verbose:
                 time.sleep(0.1)
             else:
                 time.sleep(2)
     return verified
示例#30
0
 def verify_stat(self, items, value="active"):
     mc = MemcachedClient(self.cluster.master.ip, constants.memcached_port)
     mc.sasl_auth_plain(self.cluster.master.rest_username,
                        self.cluster.master.rest_password)
     mc.bucket_select('default')
     stats = mc.stats()
     self.assertEquals(stats['ep_compression_mode'], value)
     self.assertEquals(int(stats['ep_item_compressor_num_compressed']),
                       items)
     self.assertNotEquals(int(stats['vb_active_itm_memory']),
                          int(stats['vb_active_itm_memory_uncompressed']))
示例#31
0
    def wait_for_mc_stats_no_timeout(master, bucket, stat_key, stat_value, timeout_in_seconds=-1, verbose=True):
        log.info("waiting for bucket {0} stat : {1} to match {2} on {3}".format(bucket, stat_key, \
                                                                                stat_value, master.ip))
        c = MemcachedClient(master.ip, 11210)
        stats = c.stats()
        c.close()

        while str(stats[stat_key]) != str(stat_value):
            c = MemcachedClient(master.ip, 11210)
            stats = c.stats()
            c.close()
            if verbose:
                log.info("{0} : {1}".format(stat_key, stats[stat_key]))
            time.sleep(5)
        return True
示例#32
0
    def _build_conns(self):
        """build separate connections based on store"""
        if not self.store:
            print "<%s> failed to build connections, invalid store object"\
                % self.__class__.__name__
            return False

        if self.store.__class__.__name__ == "StoreMemcachedBinary":
            conn = MemcachedClient(self.store.conn.host, self.store.conn.port)
            server_str = "{0}:{1}".format(self.store.conn.host, self.store.conn.port)
            self.conns[server_str] = conn
        elif self.store.__class__.__name__ == "StoreMembaseBinary":
            for memcached in self.store.awareness.memcacheds.itervalues():
                conn = MemcachedClient(memcached.host, memcached.port)
                server_str = "{0}:{1}".format(conn.host, conn.port)
                self.conns[server_str] = conn
            self.awareness = self.store.awareness
        else:
            print "<%s> error: unsupported store object %s" %\
                  (self.__class__.__name__, store.__class__.__name__)
            return False

        return True
示例#33
0
    def _add_conn(self, server):
        if not self.store:
            print "<%s> failed to add conn, invalid store object"\
                % self.__class__.__name__
            return False

        if self.store.__class__.__name__ == "StoreMembaseBinary":
            print "<%s> _add_conn: %s"\
                % (self.__class__.__name__, server)
            host, port = server.split(":")
            conn = MemcachedClient(host, int(port))
            self.conns[server] = conn

        return True
示例#34
0
    def init_clients(self):
        """Initialise clients for all servers there are vBuckets on

        It returns a dict with 'ip:port' as key (this information is also
        stored this way in every vBucket in the `master` property) and
        the MemcachedClient as the value
        """
        clients = {}

        for vbucket in self.bucket.vbuckets:
            if vbucket.master not in clients:
                ip, port = vbucket.master.split(':')
                clients[vbucket.master] =  MemcachedClient(ip, int(port))
        return clients
示例#35
0
def getDirectMC(key, ip, port = 8091, bucket = "default", password = ""):

    real_mc_client = None

    # get initial mc client
    client = MemcachedClient(ip, int(port))
    vbId = (((zlib.crc32(key)) >> 16) & 0x7fff) & (client.vbucket_count - 1)

    # get vbucket map
    rest = create_rest(ip, port)
    vbuckets = rest.get_vbuckets(bucket)

    # find vbucket responsible to this key and mapping host
    if vbuckets is not None:

        vbucket = [vbucket for vbucket in vbuckets if vbucket.id == vbId]

        if len(vbucket) == 1:
            mc_ip, mc_port = vbucket[0].master.split(":")
            real_mc_client = MemcachedClient(mc_ip, int(mc_port))
            real_mc_client.sasl_auth_plain(bucket, password)

    return real_mc_client
示例#36
0
 def verify_revid(self):
     missing_keys = False
     src_node = self.get_active_vb0_node(self.src_master)
     dest_node = self.get_active_vb0_node(self.dest_master)
     src_client = MemcachedClient(src_node.ip, 11210)
     dest_client = MemcachedClient(dest_node.ip, 11210)
     for key in self.keys_loaded:
         try:
             src_meta = src_client.getMeta(key)
             dest_meta = dest_client.getMeta(key)
             self.log.info("deleted, flags, exp, rev_id, cas for key from Source({0}) {1} = {2}"
                            .format(src_node.ip, key, src_meta))
             self.log.info("deleted, flags, exp, rev_id, cas for key from Destination({0}) {1} = {2}"
                            .format(dest_node.ip, key, dest_meta))
             if src_meta == dest_meta:
                 self.log.info("RevID verification successful for key {}".format(key))
             else:
                 self.fail("RevID verification failed for key {}".format(key))
         except MemcachedError:
             self.log.error("Key {} is missing at destination".format(key))
             missing_keys = True
     if missing_keys:
         self.fail("Some keys are missing at destination")
示例#37
0
def getDirectMC(key, ip, port=8091, bucket="default", password=""):

    real_mc_client = None

    # get initial mc client
    client = MemcachedClient(ip, int(port))
    vbId = (((zlib.crc32(key)) >> 16) & 0x7fff) & (client.vbucket_count - 1)

    # get vbucket map
    rest = create_rest(ip, port)
    vbuckets = rest.get_vbuckets(bucket)

    # find vbucket responsible to this key and mapping host
    if vbuckets is not None:

        vbucket = [vbucket for vbucket in vbuckets if vbucket.id == vbId]

        if len(vbucket) == 1:
            mc_ip, mc_port = vbucket[0].master.split(":")
            real_mc_client = MemcachedClient(mc_ip, int(mc_port))
            real_mc_client.sasl_auth_plain(bucket, password)

    return real_mc_client
    def __init__(self,
                 host = cfg.COUCHBASE_IP,
                 port = 11210):
        self.host = host
        self.port = port
        self.client = None

        try:
            self.client = MemcachedClient(self.host, self.port)
        except MemcachedError as ex:
            msg = "error connecting to host %s for gathering postconditions"\
                   %  self.host
            logger.error(msg)
            pass
示例#39
0
 def load_one_mutation_into_source_vb0(self, vb0_active_src_node):
     key = self.vb0_keys[self.key_counter]
     memc_client = MemcachedClient(vb0_active_src_node.ip, 11210)
     memc_client.sasl_auth_plain("cbadminbucket","password")
     memc_client.bucket_select("default")
     try:
         memc_client.set(key, exp=0, flags=0, val="dummy val")
         self.key_counter += 1
         self.keys_loaded.append(key)
         self.log.info("Loaded key {0} onto vb0 in {1}".format(key, vb0_active_src_node.ip))
         self.log.info ("deleted, flags, exp, rev_id, cas for key {0} = {1}".format(key, memc_client.getMeta(key)))
     except MemcachedError as e:
         self.log.error(e)
示例#40
0
 def wait_for_vbuckets_ready_state(node,
                                   bucket,
                                   timeout_in_seconds=300,
                                   log_msg=''):
     log = logger.Logger.get_logger()
     start_time = time.time()
     end_time = start_time + timeout_in_seconds
     ready_vbuckets = {}
     rest = RestConnection(node)
     servers = rest.get_nodes()
     RestHelper(rest).vbucket_map_ready(bucket, 60)
     vbucket_count = len(rest.get_vbuckets(bucket))
     vbuckets = rest.get_vbuckets(bucket)
     obj = VBucketAwareMemcached(rest, bucket)
     memcacheds, vbucket_map, vbucket_map_replica = obj.request_map(
         rest, bucket)
     #Create dictionary with key:"ip:port" and value: a list of vbuckets
     server_dict = defaultdict(list)
     for everyID in range(0, vbucket_count):
         memcached_ip_port = str(vbucket_map[everyID])
         server_dict[memcached_ip_port].append(everyID)
     while time.time() < end_time and len(ready_vbuckets) < vbucket_count:
         for every_ip_port in server_dict:
             #Retrieve memcached ip and port
             ip, port = every_ip_port.split(":")
             client = MemcachedClient(ip, int(port), timeout=30)
             client.vbucket_count = len(vbuckets)
             bucket_info = rest.get_bucket(bucket)
             client.sasl_auth_plain(
                 bucket_info.name.encode('ascii'),
                 bucket_info.saslPassword.encode('ascii'))
             for i in server_dict[every_ip_port]:
                 try:
                     (a, b, c) = client.get_vbucket_state(i)
                 except mc_bin_client.MemcachedError as e:
                     log.error("%s: %s" % (log_msg, e))
                     continue
                 if c.find("\x01") > 0 or c.find("\x02") > 0:
                     ready_vbuckets[i] = True
                 elif i in ready_vbuckets:
                     log.warning(
                         "vbucket state changed from active to {0}".format(
                             c))
                     del ready_vbuckets[i]
             client.close()
     return len(ready_vbuckets) == vbucket_count
示例#41
0
    def do_get_random_key(self):
        # MB-31548, get_Random key gets hung sometimes.
        mc = MemcachedClient(self.cluster.master.ip, constants.memcached_port)
        mc.sasl_auth_plain(self.cluster.master.rest_username,
                           self.cluster.master.rest_password)
        mc.bucket_select('default')

        count = 0
        while count < 1000000:
            count += 1
            try:
                mc.get_random_key()
            except MemcachedError as error:
                self.fail("<MemcachedError #%d ``%s''>" %
                          (error.status, error.message))
            if count % 1000 == 0:
                self.log.info('The number of iteration is {}'.format(count))
示例#42
0
    def __init__(self, vbs_server):
        cmd = "curl -s -L http://%s/%s" % (vbs_server, consts.VBS_API_PATH)
        status, output = getcommandoutput(cmd)
        if status > 0:
            log("ERROR: Unable to contact VBS: %s" % (vbs_server))
            sys.exit(1)

        j = json.loads(output)
        vbsmap = j['buckets'][0]['vBucketServerMap']['vBucketMap']
        srvlist = j['buckets'][0]['vBucketServerMap']['serverList']
        self.vbmap = {}

        for i, v in enumerate(vbsmap):
            ip, port = srvlist[v[0]].split(':')
            mc = MemcachedClient(host=ip, port=int(port))
            self.vbmap[i] = mc
示例#43
0
 def connection(self, client_ip, bucket_name, user,password, port=11210):
     log.info("Bucket name for connection is ---- {0}, username -- {1}, ----- password -- {2}".format(bucket_name, user, \
                                                                                                      password))
     try:
         mc = MemcachedClient(host=client_ip, port=port)
         mc.sasl_auth_plain(user, password)
         mc.bucket_select(bucket_name)
         return mc, True
     except Exception as e:
         log.info( "Exception is from connection function {0}".format(e))
         return False, False
示例#44
0
 def wait_for_vbuckets_ready_state(node, bucket, timeout_in_seconds=300, log_msg=''):
     log = logger.Logger.get_logger()
     start_time = time.time()
     end_time = start_time + timeout_in_seconds
     ready_vbuckets = {}
     rest = RestConnection(node)
     servers = rest.get_nodes()
     RestHelper(rest).vbucket_map_ready(bucket, 60)
     vbucket_count = len(rest.get_vbuckets(bucket))
     vbuckets = rest.get_vbuckets(bucket)
     obj = VBucketAwareMemcached(rest, bucket)
     memcacheds, vbucket_map, vbucket_map_replica = obj.request_map(rest, bucket)
     #Create dictionary with key:"ip:port" and value: a list of vbuckets
     server_dict = defaultdict(list)
     for everyID in range(0, vbucket_count):
         memcached_ip_port = str(vbucket_map[everyID])
         server_dict[memcached_ip_port].append(everyID)
     while time.time() < end_time and len(ready_vbuckets) < vbucket_count:
         for every_ip_port in server_dict:
             #Retrieve memcached ip and port
             ip, port = every_ip_port.split(":")
             client = MemcachedClient(ip, int(port), timeout=30)
             client.vbucket_count = len(vbuckets)
             bucket_info = rest.get_bucket(bucket)
             client.sasl_auth_plain(bucket_info.name.encode('ascii'),
                                 bucket_info.saslPassword.encode('ascii'))
             for i in server_dict[every_ip_port]:
                 try:
                     (a, b, c) = client.get_vbucket_state(i)
                 except mc_bin_client.MemcachedError as e:
                     ex_msg = str(e)
                     if "Not my vbucket" in log_msg:
                         log_msg = log_msg[:log_msg.find("vBucketMap") + 12] + "..."
                     if "Not my vbucket" in ex_msg:
                         #reduce output
                         ex_msg = str(e)[:str(e).find('Not my vbucket') + 14] + "..."
                     log.error("%s: %s" % (log_msg, ex_msg))
                     continue
                 if c.find("\x01") > 0 or c.find("\x02") > 0:
                     ready_vbuckets[i] = True
                 elif i in ready_vbuckets:
                     log.warning("vbucket state changed from active to {0}".format(c))
                     del ready_vbuckets[i]
             client.close()
     return len(ready_vbuckets) == vbucket_count
class EPStatChecker(StatChecker):

    def __init__(self,
                 host = cfg.COUCHBASE_IP,
                 port = 11210):
        self.host = host
        self.port = port
        self.client = None

        try:
            self.client = MemcachedClient(self.host, self.port)
        except MemcachedError as ex:
            msg = "error connecting to host %s for gathering postconditions"\
                   %  self.host
            logger.error(msg)
            pass

    @property
    def stat_keys(self):
        keys = []

        stats = self.get_stats()
        if stats:
            keys = stats.keys()
        return keys

    def get_stats(self):
        stats = None

        if self.client:
            try:
                stats = self.client.stats()
            except Exception:
                logger.info("unable to get stats from host: %s:%s" %\
                    (self.host, self.port))

        return stats
示例#46
0
    def client_helper(self, node, type_, vbucket):
        self.assertTrue(type_ in (MCD, DCP), msg="type not found")

        client = None
        ip = None
        port = None
        rest = RestConnection(node)

        if vbucket is not None:
            host = self.vbucket_host(rest, vbucket)
            ip = host.split(':')[0]
            port = int(host.split(':')[1])

        else:
            client_node = rest.get_nodes_self()
            ip = client_node.hostname.split(':')[0]
            port = client_node.memcached

        if type_ == MCD:
            client = MemcachedClient(ip, port)
        else:
            client = DcpClient(ip, port)

        return client
示例#47
0
    def wait_for_mc_stats_no_timeout(master,
                                     bucket,
                                     stat_key,
                                     stat_value,
                                     timeout_in_seconds=-1,
                                     verbose=True):
        log.info("waiting for bucket {0} stat : {1} to match {2} on {3}".format(bucket, stat_key, \
                                                                                stat_value, master.ip))
        # keep retrying until reaches the server
        stats = {}
        while not stats:
            try:
                c = MemcachedClient(master.ip, 11210)
                c.sasl_auth_plain(bucket, '')
                stats = c.stats()
            except Exception as e:
                log.info("Exception: {0}, retry in 2 seconds ...".format(
                    str(e)))
                stats = {}
                time.sleep(2)
            finally:
                c.close()

        while str(stats[stat_key]) != str(stat_value):
            c = MemcachedClient(master.ip, 11210)
            c.sasl_auth_plain(bucket, '')
            stats = c.stats()
            c.close()
            if verbose:
                log.info("{0} : {1}".format(stat_key, stats[stat_key]))
            time.sleep(5)
        return True
示例#48
0
 def verify_revid(self):
     missing_keys = False
     src_node = self.get_active_vb0_node(self.src_master)
     dest_node = self.get_active_vb0_node(self.dest_master)
     src_client = MemcachedClient(src_node.ip, 11210)
     dest_client = MemcachedClient(dest_node.ip, 11210)
     src_client.sasl_auth_plain("cbadminbucket","password")
     src_client.bucket_select("default")
     dest_client.sasl_auth_plain("cbadminbucket","password")
     dest_client.bucket_select("default")
     for key in self.keys_loaded:
         try:
             src_meta = src_client.getMeta(key)
             dest_meta = dest_client.getMeta(key)
             self.log.info("deleted, flags, exp, rev_id, cas for key from Source({0}) {1} = {2}"
                            .format(src_node.ip, key, src_meta))
             self.log.info("deleted, flags, exp, rev_id, cas for key from Destination({0}) {1} = {2}"
                            .format(dest_node.ip, key, dest_meta))
             if src_meta == dest_meta:
                 self.log.info("RevID verification successful for key {0}".format(key))
             else:
                 self.fail("RevID verification failed for key {0}".format(key))
         except MemcachedError as e:
             self.log.error("Key {0} threw {1} on getMeta()".format(key, e))
             missing_keys = True
     if missing_keys:
         self.fail("Some keys are missing at destination")
示例#49
0
    parser.add_option("-n", "--node", dest="node", default="127.0.0.1",
                      help="couchdb ip , defaults to 127.0.0.1:5984")
    parser.add_option("-s", "--sleep", dest="sleep", default="600",
                      help="sleep time before each iteration")
    parser.add_option("-i", "--items", dest="items", default="10000",
                      help="number of items")

    options, args = parser.parse_args()

    node = options.node
    sleep_time = int(options.sleep)

    prefix = str(uuid.uuid4())
    number_of_items = int(options.items)

    mc = MemcachedClient("127.0.0.1", 11211)

    keys = ["{0}-{1}".format(prefix, i) for i in range(0, number_of_items)]
    info("inserting {0} items".format(number_of_items))
    for k in keys:
        mc.set(k, 0, 0, str(uuid.uuid4())[0:16])

    while True:
        info("now remove 3 chars from 80% of keys - if < 3 chars delete the key - if key does not exist create it")
        for i in range(0, 3):
            for k in keys:
                try:
                    a, b, value = mc.get(k)
                    if len(value) < 3:
                        mc.delete(k)
                    else:
示例#50
0
queue = Queue(maxsize=10000)

server = ServerInfo()
server.ip = "10.17.12.20"

bucket = {'name': 'default', 'port': 11220, 'password': ''}
#vam = VBucketAwareMemcached(RestConnection(server), bucket)
#print vam.memcacheds
#print vam.vBucketMap
payload = MemcachedClientHelper.create_value('*', 10240)
keys = ["key_%d" % (i) for i in range(4000)]
#keys = ["key_%s_%d" % (str(uuid.uuid4()), i) for i in range(4)]
total_size = 0
#mc = MemcachedClientHelper.create_memcached_client("172.16.75.128","default",11210,"default")
mc = MemcachedClient("10.17.12.20", 11210)
#for key in keys:
#    vam.memcached(key).set(key, 1, 0, payload)
#    total_size += len(key) + len(payload) + 200
#time.sleep(10)
#for i in range(0,1023):
#    mc.set_vbucket_state(i, 'active')
new_thread = TapListener(queue, server)
new_thread.start()


i = 0
while i < 4000:
    for key in keys:
    #    vam.memcached(key).get(key)
        mc.set(key, 10, 0, payload, vbucket=0)
示例#51
0
    def wait_for_vbuckets_ready_state(node, bucket, timeout_in_seconds=300, log_msg='', admin_user='******',
                                      admin_pass='******'):
        log = logger.Logger.get_logger()
        start_time = time.time()
        end_time = start_time + timeout_in_seconds
        ready_vbuckets = {}
        rest = RestConnection(node)
        servers = rest.get_nodes()
        RestHelper(rest).vbucket_map_ready(bucket, 60)
        vbucket_count = len(rest.get_vbuckets(bucket))
        vbuckets = rest.get_vbuckets(bucket)
        obj = VBucketAwareMemcached(rest, bucket)
        memcacheds, vbucket_map, vbucket_map_replica = obj.request_map(rest, bucket)
        #Create dictionary with key:"ip:port" and value: a list of vbuckets
        server_dict = defaultdict(list)
        for everyID in range(0, vbucket_count):
            memcached_ip_port = str(vbucket_map[everyID])
            server_dict[memcached_ip_port].append(everyID)
        while time.time() < end_time and len(ready_vbuckets) < vbucket_count:
            for every_ip_port in server_dict:
                #Retrieve memcached ip and port
                ip = every_ip_port.rsplit(":", 1)[0]
                port = every_ip_port.rsplit(":", 1)[1]
                client = MemcachedClient(ip, int(port), timeout=30)
                client.vbucket_count = len(vbuckets)
                bucket_info = rest.get_bucket(bucket)
                cluster_compatibility = rest.check_cluster_compatibility("5.0")
                if cluster_compatibility is None:
                    pre_spock = True
                else:
                    pre_spock = not cluster_compatibility
                if pre_spock:
                    log.info("Atleast 1 of the server is on pre-spock "
                             "version. Using the old ssl auth to connect to "
                             "bucket.")
                    client.sasl_auth_plain(
                    bucket_info.name.encode('ascii'),
                    bucket_info.saslPassword.encode('ascii'))
                else:
                    client.sasl_auth_plain(admin_user, admin_pass)
                    bucket = bucket.encode('ascii')
                    client.bucket_select(bucket)
                for i in server_dict[every_ip_port]:
                    try:
                        (a, b, c) = client.get_vbucket_state(i)
                    except mc_bin_client.MemcachedError as e:
                        ex_msg = str(e)
                        if "Not my vbucket" in log_msg:
                            log_msg = log_msg[:log_msg.find("vBucketMap") + 12] + "..."
                        if e.status == memcacheConstants.ERR_NOT_MY_VBUCKET:
                            # May receive this while waiting for vbuckets, continue and retry...S
                            continue
                        log.error("%s: %s" % (log_msg, ex_msg))
                        continue
                    except exceptions.EOFError:
                        # The client was disconnected for some reason. This can
                        # happen just after the bucket REST API is returned (before
                        # the buckets are created in each of the memcached processes.)
                        # See here for some details: http://review.couchbase.org/#/c/49781/
                        # Longer term when we don't disconnect clients in this state we
                        # should probably remove this code.
                        log.error("got disconnected from the server, reconnecting")
                        client.reconnect()
                        client.sasl_auth_plain(bucket_info.name.encode('ascii'),
                                               bucket_info.saslPassword.encode('ascii'))
                        continue

                    if c.find("\x01") > 0 or c.find("\x02") > 0:
                        ready_vbuckets[i] = True
                    elif i in ready_vbuckets:
                        log.warning("vbucket state changed from active to {0}".format(c))
                        del ready_vbuckets[i]
                client.close()
        return len(ready_vbuckets) == vbucket_count
示例#52
0
文件: jinja.py 项目: subalakr/jinja
def storeJob(jobDoc, bucket, first_pass = True):

    client = McdClient(HOST, PORT)
    client.sasl_auth_plain(bucket, "")

    doc = jobDoc
    url = doc["url"]

    res = getJS(url, {"depth" : 0}).json()

    if res is None:
        return

    buildHist = {}
    if res["lastBuild"]:

        bids = [b["number"] for b in res["builds"]]

        lastTotalCount = -1
        idx=0

        for bid in bids:
            idx = idx + 1
            i = 1
            if idx < len(bids):
                while bids[idx] != bid-i:
                    key = "%s-%s" % (doc["name"], bid-i)
                    key = hashlib.md5(key).hexdigest()
                    try:
                        client.delete(key, vbucket=0)
                    except:
                        pass
                    i = i + 1
            if bid in JOBS[doc["name"]]:
                continue # job already stored
            else:
                if first_pass == False:
                    JOBS[doc["name"]].append(bid)

            doc["build_id"] = bid
            res = getJS(url+str(bid), {"depth" : 0}).json()
            if res is None:
                return

            if "result" not in res:
                continue

            doc["result"] = res["result"]
            doc["duration"] = res["duration"]

            if bucket == "server":
                if res["result"] not in ["SUCCESS", "UNSTABLE", "FAILURE", "ABORTED"]:
                    continue # unknown result state

                actions = res["actions"]
                totalCount = getAction(actions, "totalCount") or 0
                failCount  = getAction(actions, "failCount") or 0
                skipCount  = getAction(actions, "skipCount") or 0
                if totalCount == 0:
                    if lastTotalCount == -1:
                        continue # no tests ever passed for this build
                    else:
                        totalCount = lastTotalCount
                        failCount = totalCount
                else:
                    lastTotalCount = totalCount

                doc["failCount"] = failCount
                doc["totalCount"] = totalCount - skipCount

                params = getAction(actions, "parameters")
                if params is None:
                    doc["priority"] = P1
                    doc["build"] = DEFAULT_BUILD
                else:
                    doc["build"] = getAction(params, "name", "version_number") or getAction(params, "name", "cluster_version") or  DEFAULT_BUILD
                    doc["priority"] = getAction(params, "name", "priority") or P1
                    if doc["priority"].upper() not in [P0, P1, P2]:
                        doc["priority"] = P1
                doc["build"] = doc["build"].replace("-rel","").split(",")[0]
                try:
                    _build= doc["build"].split("-")
                    rel, bno = _build[0], _build[1]
                    # check partial rel #'s
                    rlen = len(rel.split("."))
                    while rlen < 3:
                        rel = rel+".0"
                        rlen+=1

                    # verify rel, build
                    m=re.match("^\d\.\d\.\d{1,5}", rel)
                    if m is None:
                        print "unsupported version_number: "+doc["build"]
                        continue
                    m=re.match("^\d{1,10}", bno)
                    if m is None:
                        print "unsupported version_number: "+doc["build"]
                        continue

                    doc["build"] = "%s-%s" % (rel, bno.zfill(4))
                except:
                    print "unsupported version_number: "+doc["build"]
                    continue

            else:
                # use date as version for sdk and mobile
                if res["result"] not in ["SUCCESS", "UNSTABLE", "FAILURE", "ABORTED"]:
                    continue 

                actions = res["actions"]
                totalCount = getAction(actions, "totalCount") or 0
                failCount  = getAction(actions, "failCount") or 0
                skipCount  = getAction(actions, "skipCount") or 0
                if totalCount == 0:
                    if lastTotalCount == -1:
                        continue # no tests ever passed for this build
                    else:
                        totalCount = lastTotalCount
                        failCount = totalCount
                else:
                    lastTotalCount = totalCount

                doc["failCount"] = failCount
                doc["totalCount"] = totalCount - skipCount
                doc["priority"] =  P0

                ts =  res["timestamp"]/1000;
                month = int(datetime.datetime.fromtimestamp(ts).strftime("%m"))
                _ts = datetime.datetime.fromtimestamp(ts).strftime("%Y-%m%d")
                yr, md = _ts.split("-")
                doc["build"] = "%s-%s%s" % (MOBILE_VERSION, yr[-1], md)


            if doc["build"] in buildHist:

                #print "REJECTED- doc already in build results: %s" % doc
                #print buildHist

                # attempt to delete if this record has been stored in couchbase

                try:
                    oldKey = "%s-%s" % (doc["name"], doc["build_id"])
                    oldKey = hashlib.md5(oldKey).hexdigest()
                    client.delete(oldKey, vbucket = 0)
                    #print "DELETED- %s:%s" % (doc["build"],doc["build_id"])
                except:
                    pass

                continue # already have this build results


            key = "%s-%s" % (doc["name"], doc["build_id"])
            key = hashlib.md5(key).hexdigest()
            val = json.dumps(doc)
            try:
                #print val
                client.set(key, 0, 0, val, 0)
                buildHist[doc["build"]] = doc["build_id"]
            except:
                print "set failed, couchbase down?: %s:%s"  % (HOST,PORT)


    if first_pass:
        storeJob(jobDoc, bucket, first_pass = False)
示例#53
0
 def flushctl_stop(servers, username=None, password=None):
     for server in servers:
         c = MemcachedClient(server.ip, 11210)
         if username:
             c.sasl_auth_plain(username, password)
         c.stop_persistence()
示例#54
0
class ComplianceTest(unittest.TestCase):

    def setUp(self):
        self.mc=MemcachedClient()
        self.mc.flush()

    def tearDown(self):
        self.mc.flush()
        self.mc.close()

    def testVersion(self):
        """Test the version command returns something."""
        v=self.mc.version()
        self.assertTrue(len(v) > 0, "Bad version:  ``" + str(v) + "''")

    def testSimpleSetGet(self):
        """Test a simple set and get."""
        self.mc.set("x", 5, 19, "somevalue")
        self.assertGet((19, "somevalue"), self.mc.get("x"))

    def testZeroExpiration(self):
        """Ensure zero-expiration sets work properly."""
        self.mc.set("x", 0, 19, "somevalue")
        time.sleep(1.1)
        self.assertGet((19, "somevalue"), self.mc.get("x"))

    def assertNotExists(self, key):
        try:
            x=self.mc.get(key)
            self.fail("Expected an exception, got " + `x`)
        except MemcachedError, e:
            self.assertEquals(memcacheConstants.ERR_NOT_FOUND, e.status)
示例#55
0
    def wait_for_vbuckets_ready_state(node,
                                      bucket,
                                      timeout_in_seconds=300,
                                      log_msg='',
                                      admin_user='******',
                                      admin_pass='******'):
        start_time = time.time()
        end_time = start_time + timeout_in_seconds
        ready_vbuckets = {}
        log = logger.get("infra")
        rest = RestConnection(node)
        #         servers = rest.get_nodes()
        bucket_conn = BucketHelper(node)
        bucket_conn.vbucket_map_ready(bucket, 60)
        vbucket_count = len(bucket_conn.get_vbuckets(bucket))
        vbuckets = bucket_conn.get_vbuckets(bucket)
        obj = VBucketAwareMemcached(rest, bucket, info=node)
        memcacheds, vbucket_map, vbucket_map_replica = obj.request_map(
            rest, bucket)
        #Create dictionary with key:"ip:port" and value: a list of vbuckets
        server_dict = defaultdict(list)
        for everyID in range(0, vbucket_count):
            memcached_ip_port = str(vbucket_map[everyID])
            server_dict[memcached_ip_port].append(everyID)
        while time.time() < end_time and len(ready_vbuckets) < vbucket_count:
            for every_ip_port in server_dict:
                #Retrieve memcached ip and port
                ip, port = every_ip_port.split(":")
                client = MemcachedClient(ip, int(port), timeout=30)
                client.vbucket_count = len(vbuckets)
                bucket_info = bucket_conn.get_bucket(bucket)
                versions = rest.get_nodes_versions(logging=False)
                pre_spock = False
                for version in versions:
                    if "5" > version:
                        pre_spock = True
                if pre_spock:
                    log.info("Atleast 1 of the server is on pre-spock "
                             "version. Using the old ssl auth to connect to "
                             "bucket.")
                    client.sasl_auth_plain(
                        bucket_info.name.encode('ascii'),
                        bucket_info.saslPassword.encode('ascii'))
                else:
                    client.sasl_auth_plain(admin_user, admin_pass)
                    bucket = bucket.encode('ascii')
                    client.bucket_select(bucket)
                for i in server_dict[every_ip_port]:
                    try:
                        (a, b, c) = client.get_vbucket_state(i)
                    except mc_bin_client.MemcachedError as e:
                        ex_msg = str(e)
                        if "Not my vbucket" in log_msg:
                            log_msg = log_msg[:log_msg.find("vBucketMap") +
                                              12] + "..."
                        if e.status == memcacheConstants.ERR_NOT_MY_VBUCKET:
                            # May receive this while waiting for vbuckets, continue and retry...S
                            continue
                        log.error("%s: %s" % (log_msg, ex_msg))
                        continue
                    except exceptions.EOFError:
                        # The client was disconnected for some reason. This can
                        # happen just after the bucket REST API is returned (before
                        # the buckets are created in each of the memcached processes.)
                        # See here for some details: http://review.couchbase.org/#/c/49781/
                        # Longer term when we don't disconnect clients in this state we
                        # should probably remove this code.
                        log.error(
                            "got disconnected from the server, reconnecting")
                        continue

                    if c.find("\x01") > 0 or c.find("\x02") > 0:
                        ready_vbuckets[i] = True
                    elif i in ready_vbuckets:
                        log.warning(
                            "vbucket state changed from active to {0}".format(
                                c))
                        del ready_vbuckets[i]
                client.close()
        return len(ready_vbuckets) == vbucket_count
示例#56
0
 def flushctl_stop(servers, username=None, password=None):
     for server in servers:
         c = MemcachedClient(server.ip, 11210)
         if username:
             c.sasl_auth_plain(username, password)
         c.stop_persistence()
示例#57
0
def poll():
    url = "https://macbuild.hq.couchbase.com/xcode/api/integrations/filter/latest"
    latest = getJS(url)
    now = datetime.datetime.now()
    ts = now.strftime("%Y%m%d")
    build_no  = "%s-%s" % (MOBILE_VERSION, ts[-5:])
    client = McdClient(HOST, PORT)
    client.sasl_auth_plain("mobile", "")

    buildHist = {}
    for build in latest:
        if 'revisionBlueprint' not in latest[build]:
            continue
        key = latest[build]['revisionBlueprint']['DVTSourceControlWorkspaceBlueprintPrimaryRemoteRepositoryKey']
        rev = latest[build]['revisionBlueprint']['DVTSourceControlWorkspaceBlueprintLocationsKey'][key]['DVTSourceControlLocationRevisionKey']
        name = "iOS-"+latest[build]['bot']['name'].replace(" ","")
        build_id = latest[build]['number']
        results = latest[build]['buildResultSummary']
        totalCount = results['testsCount']
        failCount = results['errorCount']
        result = 'SUCCESS'

        if name in  TOTAL_COUNT_LOG:
            if totalCount == 0:
                # use historical value for total count
                totalCount = TOTAL_COUNT_LOG[name]
            elif totalCount != TOTAL_COUNT_LOG:
                # update total count log
                TOTAL_COUNT_LOG[name] = totalCount
        else:
            TOTAL_COUNT_LOG[name] = totalCount

        if (failCount > 0):
            result = 'UNSTABLE'
        component = None
        for feature in MOBILE_FEATURES:
            tag, _c = feature.split("-")
            docname = name.upper()
            docname = docname.replace("-","_")
            if tag in docname:
               component = _c 
        if component:
            doc = {'build_id': build_id,
                   'priority': 'P0',
                   'name': name,
                   'url': url,
                   'component': component,
                   'failCount':  failCount,
                   'totalCount': totalCount,
                   'result': result,
                   'os': 'iOS',
                   'build': build_no}

            key = "%s-%s" % (doc["name"], doc["build_id"])
            val = json.dumps(doc)
            try:
                #print val
                key = hashlib.md5(key).hexdigest()
                print val
                client.set(key, 0, 0, val, 0)
                buildHist[doc["build"]] = doc["build_id"]
            except Exception as ex:
                print ex
                print "set failed, couchbase down?: %s:%s"  % (HOST,PORT)
示例#58
0
    def wait_for_mc_stats_no_timeout(master, bucket, stat_key, stat_value, timeout_in_seconds=-1, verbose=True):
        log.info(
            "waiting for bucket {0} stat : {1} to match {2} on {3}".format(bucket, stat_key, stat_value, master.ip)
        )
        # keep retrying until reaches the server
        stats = {}
        while not stats:
            try:
                c = MemcachedClient(master.ip, 11210)
                c.sasl_auth_plain(bucket, "")
                stats = c.stats()
            except Exception as e:
                log.info("Exception: {0}, retry in 2 seconds ...".format(str(e)))
                stats = {}
                time.sleep(2)
            finally:
                c.close()

        while str(stats[stat_key]) != str(stat_value):
            c = MemcachedClient(master.ip, 11210)
            c.sasl_auth_plain(bucket, "")
            stats = c.stats()
            c.close()
            if verbose:
                log.info("{0} : {1}".format(stat_key, stats[stat_key]))
            time.sleep(5)
        return True
示例#59
0
 def setUp(self):
     self.mc=MemcachedClient()
     self.mc.flush()