def get_vBuckets_info(master): """ return state and count items for all vbuckets for each node format: dict: {u'1node_ip1': {'vb_79': ['replica', '0'], 'vb_78': ['active', '0']..}, u'1node_ip1':....} """ rest = RestConnection(master) port = rest.get_nodes_self().memcached nodes = rest.node_statuses() _nodes_stats= {} for node in nodes: stat={} buckets = [] _server = {"ip": node.ip, "port": node.port, "username": master.rest_username, "password": master.rest_password} try: buckets = rest.get_buckets() mc = MemcachedClient(node.ip, port) stat_hash = mc.stats("hash") except Exception: if not buckets: log.error("There are not any buckets in {0}:{1} node".format(node.ip, node.port)) else: log.error("Impossible to get vBucket's information for {0}:{1} node".format(node.ip, node.port)) _nodes_stats[node.ip+":"+str(node.port)] continue mc.close() vb_names=[key[:key.index(":")] for key in stat_hash.keys()] for name in vb_names: stat[name]=[stat_hash[name + ":state"], stat_hash[name+":counted"]] _nodes_stats[node.ip+":"+str(port)] = stat log.info(_nodes_stats) return _nodes_stats
def direct_client(server, bucket, timeout=30): log = logger.Logger.get_logger() rest = RestConnection(server) node = None try: node = rest.get_nodes_self() except ValueError as e: log.info("could not connect to server {0}, will try scanning all nodes".format(server)) if not node: nodes = rest.get_nodes() for n in nodes: if n.ip == server.ip and n.port == server.port: node = n if isinstance(server, dict): log.info("dict:{0}".format(server)) log.info("creating direct client {0}:{1} {2}".format(server["ip"], node.memcached, bucket)) else: log.info("creating direct client {0}:{1} {2}".format(server.ip, node.memcached, bucket)) RestHelper(rest).vbucket_map_ready(bucket, 60) vBuckets = RestConnection(server).get_vbuckets(bucket) if isinstance(server, dict): client = MemcachedClient(server["ip"], node.memcached, timeout=timeout) else: client = MemcachedClient(server.ip, node.memcached, timeout=timeout) client.vbucket_count = len(vBuckets) bucket_info = rest.get_bucket(bucket) # todo raise exception for not bucket_info client.sasl_auth_plain(bucket_info.name.encode("ascii"), bucket_info.saslPassword.encode("ascii")) return client
def get_vBuckets_info(master): """ return state and count items for all vbuckets for each node format: dict: {u'1node_ip1': {'vb_79': ['replica', '0'], 'vb_78': ['active', '0']..}, u'1node_ip1':....} """ rest = RestConnection(master) port = rest.get_nodes_self().memcached nodes = rest.node_statuses() _nodes_stats = {} for node in nodes: stat = {} buckets = [] _server = {"ip": node.ip, "port": node.port, "username": master.rest_username, "password": master.rest_password} try: buckets = rest.get_buckets() mc = MemcachedClient(node.ip, port) stat_hash = mc.stats("hash") except Exception: if not buckets: log.error("There are not any buckets in {0}:{1} node".format(node.ip, node.port)) else: log.error("Impossible to get vBucket's information for {0}:{1} node".format(node.ip, node.port)) _nodes_stats[node.ip + ":" + str(node.port)] continue mc.close() vb_names = [key[:key.index(":")] for key in list(stat_hash.keys())] for name in vb_names: stat[name] = [stat_hash[name + ":state"], stat_hash[name + ":counted"]] _nodes_stats[node.ip + ":" + str(port)] = stat log.info(_nodes_stats) return _nodes_stats
def wait_for_mc_stats(master, bucket, stat_key, stat_value, timeout_in_seconds=120, verbose=True): log = logger.get("infra") log.info("Waiting for bucket {0} stat: {1} to match {2} on {3}".format( bucket, stat_key, stat_value, master.ip)) start = time.time() verified = False while (time.time() - start) <= timeout_in_seconds: c = MemcachedClient(master.ip, constants.memcached_port) stats = c.stats() c.close() if stats and stat_key in stats \ and str(stats[stat_key]) == str(stat_value): log.info("{0} : {1}".format(stat_key, stats[stat_key])) verified = True break else: if stats and stat_key in stats: if verbose: log.info("{0} : {1}".format(stat_key, stats[stat_key])) sleep_time = 2 if not verbose: sleep_time = 0.1 sleep(sleep_time, log_type="infra") return verified
def wait_for_mc_stats(master, bucket, stat_key, stat_value, timeout_in_seconds=120, verbose=True): log.info("waiting for bucket {0} stat : {1} to match {2} on {3}".format(bucket, stat_key, \ stat_value, master.ip)) start = time.time() verified = False while (time.time() - start) <= timeout_in_seconds: c = MemcachedClient(master.ip, 11210) stats = c.stats() c.close() if stats and stat_key in stats and str( stats[stat_key]) == str(stat_value): log.info("{0} : {1}".format(stat_key, stats[stat_key])) verified = True break else: if stats and stat_key in stats: if verbose: log.info("{0} : {1}".format(stat_key, stats[stat_key])) if not verbose: time.sleep(0.1) else: time.sleep(2) return verified
def get_mb_stats(servers, key): for server in servers: c = MemcachedClient(server.ip, 11210) log.info("Get flush param on server {0}, {1}".format(server, key)) value = c.stats().get(key, None) log.info("Get flush param on server {0}, {1}".format(server, value)) c.close()
def test_vbucket_id_option(self): bucket = RestConnection(self.server_origin).get_bucket(self.buckets[0]) self.num_items = self.num_items - (self.num_items % len(bucket.vbuckets)) num_items_per_vb = self.num_items / len(bucket.vbuckets) template = '{{ "mutated" : 0, "age": {0}, "first_name": "{1}" }}' gen_load = DocumentGenerator('cbtransfer', template, range(5), ['james', 'john'], start=0, end=self.num_items) client = MemcachedClient(self.server_origin.ip, int(bucket.vbuckets[0].master.split(':')[1])) kv_value_dict = {} vb_id_to_check = bucket.vbuckets[-1].id for vb_id in xrange(len(bucket.vbuckets)): cur_items_per_vb = 0 while cur_items_per_vb < num_items_per_vb: key, value = gen_load.next() client.set(key, 0, 0, value, vb_id) if vb_id_to_check == vb_id: kv_value_dict[key] = value cur_items_per_vb += 1 transfer_source = 'http://%s:%s' % (self.server_origin.ip, self.server_origin.port) transfer_destination = 'http://%s:%s' % (self.server_recovery.ip, self.server_recovery.port) output = self.shell.execute_cbtransfer(transfer_source, transfer_destination, "-b %s -B %s -i %s" % (bucket.name, bucket.name, vb_id_to_check)) client = MemcachedClient(self.server_recovery.ip, int(bucket.vbuckets[0].master.split(':')[1])) for key, value in kv_value_dict.iteritems(): _, _, d = client.get(key, vbucket=vb_id_to_check) self.assertEquals(d, value, 'Key: %s expected. Value expected %s. Value actual %s' % ( key, value, d))
def get_mb_stats(servers, key): log = logger.get("infra") for server in servers: c = MemcachedClient(server.ip, constants.memcached_port) log.info("Get flush param on server {0}, {1}".format(server, key)) val = c.stats().get(key, None) log.info("Get flush param on server {0}, {1}".format(server, val)) c.close()
def get_mb_stats(servers, key): log = logger.Logger.get_logger() for server in servers: c = MemcachedClient(server.ip, 11210) log.info("Get flush param on server {0}, {1}".format(server, key)) value = c.stats().get(key, None) log.info("Get flush param on server {0}, {1}".format(server, value)) c.close()
def flushctl_set(servers, key, val): log = logger.Logger.get_logger() for server in servers: c = MemcachedClient(server.ip, 11210) log.info("Setting flush param on server {0}, {1} to {2}".format(server, key, val)) rv = c.set_flush_param(key, val) log.info("Setting flush param on server {0}, {1} to {2}, result: {3}".format(server, key, val, rv)) c.close()
def test_list_mechs(self): nodes = RestConnection(self.master).get_nodes() for n in nodes: if n.ip == self.master.ip and n.port == self.master.port: node = n client = MemcachedClient(self.master.ip, node.memcached) mechs = list(client.sasl_mechanisms()) assert "CRAM-MD5" in mechs assert "PLAIN" in mechs assert len(list(mechs)) == 2
def connection(self, client_ip, bucket_name, user,password, port=11210): log.info("Bucket name for connection is ---- {0}, username -- {1}, ----- password -- {2}".format(bucket_name,user, \ password)) try: mc = MemcachedClient(host=client_ip, port=port) mc.sasl_auth_plain(user,password) mc.bucket_select(bucket_name) return mc, True except Exception as e: log.info( "Exception is from connection function {0}".format(e)) return False, False
def load_one_mutation_into_source_vb0(self, vb0_active_src_node): key = self.vb0_keys[self.key_counter] memc_client = MemcachedClient(vb0_active_src_node.ip, 11210) try: memc_client.set(key, exp=0, flags=0, val="dummy val") self.key_counter += 1 self.keys_loaded.append(key) self.log.info("Loaded key {} onto vb0 in {}".format(key, vb0_active_src_node.ip)) self.log.info ("deleted, flags, exp, rev_id, cas for key {} = {}".format(key, memc_client.getMeta(key))) except MemcachedError as e: self.log.error(e)
def __init__(self, host=cfg.COUCHBASE_IP, port=11210): self.host = host self.port = port self.client = None try: self.client = MemcachedClient(self.host, self.port) except MemcachedError as ex: msg = "error connecting to host %s for gathering postconditions"\ % self.host logger.error(msg) pass
def test_list_mechs(self): nodes = RestConnection(self.master).get_nodes() for n in nodes: if n.ip == self.master.ip and n.port == self.master.port: node = n client = MemcachedClient(self.master.ip, node.memcached) mechs = list(client.sasl_mechanisms()) self.log.info("Start check mech types") assert "SCRAM-SHA1" in mechs assert "SCRAM-SHA256" in mechs assert "SCRAM-SHA512" in mechs assert "PLAIN" in mechs assert len(list(mechs)) == 4
def test_list_mechs(self): nodes = RestConnection(self.master).get_nodes() for n in nodes: if n.ip == self.master.ip and n.port == self.master.port: node = n client = MemcachedClient(self.master.ip, node.memcached) mechs = list(client.sasl_mechanisms()) self.log.info("Start check mech types") assert b"SCRAM-SHA1" in mechs assert b"SCRAM-SHA256" in mechs assert b"SCRAM-SHA512" in mechs assert b"PLAIN" in mechs assert len(list(mechs)) == 4
def load_one_mutation_into_source_vb0(self, vb0_active_src_node): key = self.vb0_keys[self.key_counter] memc_client = MemcachedClient(vb0_active_src_node.ip, 11210) try: memc_client.set(key, exp=0, flags=0, val="dummy val") self.key_counter += 1 self.keys_loaded.append(key) self.log.info("Loaded key {0} onto vb0 in {1}".format( key, vb0_active_src_node.ip)) self.log.info( "deleted, flags, exp, rev_id, cas for key {0} = {1}".format( key, memc_client.getMeta(key))) except MemcachedError as e: self.log.error(e)
def wait_for_warmup(self, host, port): if self.bucket_type == 'ephemeral': return while True: client = McdClient(host, port) try: client.bucket_select("default") response = client.stats() # check the old style or new style (as of 4.5) results mode = response.get('ep_degraded_mode') if mode is not None: if mode == '0' or mode == 'false': break except Exception as ex: pass self.sleep(1)
class EPStatChecker(StatChecker): def __init__(self, host=cfg.COUCHBASE_IP, port=11210): self.host = host self.port = port self.client = None try: self.client = MemcachedClient(self.host, self.port) except MemcachedError as ex: msg = "error connecting to host %s for gathering postconditions"\ % self.host logger.error(msg) pass @property def stat_keys(self): keys = [] stats = self.get_stats() if stats: keys = list(stats.keys()) return keys def get_stats(self): stats = None if self.client: try: stats = self.client.stats() except Exception: logger.info("unable to get stats from host: %s:%s" %\ (self.host, self.port)) return stats
def _get_clients(self, bucket): for vbucket in bucket.vbuckets: if vbucket.master not in self.clients: ip, port = vbucket.master.split(':') if bucket.port: port = bucket.port self.clients[vbucket.master] = MemcachedClient(ip, int(port))
def do_auth(self, bucket, password): """ default self.auth_mech is 'PLAIN' """ self.log.info("Authenticate with {0} to {1}:{2}".format( self.auth_mech, bucket, password)) ret = None nodes = RestConnection(self.master).get_nodes() for n in nodes: if n.ip == self.master.ip and n.port == self.master.port: node = n client = MemcachedClient(self.master.ip, node.memcached) try: if self.auth_mech == "PLAIN": ret = client.sasl_auth_plain(bucket, password)[2] else: self.fail("Invalid auth mechanism {0}".format(self.auth_mech)) except MemcachedError, e: ret = e[0].split(' for vbucket')[0]
def test_vbucket_id_option(self): bucket = RestConnection(self.server_origin).get_bucket(self.buckets[0]) self.num_items = self.num_items - (self.num_items % len(bucket.vbuckets)) num_items_per_vb = self.num_items // len(bucket.vbuckets) template = '{{ "mutated" : 0, "age": {0}, "first_name": "{1}" }}' gen_load = DocumentGenerator('cbtransfer', template, list(range(5)), ['james', 'john'], start=0, end=self.num_items) client = MemcachedClient(self.server_origin.ip, int(bucket.vbuckets[0].master.split(':')[1])) kv_value_dict = {} vb_id_to_check = bucket.vbuckets[-1].id for vb_id in range(len(bucket.vbuckets)): cur_items_per_vb = 0 while cur_items_per_vb < num_items_per_vb: key, value = next(gen_load) client.set(key, 0, 0, value, vb_id) if vb_id_to_check == vb_id: kv_value_dict[key] = value cur_items_per_vb += 1 transfer_source = 'http://%s:%s' % (self.server_origin.ip, self.server_origin.port) transfer_destination = 'http://%s:%s' % (self.server_recovery.ip, self.server_recovery.port) output = self.shell.execute_cbtransfer(transfer_source, transfer_destination, "-b %s -B %s -i %s" % (bucket.name, bucket.name, vb_id_to_check)) client = MemcachedClient(self.server_recovery.ip, int(bucket.vbuckets[0].master.split(':')[1])) for key, value in kv_value_dict.items(): _, _, d = client.get(key, vbucket=vb_id_to_check) self.assertEqual(d, value, 'Key: %s expected. Value expected %s. Value actual %s' % ( key, value, d))
def addServer(self, ipaddr): try: mc = MemcachedClient(host=ipaddr, port=11211) self.servers.append(mc) return True except Exception, e: log("Unable to add server %s (%s)" % (ipaddr, str(e))) sys.exit(1) return False
def do_auth(self, bucket, password): """ default self.auth_mech is 'PLAIN' """ self.log.info("Authenticate with {0} to {1}:{2}".format(self.auth_mech, bucket, password)) ret = None nodes = RestConnection(self.master).get_nodes() for n in nodes: if n.ip == self.master.ip and n.port == self.master.port: node = n client = MemcachedClient(self.master.ip, node.memcached) try: if self.auth_mech == "PLAIN": ret = client.sasl_auth_plain(bucket, password)[2] else: self.fail("Invalid auth mechanism {0}".format(self.auth_mech)) except MemcachedError, e: ret = e[0].split(' for vbucket')[0]
def vbMcdClient(self, vb): server = self.serverVb(vb) if server in self.mcdClientMap: client = self.mcdClientMap[server] else: ip, port = server.split(':') client = MemcachedClient(ip, int(port)) self.mcdClientMap[server] = client return client
def direct_client(server, bucket, timeout=30): rest = RestConnection(server) node = rest.get_nodes_self() log = logger.Logger.get_logger() if isinstance(server, dict): log.info("dict:{0}".format(server)) log.info("creating direct client {0}:{1} {2}".format(server["ip"], node.memcached, bucket)) else: log.info("creating direct client {0}:{1} {2}".format(server.ip, node.memcached, bucket)) RestHelper(rest).vbucket_map_ready(bucket, 60) vBuckets = RestConnection(server).get_vbuckets(bucket) if isinstance(server, dict): client = MemcachedClient(server["ip"], node.memcached, timeout=timeout) else: client = MemcachedClient(server.ip, node.memcached, timeout=timeout) client.vbucket_count = len(vBuckets) bucket_info = rest.get_bucket(bucket) #todo raise exception for not bucket_info client.sasl_auth_plain(bucket_info.name.encode('ascii'), bucket_info.saslPassword.encode('ascii')) return client
def verify_revid(self): missing_keys = False src_node = self.get_active_vb0_node(self.src_master) dest_node = self.get_active_vb0_node(self.dest_master) src_client = MemcachedClient(src_node.ip, 11210) dest_client = MemcachedClient(dest_node.ip, 11210) for key in self.keys_loaded: try: src_meta = src_client.getMeta(key) dest_meta = dest_client.getMeta(key) self.log.info( "deleted, flags, exp, rev_id, cas for key from Source({0}) {1} = {2}" .format(src_node.ip, key, src_meta)) self.log.info( "deleted, flags, exp, rev_id, cas for key from Destination({0}) {1} = {2}" .format(dest_node.ip, key, dest_meta)) if src_meta == dest_meta: self.log.info( "RevID verification successful for key {0}".format( key)) else: self.fail( "RevID verification failed for key {0}".format(key)) except MemcachedError as e: self.log.error("Key {0} threw {1} on getMeta()".format(key, e)) missing_keys = True if missing_keys: self.fail("Some keys are missing at destination")
def wait_for_mc_stats(master, bucket, stat_key, stat_value, timeout_in_seconds=120, verbose=True): log.info("waiting for bucket {0} stat : {1} to match {2} on {3}".format(bucket, stat_key, \ stat_value, master.ip)) start = time.time() verified = False while (time.time() - start) <= timeout_in_seconds: c = MemcachedClient(master.ip, 11210) stats = c.stats() c.close() if stats and stat_key in stats and str(stats[stat_key]) == str(stat_value): log.info("{0} : {1}".format(stat_key, stats[stat_key])) verified = True break else: if stats and stat_key in stats: if verbose: log.info("{0} : {1}".format(stat_key, stats[stat_key])) if not verbose: time.sleep(0.1) else: time.sleep(2) return verified
def verify_stat(self, items, value="active"): mc = MemcachedClient(self.cluster.master.ip, constants.memcached_port) mc.sasl_auth_plain(self.cluster.master.rest_username, self.cluster.master.rest_password) mc.bucket_select('default') stats = mc.stats() self.assertEquals(stats['ep_compression_mode'], value) self.assertEquals(int(stats['ep_item_compressor_num_compressed']), items) self.assertNotEquals(int(stats['vb_active_itm_memory']), int(stats['vb_active_itm_memory_uncompressed']))
def wait_for_mc_stats_no_timeout(master, bucket, stat_key, stat_value, timeout_in_seconds=-1, verbose=True): log.info("waiting for bucket {0} stat : {1} to match {2} on {3}".format(bucket, stat_key, \ stat_value, master.ip)) c = MemcachedClient(master.ip, 11210) stats = c.stats() c.close() while str(stats[stat_key]) != str(stat_value): c = MemcachedClient(master.ip, 11210) stats = c.stats() c.close() if verbose: log.info("{0} : {1}".format(stat_key, stats[stat_key])) time.sleep(5) return True
def _build_conns(self): """build separate connections based on store""" if not self.store: print "<%s> failed to build connections, invalid store object"\ % self.__class__.__name__ return False if self.store.__class__.__name__ == "StoreMemcachedBinary": conn = MemcachedClient(self.store.conn.host, self.store.conn.port) server_str = "{0}:{1}".format(self.store.conn.host, self.store.conn.port) self.conns[server_str] = conn elif self.store.__class__.__name__ == "StoreMembaseBinary": for memcached in self.store.awareness.memcacheds.itervalues(): conn = MemcachedClient(memcached.host, memcached.port) server_str = "{0}:{1}".format(conn.host, conn.port) self.conns[server_str] = conn self.awareness = self.store.awareness else: print "<%s> error: unsupported store object %s" %\ (self.__class__.__name__, store.__class__.__name__) return False return True
def _add_conn(self, server): if not self.store: print "<%s> failed to add conn, invalid store object"\ % self.__class__.__name__ return False if self.store.__class__.__name__ == "StoreMembaseBinary": print "<%s> _add_conn: %s"\ % (self.__class__.__name__, server) host, port = server.split(":") conn = MemcachedClient(host, int(port)) self.conns[server] = conn return True
def init_clients(self): """Initialise clients for all servers there are vBuckets on It returns a dict with 'ip:port' as key (this information is also stored this way in every vBucket in the `master` property) and the MemcachedClient as the value """ clients = {} for vbucket in self.bucket.vbuckets: if vbucket.master not in clients: ip, port = vbucket.master.split(':') clients[vbucket.master] = MemcachedClient(ip, int(port)) return clients
def getDirectMC(key, ip, port = 8091, bucket = "default", password = ""): real_mc_client = None # get initial mc client client = MemcachedClient(ip, int(port)) vbId = (((zlib.crc32(key)) >> 16) & 0x7fff) & (client.vbucket_count - 1) # get vbucket map rest = create_rest(ip, port) vbuckets = rest.get_vbuckets(bucket) # find vbucket responsible to this key and mapping host if vbuckets is not None: vbucket = [vbucket for vbucket in vbuckets if vbucket.id == vbId] if len(vbucket) == 1: mc_ip, mc_port = vbucket[0].master.split(":") real_mc_client = MemcachedClient(mc_ip, int(mc_port)) real_mc_client.sasl_auth_plain(bucket, password) return real_mc_client
def verify_revid(self): missing_keys = False src_node = self.get_active_vb0_node(self.src_master) dest_node = self.get_active_vb0_node(self.dest_master) src_client = MemcachedClient(src_node.ip, 11210) dest_client = MemcachedClient(dest_node.ip, 11210) for key in self.keys_loaded: try: src_meta = src_client.getMeta(key) dest_meta = dest_client.getMeta(key) self.log.info("deleted, flags, exp, rev_id, cas for key from Source({0}) {1} = {2}" .format(src_node.ip, key, src_meta)) self.log.info("deleted, flags, exp, rev_id, cas for key from Destination({0}) {1} = {2}" .format(dest_node.ip, key, dest_meta)) if src_meta == dest_meta: self.log.info("RevID verification successful for key {}".format(key)) else: self.fail("RevID verification failed for key {}".format(key)) except MemcachedError: self.log.error("Key {} is missing at destination".format(key)) missing_keys = True if missing_keys: self.fail("Some keys are missing at destination")
def getDirectMC(key, ip, port=8091, bucket="default", password=""): real_mc_client = None # get initial mc client client = MemcachedClient(ip, int(port)) vbId = (((zlib.crc32(key)) >> 16) & 0x7fff) & (client.vbucket_count - 1) # get vbucket map rest = create_rest(ip, port) vbuckets = rest.get_vbuckets(bucket) # find vbucket responsible to this key and mapping host if vbuckets is not None: vbucket = [vbucket for vbucket in vbuckets if vbucket.id == vbId] if len(vbucket) == 1: mc_ip, mc_port = vbucket[0].master.split(":") real_mc_client = MemcachedClient(mc_ip, int(mc_port)) real_mc_client.sasl_auth_plain(bucket, password) return real_mc_client
def __init__(self, host = cfg.COUCHBASE_IP, port = 11210): self.host = host self.port = port self.client = None try: self.client = MemcachedClient(self.host, self.port) except MemcachedError as ex: msg = "error connecting to host %s for gathering postconditions"\ % self.host logger.error(msg) pass
def load_one_mutation_into_source_vb0(self, vb0_active_src_node): key = self.vb0_keys[self.key_counter] memc_client = MemcachedClient(vb0_active_src_node.ip, 11210) memc_client.sasl_auth_plain("cbadminbucket","password") memc_client.bucket_select("default") try: memc_client.set(key, exp=0, flags=0, val="dummy val") self.key_counter += 1 self.keys_loaded.append(key) self.log.info("Loaded key {0} onto vb0 in {1}".format(key, vb0_active_src_node.ip)) self.log.info ("deleted, flags, exp, rev_id, cas for key {0} = {1}".format(key, memc_client.getMeta(key))) except MemcachedError as e: self.log.error(e)
def wait_for_vbuckets_ready_state(node, bucket, timeout_in_seconds=300, log_msg=''): log = logger.Logger.get_logger() start_time = time.time() end_time = start_time + timeout_in_seconds ready_vbuckets = {} rest = RestConnection(node) servers = rest.get_nodes() RestHelper(rest).vbucket_map_ready(bucket, 60) vbucket_count = len(rest.get_vbuckets(bucket)) vbuckets = rest.get_vbuckets(bucket) obj = VBucketAwareMemcached(rest, bucket) memcacheds, vbucket_map, vbucket_map_replica = obj.request_map( rest, bucket) #Create dictionary with key:"ip:port" and value: a list of vbuckets server_dict = defaultdict(list) for everyID in range(0, vbucket_count): memcached_ip_port = str(vbucket_map[everyID]) server_dict[memcached_ip_port].append(everyID) while time.time() < end_time and len(ready_vbuckets) < vbucket_count: for every_ip_port in server_dict: #Retrieve memcached ip and port ip, port = every_ip_port.split(":") client = MemcachedClient(ip, int(port), timeout=30) client.vbucket_count = len(vbuckets) bucket_info = rest.get_bucket(bucket) client.sasl_auth_plain( bucket_info.name.encode('ascii'), bucket_info.saslPassword.encode('ascii')) for i in server_dict[every_ip_port]: try: (a, b, c) = client.get_vbucket_state(i) except mc_bin_client.MemcachedError as e: log.error("%s: %s" % (log_msg, e)) continue if c.find("\x01") > 0 or c.find("\x02") > 0: ready_vbuckets[i] = True elif i in ready_vbuckets: log.warning( "vbucket state changed from active to {0}".format( c)) del ready_vbuckets[i] client.close() return len(ready_vbuckets) == vbucket_count
def do_get_random_key(self): # MB-31548, get_Random key gets hung sometimes. mc = MemcachedClient(self.cluster.master.ip, constants.memcached_port) mc.sasl_auth_plain(self.cluster.master.rest_username, self.cluster.master.rest_password) mc.bucket_select('default') count = 0 while count < 1000000: count += 1 try: mc.get_random_key() except MemcachedError as error: self.fail("<MemcachedError #%d ``%s''>" % (error.status, error.message)) if count % 1000 == 0: self.log.info('The number of iteration is {}'.format(count))
def __init__(self, vbs_server): cmd = "curl -s -L http://%s/%s" % (vbs_server, consts.VBS_API_PATH) status, output = getcommandoutput(cmd) if status > 0: log("ERROR: Unable to contact VBS: %s" % (vbs_server)) sys.exit(1) j = json.loads(output) vbsmap = j['buckets'][0]['vBucketServerMap']['vBucketMap'] srvlist = j['buckets'][0]['vBucketServerMap']['serverList'] self.vbmap = {} for i, v in enumerate(vbsmap): ip, port = srvlist[v[0]].split(':') mc = MemcachedClient(host=ip, port=int(port)) self.vbmap[i] = mc
def connection(self, client_ip, bucket_name, user,password, port=11210): log.info("Bucket name for connection is ---- {0}, username -- {1}, ----- password -- {2}".format(bucket_name, user, \ password)) try: mc = MemcachedClient(host=client_ip, port=port) mc.sasl_auth_plain(user, password) mc.bucket_select(bucket_name) return mc, True except Exception as e: log.info( "Exception is from connection function {0}".format(e)) return False, False
def wait_for_vbuckets_ready_state(node, bucket, timeout_in_seconds=300, log_msg=''): log = logger.Logger.get_logger() start_time = time.time() end_time = start_time + timeout_in_seconds ready_vbuckets = {} rest = RestConnection(node) servers = rest.get_nodes() RestHelper(rest).vbucket_map_ready(bucket, 60) vbucket_count = len(rest.get_vbuckets(bucket)) vbuckets = rest.get_vbuckets(bucket) obj = VBucketAwareMemcached(rest, bucket) memcacheds, vbucket_map, vbucket_map_replica = obj.request_map(rest, bucket) #Create dictionary with key:"ip:port" and value: a list of vbuckets server_dict = defaultdict(list) for everyID in range(0, vbucket_count): memcached_ip_port = str(vbucket_map[everyID]) server_dict[memcached_ip_port].append(everyID) while time.time() < end_time and len(ready_vbuckets) < vbucket_count: for every_ip_port in server_dict: #Retrieve memcached ip and port ip, port = every_ip_port.split(":") client = MemcachedClient(ip, int(port), timeout=30) client.vbucket_count = len(vbuckets) bucket_info = rest.get_bucket(bucket) client.sasl_auth_plain(bucket_info.name.encode('ascii'), bucket_info.saslPassword.encode('ascii')) for i in server_dict[every_ip_port]: try: (a, b, c) = client.get_vbucket_state(i) except mc_bin_client.MemcachedError as e: ex_msg = str(e) if "Not my vbucket" in log_msg: log_msg = log_msg[:log_msg.find("vBucketMap") + 12] + "..." if "Not my vbucket" in ex_msg: #reduce output ex_msg = str(e)[:str(e).find('Not my vbucket') + 14] + "..." log.error("%s: %s" % (log_msg, ex_msg)) continue if c.find("\x01") > 0 or c.find("\x02") > 0: ready_vbuckets[i] = True elif i in ready_vbuckets: log.warning("vbucket state changed from active to {0}".format(c)) del ready_vbuckets[i] client.close() return len(ready_vbuckets) == vbucket_count
class EPStatChecker(StatChecker): def __init__(self, host = cfg.COUCHBASE_IP, port = 11210): self.host = host self.port = port self.client = None try: self.client = MemcachedClient(self.host, self.port) except MemcachedError as ex: msg = "error connecting to host %s for gathering postconditions"\ % self.host logger.error(msg) pass @property def stat_keys(self): keys = [] stats = self.get_stats() if stats: keys = stats.keys() return keys def get_stats(self): stats = None if self.client: try: stats = self.client.stats() except Exception: logger.info("unable to get stats from host: %s:%s" %\ (self.host, self.port)) return stats
def client_helper(self, node, type_, vbucket): self.assertTrue(type_ in (MCD, DCP), msg="type not found") client = None ip = None port = None rest = RestConnection(node) if vbucket is not None: host = self.vbucket_host(rest, vbucket) ip = host.split(':')[0] port = int(host.split(':')[1]) else: client_node = rest.get_nodes_self() ip = client_node.hostname.split(':')[0] port = client_node.memcached if type_ == MCD: client = MemcachedClient(ip, port) else: client = DcpClient(ip, port) return client
def wait_for_mc_stats_no_timeout(master, bucket, stat_key, stat_value, timeout_in_seconds=-1, verbose=True): log.info("waiting for bucket {0} stat : {1} to match {2} on {3}".format(bucket, stat_key, \ stat_value, master.ip)) # keep retrying until reaches the server stats = {} while not stats: try: c = MemcachedClient(master.ip, 11210) c.sasl_auth_plain(bucket, '') stats = c.stats() except Exception as e: log.info("Exception: {0}, retry in 2 seconds ...".format( str(e))) stats = {} time.sleep(2) finally: c.close() while str(stats[stat_key]) != str(stat_value): c = MemcachedClient(master.ip, 11210) c.sasl_auth_plain(bucket, '') stats = c.stats() c.close() if verbose: log.info("{0} : {1}".format(stat_key, stats[stat_key])) time.sleep(5) return True
def verify_revid(self): missing_keys = False src_node = self.get_active_vb0_node(self.src_master) dest_node = self.get_active_vb0_node(self.dest_master) src_client = MemcachedClient(src_node.ip, 11210) dest_client = MemcachedClient(dest_node.ip, 11210) src_client.sasl_auth_plain("cbadminbucket","password") src_client.bucket_select("default") dest_client.sasl_auth_plain("cbadminbucket","password") dest_client.bucket_select("default") for key in self.keys_loaded: try: src_meta = src_client.getMeta(key) dest_meta = dest_client.getMeta(key) self.log.info("deleted, flags, exp, rev_id, cas for key from Source({0}) {1} = {2}" .format(src_node.ip, key, src_meta)) self.log.info("deleted, flags, exp, rev_id, cas for key from Destination({0}) {1} = {2}" .format(dest_node.ip, key, dest_meta)) if src_meta == dest_meta: self.log.info("RevID verification successful for key {0}".format(key)) else: self.fail("RevID verification failed for key {0}".format(key)) except MemcachedError as e: self.log.error("Key {0} threw {1} on getMeta()".format(key, e)) missing_keys = True if missing_keys: self.fail("Some keys are missing at destination")
parser.add_option("-n", "--node", dest="node", default="127.0.0.1", help="couchdb ip , defaults to 127.0.0.1:5984") parser.add_option("-s", "--sleep", dest="sleep", default="600", help="sleep time before each iteration") parser.add_option("-i", "--items", dest="items", default="10000", help="number of items") options, args = parser.parse_args() node = options.node sleep_time = int(options.sleep) prefix = str(uuid.uuid4()) number_of_items = int(options.items) mc = MemcachedClient("127.0.0.1", 11211) keys = ["{0}-{1}".format(prefix, i) for i in range(0, number_of_items)] info("inserting {0} items".format(number_of_items)) for k in keys: mc.set(k, 0, 0, str(uuid.uuid4())[0:16]) while True: info("now remove 3 chars from 80% of keys - if < 3 chars delete the key - if key does not exist create it") for i in range(0, 3): for k in keys: try: a, b, value = mc.get(k) if len(value) < 3: mc.delete(k) else:
queue = Queue(maxsize=10000) server = ServerInfo() server.ip = "10.17.12.20" bucket = {'name': 'default', 'port': 11220, 'password': ''} #vam = VBucketAwareMemcached(RestConnection(server), bucket) #print vam.memcacheds #print vam.vBucketMap payload = MemcachedClientHelper.create_value('*', 10240) keys = ["key_%d" % (i) for i in range(4000)] #keys = ["key_%s_%d" % (str(uuid.uuid4()), i) for i in range(4)] total_size = 0 #mc = MemcachedClientHelper.create_memcached_client("172.16.75.128","default",11210,"default") mc = MemcachedClient("10.17.12.20", 11210) #for key in keys: # vam.memcached(key).set(key, 1, 0, payload) # total_size += len(key) + len(payload) + 200 #time.sleep(10) #for i in range(0,1023): # mc.set_vbucket_state(i, 'active') new_thread = TapListener(queue, server) new_thread.start() i = 0 while i < 4000: for key in keys: # vam.memcached(key).get(key) mc.set(key, 10, 0, payload, vbucket=0)
def wait_for_vbuckets_ready_state(node, bucket, timeout_in_seconds=300, log_msg='', admin_user='******', admin_pass='******'): log = logger.Logger.get_logger() start_time = time.time() end_time = start_time + timeout_in_seconds ready_vbuckets = {} rest = RestConnection(node) servers = rest.get_nodes() RestHelper(rest).vbucket_map_ready(bucket, 60) vbucket_count = len(rest.get_vbuckets(bucket)) vbuckets = rest.get_vbuckets(bucket) obj = VBucketAwareMemcached(rest, bucket) memcacheds, vbucket_map, vbucket_map_replica = obj.request_map(rest, bucket) #Create dictionary with key:"ip:port" and value: a list of vbuckets server_dict = defaultdict(list) for everyID in range(0, vbucket_count): memcached_ip_port = str(vbucket_map[everyID]) server_dict[memcached_ip_port].append(everyID) while time.time() < end_time and len(ready_vbuckets) < vbucket_count: for every_ip_port in server_dict: #Retrieve memcached ip and port ip = every_ip_port.rsplit(":", 1)[0] port = every_ip_port.rsplit(":", 1)[1] client = MemcachedClient(ip, int(port), timeout=30) client.vbucket_count = len(vbuckets) bucket_info = rest.get_bucket(bucket) cluster_compatibility = rest.check_cluster_compatibility("5.0") if cluster_compatibility is None: pre_spock = True else: pre_spock = not cluster_compatibility if pre_spock: log.info("Atleast 1 of the server is on pre-spock " "version. Using the old ssl auth to connect to " "bucket.") client.sasl_auth_plain( bucket_info.name.encode('ascii'), bucket_info.saslPassword.encode('ascii')) else: client.sasl_auth_plain(admin_user, admin_pass) bucket = bucket.encode('ascii') client.bucket_select(bucket) for i in server_dict[every_ip_port]: try: (a, b, c) = client.get_vbucket_state(i) except mc_bin_client.MemcachedError as e: ex_msg = str(e) if "Not my vbucket" in log_msg: log_msg = log_msg[:log_msg.find("vBucketMap") + 12] + "..." if e.status == memcacheConstants.ERR_NOT_MY_VBUCKET: # May receive this while waiting for vbuckets, continue and retry...S continue log.error("%s: %s" % (log_msg, ex_msg)) continue except exceptions.EOFError: # The client was disconnected for some reason. This can # happen just after the bucket REST API is returned (before # the buckets are created in each of the memcached processes.) # See here for some details: http://review.couchbase.org/#/c/49781/ # Longer term when we don't disconnect clients in this state we # should probably remove this code. log.error("got disconnected from the server, reconnecting") client.reconnect() client.sasl_auth_plain(bucket_info.name.encode('ascii'), bucket_info.saslPassword.encode('ascii')) continue if c.find("\x01") > 0 or c.find("\x02") > 0: ready_vbuckets[i] = True elif i in ready_vbuckets: log.warning("vbucket state changed from active to {0}".format(c)) del ready_vbuckets[i] client.close() return len(ready_vbuckets) == vbucket_count
def storeJob(jobDoc, bucket, first_pass = True): client = McdClient(HOST, PORT) client.sasl_auth_plain(bucket, "") doc = jobDoc url = doc["url"] res = getJS(url, {"depth" : 0}).json() if res is None: return buildHist = {} if res["lastBuild"]: bids = [b["number"] for b in res["builds"]] lastTotalCount = -1 idx=0 for bid in bids: idx = idx + 1 i = 1 if idx < len(bids): while bids[idx] != bid-i: key = "%s-%s" % (doc["name"], bid-i) key = hashlib.md5(key).hexdigest() try: client.delete(key, vbucket=0) except: pass i = i + 1 if bid in JOBS[doc["name"]]: continue # job already stored else: if first_pass == False: JOBS[doc["name"]].append(bid) doc["build_id"] = bid res = getJS(url+str(bid), {"depth" : 0}).json() if res is None: return if "result" not in res: continue doc["result"] = res["result"] doc["duration"] = res["duration"] if bucket == "server": if res["result"] not in ["SUCCESS", "UNSTABLE", "FAILURE", "ABORTED"]: continue # unknown result state actions = res["actions"] totalCount = getAction(actions, "totalCount") or 0 failCount = getAction(actions, "failCount") or 0 skipCount = getAction(actions, "skipCount") or 0 if totalCount == 0: if lastTotalCount == -1: continue # no tests ever passed for this build else: totalCount = lastTotalCount failCount = totalCount else: lastTotalCount = totalCount doc["failCount"] = failCount doc["totalCount"] = totalCount - skipCount params = getAction(actions, "parameters") if params is None: doc["priority"] = P1 doc["build"] = DEFAULT_BUILD else: doc["build"] = getAction(params, "name", "version_number") or getAction(params, "name", "cluster_version") or DEFAULT_BUILD doc["priority"] = getAction(params, "name", "priority") or P1 if doc["priority"].upper() not in [P0, P1, P2]: doc["priority"] = P1 doc["build"] = doc["build"].replace("-rel","").split(",")[0] try: _build= doc["build"].split("-") rel, bno = _build[0], _build[1] # check partial rel #'s rlen = len(rel.split(".")) while rlen < 3: rel = rel+".0" rlen+=1 # verify rel, build m=re.match("^\d\.\d\.\d{1,5}", rel) if m is None: print "unsupported version_number: "+doc["build"] continue m=re.match("^\d{1,10}", bno) if m is None: print "unsupported version_number: "+doc["build"] continue doc["build"] = "%s-%s" % (rel, bno.zfill(4)) except: print "unsupported version_number: "+doc["build"] continue else: # use date as version for sdk and mobile if res["result"] not in ["SUCCESS", "UNSTABLE", "FAILURE", "ABORTED"]: continue actions = res["actions"] totalCount = getAction(actions, "totalCount") or 0 failCount = getAction(actions, "failCount") or 0 skipCount = getAction(actions, "skipCount") or 0 if totalCount == 0: if lastTotalCount == -1: continue # no tests ever passed for this build else: totalCount = lastTotalCount failCount = totalCount else: lastTotalCount = totalCount doc["failCount"] = failCount doc["totalCount"] = totalCount - skipCount doc["priority"] = P0 ts = res["timestamp"]/1000; month = int(datetime.datetime.fromtimestamp(ts).strftime("%m")) _ts = datetime.datetime.fromtimestamp(ts).strftime("%Y-%m%d") yr, md = _ts.split("-") doc["build"] = "%s-%s%s" % (MOBILE_VERSION, yr[-1], md) if doc["build"] in buildHist: #print "REJECTED- doc already in build results: %s" % doc #print buildHist # attempt to delete if this record has been stored in couchbase try: oldKey = "%s-%s" % (doc["name"], doc["build_id"]) oldKey = hashlib.md5(oldKey).hexdigest() client.delete(oldKey, vbucket = 0) #print "DELETED- %s:%s" % (doc["build"],doc["build_id"]) except: pass continue # already have this build results key = "%s-%s" % (doc["name"], doc["build_id"]) key = hashlib.md5(key).hexdigest() val = json.dumps(doc) try: #print val client.set(key, 0, 0, val, 0) buildHist[doc["build"]] = doc["build_id"] except: print "set failed, couchbase down?: %s:%s" % (HOST,PORT) if first_pass: storeJob(jobDoc, bucket, first_pass = False)
def flushctl_stop(servers, username=None, password=None): for server in servers: c = MemcachedClient(server.ip, 11210) if username: c.sasl_auth_plain(username, password) c.stop_persistence()
class ComplianceTest(unittest.TestCase): def setUp(self): self.mc=MemcachedClient() self.mc.flush() def tearDown(self): self.mc.flush() self.mc.close() def testVersion(self): """Test the version command returns something.""" v=self.mc.version() self.assertTrue(len(v) > 0, "Bad version: ``" + str(v) + "''") def testSimpleSetGet(self): """Test a simple set and get.""" self.mc.set("x", 5, 19, "somevalue") self.assertGet((19, "somevalue"), self.mc.get("x")) def testZeroExpiration(self): """Ensure zero-expiration sets work properly.""" self.mc.set("x", 0, 19, "somevalue") time.sleep(1.1) self.assertGet((19, "somevalue"), self.mc.get("x")) def assertNotExists(self, key): try: x=self.mc.get(key) self.fail("Expected an exception, got " + `x`) except MemcachedError, e: self.assertEquals(memcacheConstants.ERR_NOT_FOUND, e.status)
def wait_for_vbuckets_ready_state(node, bucket, timeout_in_seconds=300, log_msg='', admin_user='******', admin_pass='******'): start_time = time.time() end_time = start_time + timeout_in_seconds ready_vbuckets = {} log = logger.get("infra") rest = RestConnection(node) # servers = rest.get_nodes() bucket_conn = BucketHelper(node) bucket_conn.vbucket_map_ready(bucket, 60) vbucket_count = len(bucket_conn.get_vbuckets(bucket)) vbuckets = bucket_conn.get_vbuckets(bucket) obj = VBucketAwareMemcached(rest, bucket, info=node) memcacheds, vbucket_map, vbucket_map_replica = obj.request_map( rest, bucket) #Create dictionary with key:"ip:port" and value: a list of vbuckets server_dict = defaultdict(list) for everyID in range(0, vbucket_count): memcached_ip_port = str(vbucket_map[everyID]) server_dict[memcached_ip_port].append(everyID) while time.time() < end_time and len(ready_vbuckets) < vbucket_count: for every_ip_port in server_dict: #Retrieve memcached ip and port ip, port = every_ip_port.split(":") client = MemcachedClient(ip, int(port), timeout=30) client.vbucket_count = len(vbuckets) bucket_info = bucket_conn.get_bucket(bucket) versions = rest.get_nodes_versions(logging=False) pre_spock = False for version in versions: if "5" > version: pre_spock = True if pre_spock: log.info("Atleast 1 of the server is on pre-spock " "version. Using the old ssl auth to connect to " "bucket.") client.sasl_auth_plain( bucket_info.name.encode('ascii'), bucket_info.saslPassword.encode('ascii')) else: client.sasl_auth_plain(admin_user, admin_pass) bucket = bucket.encode('ascii') client.bucket_select(bucket) for i in server_dict[every_ip_port]: try: (a, b, c) = client.get_vbucket_state(i) except mc_bin_client.MemcachedError as e: ex_msg = str(e) if "Not my vbucket" in log_msg: log_msg = log_msg[:log_msg.find("vBucketMap") + 12] + "..." if e.status == memcacheConstants.ERR_NOT_MY_VBUCKET: # May receive this while waiting for vbuckets, continue and retry...S continue log.error("%s: %s" % (log_msg, ex_msg)) continue except exceptions.EOFError: # The client was disconnected for some reason. This can # happen just after the bucket REST API is returned (before # the buckets are created in each of the memcached processes.) # See here for some details: http://review.couchbase.org/#/c/49781/ # Longer term when we don't disconnect clients in this state we # should probably remove this code. log.error( "got disconnected from the server, reconnecting") continue if c.find("\x01") > 0 or c.find("\x02") > 0: ready_vbuckets[i] = True elif i in ready_vbuckets: log.warning( "vbucket state changed from active to {0}".format( c)) del ready_vbuckets[i] client.close() return len(ready_vbuckets) == vbucket_count
def poll(): url = "https://macbuild.hq.couchbase.com/xcode/api/integrations/filter/latest" latest = getJS(url) now = datetime.datetime.now() ts = now.strftime("%Y%m%d") build_no = "%s-%s" % (MOBILE_VERSION, ts[-5:]) client = McdClient(HOST, PORT) client.sasl_auth_plain("mobile", "") buildHist = {} for build in latest: if 'revisionBlueprint' not in latest[build]: continue key = latest[build]['revisionBlueprint']['DVTSourceControlWorkspaceBlueprintPrimaryRemoteRepositoryKey'] rev = latest[build]['revisionBlueprint']['DVTSourceControlWorkspaceBlueprintLocationsKey'][key]['DVTSourceControlLocationRevisionKey'] name = "iOS-"+latest[build]['bot']['name'].replace(" ","") build_id = latest[build]['number'] results = latest[build]['buildResultSummary'] totalCount = results['testsCount'] failCount = results['errorCount'] result = 'SUCCESS' if name in TOTAL_COUNT_LOG: if totalCount == 0: # use historical value for total count totalCount = TOTAL_COUNT_LOG[name] elif totalCount != TOTAL_COUNT_LOG: # update total count log TOTAL_COUNT_LOG[name] = totalCount else: TOTAL_COUNT_LOG[name] = totalCount if (failCount > 0): result = 'UNSTABLE' component = None for feature in MOBILE_FEATURES: tag, _c = feature.split("-") docname = name.upper() docname = docname.replace("-","_") if tag in docname: component = _c if component: doc = {'build_id': build_id, 'priority': 'P0', 'name': name, 'url': url, 'component': component, 'failCount': failCount, 'totalCount': totalCount, 'result': result, 'os': 'iOS', 'build': build_no} key = "%s-%s" % (doc["name"], doc["build_id"]) val = json.dumps(doc) try: #print val key = hashlib.md5(key).hexdigest() print val client.set(key, 0, 0, val, 0) buildHist[doc["build"]] = doc["build_id"] except Exception as ex: print ex print "set failed, couchbase down?: %s:%s" % (HOST,PORT)
def wait_for_mc_stats_no_timeout(master, bucket, stat_key, stat_value, timeout_in_seconds=-1, verbose=True): log.info( "waiting for bucket {0} stat : {1} to match {2} on {3}".format(bucket, stat_key, stat_value, master.ip) ) # keep retrying until reaches the server stats = {} while not stats: try: c = MemcachedClient(master.ip, 11210) c.sasl_auth_plain(bucket, "") stats = c.stats() except Exception as e: log.info("Exception: {0}, retry in 2 seconds ...".format(str(e))) stats = {} time.sleep(2) finally: c.close() while str(stats[stat_key]) != str(stat_value): c = MemcachedClient(master.ip, 11210) c.sasl_auth_plain(bucket, "") stats = c.stats() c.close() if verbose: log.info("{0} : {1}".format(stat_key, stats[stat_key])) time.sleep(5) return True
def setUp(self): self.mc=MemcachedClient() self.mc.flush()