def direct_client(server, bucket, timeout=30): log = logger.Logger.get_logger() rest = RestConnection(server) node = None try: node = rest.get_nodes_self() except ValueError as e: log.info("could not connect to server {0}, will try scanning all nodes".format(server)) if not node: nodes = rest.get_nodes() for n in nodes: if n.ip == server.ip and n.port == server.port: node = n if isinstance(server, dict): log.info("dict:{0}".format(server)) log.info("creating direct client {0}:{1} {2}".format(server["ip"], node.memcached, bucket)) else: log.info("creating direct client {0}:{1} {2}".format(server.ip, node.memcached, bucket)) RestHelper(rest).vbucket_map_ready(bucket, 60) vBuckets = RestConnection(server).get_vbuckets(bucket) if isinstance(server, dict): client = MemcachedClient(server["ip"], node.memcached, timeout=timeout) else: client = MemcachedClient(server.ip, node.memcached, timeout=timeout) client.vbucket_count = len(vBuckets) bucket_info = rest.get_bucket(bucket) # todo raise exception for not bucket_info client.sasl_auth_plain(bucket_info.name.encode("ascii"), bucket_info.saslPassword.encode("ascii")) return client
def wait_for_vbuckets_ready_state(node, bucket, timeout_in_seconds=300, log_msg=''): log = logger.Logger.get_logger() start_time = time.time() end_time = start_time + timeout_in_seconds ready_vbuckets = {} rest = RestConnection(node) servers = rest.get_nodes() RestHelper(rest).vbucket_map_ready(bucket, 60) vbucket_count = len(rest.get_vbuckets(bucket)) vbuckets = rest.get_vbuckets(bucket) obj = VBucketAwareMemcached(rest, bucket) memcacheds, vbucket_map, vbucket_map_replica = obj.request_map(rest, bucket) #Create dictionary with key:"ip:port" and value: a list of vbuckets server_dict = defaultdict(list) for everyID in range(0, vbucket_count): memcached_ip_port = str(vbucket_map[everyID]) server_dict[memcached_ip_port].append(everyID) while time.time() < end_time and len(ready_vbuckets) < vbucket_count: for every_ip_port in server_dict: #Retrieve memcached ip and port ip, port = every_ip_port.split(":") client = MemcachedClient(ip, int(port), timeout=30) client.vbucket_count = len(vbuckets) bucket_info = rest.get_bucket(bucket) client.sasl_auth_plain(bucket_info.name.encode('ascii'), bucket_info.saslPassword.encode('ascii')) for i in server_dict[every_ip_port]: try: (a, b, c) = client.get_vbucket_state(i) except mc_bin_client.MemcachedError as e: ex_msg = str(e) if "Not my vbucket" in log_msg: log_msg = log_msg[:log_msg.find("vBucketMap") + 12] + "..." if e.status == memcacheConstants.ERR_NOT_MY_VBUCKET: # May receive this while waiting for vbuckets, continue and retry...S continue log.error("%s: %s" % (log_msg, ex_msg)) continue except exceptions.EOFError: # The client was disconnected for some reason. This can # happen just after the bucket REST API is returned (before # the buckets are created in each of the memcached processes.) # See here for some details: http://review.couchbase.org/#/c/49781/ # Longer term when we don't disconnect clients in this state we # should probably remove this code. log.error("got disconnected from the server, reconnecting") client.reconnect() client.sasl_auth_plain(bucket_info.name.encode('ascii'), bucket_info.saslPassword.encode('ascii')) continue if c.find("\x01") > 0 or c.find("\x02") > 0: ready_vbuckets[i] = True elif i in ready_vbuckets: log.warning("vbucket state changed from active to {0}".format(c)) del ready_vbuckets[i] client.close() return len(ready_vbuckets) == vbucket_count
def wait_for_vbuckets_ready_state(node, bucket, timeout_in_seconds=300, log_msg=''): log = logger.Logger.get_logger() start_time = time.time() end_time = start_time + timeout_in_seconds ready_vbuckets = {} rest = RestConnection(node) servers = rest.get_nodes() RestHelper(rest).vbucket_map_ready(bucket, 60) vbucket_count = len(rest.get_vbuckets(bucket)) vbuckets = rest.get_vbuckets(bucket) obj = VBucketAwareMemcached(rest, bucket) memcacheds, vbucket_map, vbucket_map_replica = obj.request_map( rest, bucket) #Create dictionary with key:"ip:port" and value: a list of vbuckets server_dict = defaultdict(list) for everyID in range(0, vbucket_count): memcached_ip_port = str(vbucket_map[everyID]) server_dict[memcached_ip_port].append(everyID) while time.time() < end_time and len(ready_vbuckets) < vbucket_count: for every_ip_port in server_dict: #Retrieve memcached ip and port ip, port = every_ip_port.split(":") client = MemcachedClient(ip, int(port), timeout=30) client.vbucket_count = len(vbuckets) bucket_info = rest.get_bucket(bucket) client.sasl_auth_plain( bucket_info.name.encode('ascii'), bucket_info.saslPassword.encode('ascii')) for i in server_dict[every_ip_port]: try: (a, b, c) = client.get_vbucket_state(i) except mc_bin_client.MemcachedError as e: log.error("%s: %s" % (log_msg, e)) continue if c.find("\x01") > 0 or c.find("\x02") > 0: ready_vbuckets[i] = True elif i in ready_vbuckets: log.warning( "vbucket state changed from active to {0}".format( c)) del ready_vbuckets[i] client.close() return len(ready_vbuckets) == vbucket_count
def wait_for_vbuckets_ready_state(node, bucket, timeout_in_seconds=300, log_msg=''): log = logger.Logger.get_logger() start_time = time.time() end_time = start_time + timeout_in_seconds ready_vbuckets = {} rest = RestConnection(node) servers = rest.get_nodes() RestHelper(rest).vbucket_map_ready(bucket, 60) vbucket_count = len(rest.get_vbuckets(bucket)) vbuckets = rest.get_vbuckets(bucket) obj = VBucketAwareMemcached(rest, bucket) memcacheds, vbucket_map, vbucket_map_replica = obj.request_map(rest, bucket) #Create dictionary with key:"ip:port" and value: a list of vbuckets server_dict = defaultdict(list) for everyID in range(0, vbucket_count): memcached_ip_port = str(vbucket_map[everyID]) server_dict[memcached_ip_port].append(everyID) while time.time() < end_time and len(ready_vbuckets) < vbucket_count: for every_ip_port in server_dict: #Retrieve memcached ip and port ip, port = every_ip_port.split(":") client = MemcachedClient(ip, int(port), timeout=30) client.vbucket_count = len(vbuckets) bucket_info = rest.get_bucket(bucket) client.sasl_auth_plain(bucket_info.name.encode('ascii'), bucket_info.saslPassword.encode('ascii')) for i in server_dict[every_ip_port]: try: (a, b, c) = client.get_vbucket_state(i) except mc_bin_client.MemcachedError as e: ex_msg = str(e) if "Not my vbucket" in log_msg: log_msg = log_msg[:log_msg.find("vBucketMap") + 12] + "..." if "Not my vbucket" in ex_msg: #reduce output ex_msg = str(e)[:str(e).find('Not my vbucket') + 14] + "..." log.error("%s: %s" % (log_msg, ex_msg)) continue if c.find("\x01") > 0 or c.find("\x02") > 0: ready_vbuckets[i] = True elif i in ready_vbuckets: log.warning("vbucket state changed from active to {0}".format(c)) del ready_vbuckets[i] client.close() return len(ready_vbuckets) == vbucket_count
def direct_client(server, bucket, timeout=30): rest = RestConnection(server) node = rest.get_nodes_self() log = logger.Logger.get_logger() if isinstance(server, dict): log.info("dict:{0}".format(server)) log.info("creating direct client {0}:{1} {2}".format(server["ip"], node.memcached, bucket)) else: log.info("creating direct client {0}:{1} {2}".format(server.ip, node.memcached, bucket)) RestHelper(rest).vbucket_map_ready(bucket, 60) vBuckets = RestConnection(server).get_vbuckets(bucket) if isinstance(server, dict): client = MemcachedClient(server["ip"], node.memcached, timeout=timeout) else: client = MemcachedClient(server.ip, node.memcached, timeout=timeout) client.vbucket_count = len(vBuckets) bucket_info = rest.get_bucket(bucket) #todo raise exception for not bucket_info client.sasl_auth_plain(bucket_info.name.encode('ascii'), bucket_info.saslPassword.encode('ascii')) return client
def getDirectMC(key, ip, port = 8091, bucket = "default", password = ""): real_mc_client = None # get initial mc client client = MemcachedClient(ip, int(port)) # get vbucket map rest = create_rest(ip, port) vbuckets = rest.get_vbuckets(bucket) vbId = (((zlib.crc32(key)) >> 16) & 0x7fff) & (len(vbuckets) - 1) # find vbucket responsible to this key and mapping host if vbuckets is not None: vbucket = [vbucket for vbucket in vbuckets if vbucket.id == vbId] if len(vbucket) == 1: mc_ip, mc_port = vbucket[0].master.split(":") real_mc_client = MemcachedClient(mc_ip, int(mc_port)) real_mc_client.sasl_auth_plain(bucket, password) real_mc_client.vbucket_count = len(vbuckets) return real_mc_client
def wait_for_vbuckets_ready_state(node, bucket, timeout_in_seconds=300, log_msg='', admin_user='******', admin_pass='******'): start_time = time.time() end_time = start_time + timeout_in_seconds ready_vbuckets = {} log = logger.get("infra") rest = RestConnection(node) # servers = rest.get_nodes() bucket_conn = BucketHelper(node) bucket_conn.vbucket_map_ready(bucket, 60) vbucket_count = len(bucket_conn.get_vbuckets(bucket)) vbuckets = bucket_conn.get_vbuckets(bucket) obj = VBucketAwareMemcached(rest, bucket, info=node) memcacheds, vbucket_map, vbucket_map_replica = obj.request_map( rest, bucket) #Create dictionary with key:"ip:port" and value: a list of vbuckets server_dict = defaultdict(list) for everyID in range(0, vbucket_count): memcached_ip_port = str(vbucket_map[everyID]) server_dict[memcached_ip_port].append(everyID) while time.time() < end_time and len(ready_vbuckets) < vbucket_count: for every_ip_port in server_dict: #Retrieve memcached ip and port ip, port = every_ip_port.split(":") client = MemcachedClient(ip, int(port), timeout=30) client.vbucket_count = len(vbuckets) bucket_info = bucket_conn.get_bucket(bucket) versions = rest.get_nodes_versions(logging=False) pre_spock = False for version in versions: if "5" > version: pre_spock = True if pre_spock: log.info("Atleast 1 of the server is on pre-spock " "version. Using the old ssl auth to connect to " "bucket.") client.sasl_auth_plain( bucket_info.name.encode('ascii'), bucket_info.saslPassword.encode('ascii')) else: client.sasl_auth_plain(admin_user, admin_pass) bucket = bucket.encode('ascii') client.bucket_select(bucket) for i in server_dict[every_ip_port]: try: (a, b, c) = client.get_vbucket_state(i) except mc_bin_client.MemcachedError as e: ex_msg = str(e) if "Not my vbucket" in log_msg: log_msg = log_msg[:log_msg.find("vBucketMap") + 12] + "..." if e.status == memcacheConstants.ERR_NOT_MY_VBUCKET: # May receive this while waiting for vbuckets, continue and retry...S continue log.error("%s: %s" % (log_msg, ex_msg)) continue except exceptions.EOFError: # The client was disconnected for some reason. This can # happen just after the bucket REST API is returned (before # the buckets are created in each of the memcached processes.) # See here for some details: http://review.couchbase.org/#/c/49781/ # Longer term when we don't disconnect clients in this state we # should probably remove this code. log.error( "got disconnected from the server, reconnecting") continue if c.find("\x01") > 0 or c.find("\x02") > 0: ready_vbuckets[i] = True elif i in ready_vbuckets: log.warning( "vbucket state changed from active to {0}".format( c)) del ready_vbuckets[i] client.close() return len(ready_vbuckets) == vbucket_count