def direct_client(server, bucket, timeout=30):
log = logger.Logger.get_logger()
rest = RestConnection(server)
node = None
try:
node = rest.get_nodes_self()
except ValueError as e:
log.info("could not connect to server {0}, will try scanning all nodes".format(server))
if not node:
nodes = rest.get_nodes()
for n in nodes:
if n.ip == server.ip and n.port == server.port:
node = n
if isinstance(server, dict):
log.info("dict:{0}".format(server))
log.info("creating direct client {0}:{1} {2}".format(server["ip"], node.memcached, bucket))
else:
log.info("creating direct client {0}:{1} {2}".format(server.ip, node.memcached, bucket))
RestHelper(rest).vbucket_map_ready(bucket, 60)
vBuckets = RestConnection(server).get_vbuckets(bucket)
if isinstance(server, dict):
client = MemcachedClient(server["ip"], node.memcached, timeout=timeout)
else:
client = MemcachedClient(server.ip, node.memcached, timeout=timeout)
client.vbucket_count = len(vBuckets)
bucket_info = rest.get_bucket(bucket)
# todo raise exception for not bucket_info
client.sasl_auth_plain(bucket_info.name.encode("ascii"), bucket_info.saslPassword.encode("ascii"))
return client
def wait_for_vbuckets_ready_state(node, bucket, timeout_in_seconds=300, log_msg=''):
log = logger.Logger.get_logger()
start_time = time.time()
end_time = start_time + timeout_in_seconds
ready_vbuckets = {}
rest = RestConnection(node)
servers = rest.get_nodes()
RestHelper(rest).vbucket_map_ready(bucket, 60)
vbucket_count = len(rest.get_vbuckets(bucket))
vbuckets = rest.get_vbuckets(bucket)
obj = VBucketAwareMemcached(rest, bucket)
memcacheds, vbucket_map, vbucket_map_replica = obj.request_map(rest, bucket)
#Create dictionary with key:"ip:port" and value: a list of vbuckets
server_dict = defaultdict(list)
for everyID in range(0, vbucket_count):
memcached_ip_port = str(vbucket_map[everyID])
server_dict[memcached_ip_port].append(everyID)
while time.time() < end_time and len(ready_vbuckets) < vbucket_count:
for every_ip_port in server_dict:
#Retrieve memcached ip and port
ip, port = every_ip_port.split(":")
client = MemcachedClient(ip, int(port), timeout=30)
client.vbucket_count = len(vbuckets)
bucket_info = rest.get_bucket(bucket)
client.sasl_auth_plain(bucket_info.name.encode('ascii'),
bucket_info.saslPassword.encode('ascii'))
for i in server_dict[every_ip_port]:
try:
(a, b, c) = client.get_vbucket_state(i)
except mc_bin_client.MemcachedError as e:
ex_msg = str(e)
if "Not my vbucket" in log_msg:
log_msg = log_msg[:log_msg.find("vBucketMap") + 12] + "..."
if e.status == memcacheConstants.ERR_NOT_MY_VBUCKET:
# May receive this while waiting for vbuckets, continue and retry...S
continue
log.error("%s: %s" % (log_msg, ex_msg))
continue
except exceptions.EOFError:
# The client was disconnected for some reason. This can
# happen just after the bucket REST API is returned (before
# the buckets are created in each of the memcached processes.)
# See here for some details: http://review.couchbase.org/#/c/49781/
# Longer term when we don't disconnect clients in this state we
# should probably remove this code.
log.error("got disconnected from the server, reconnecting")
client.reconnect()
client.sasl_auth_plain(bucket_info.name.encode('ascii'),
bucket_info.saslPassword.encode('ascii'))
continue
if c.find("\x01") > 0 or c.find("\x02") > 0:
ready_vbuckets[i] = True
elif i in ready_vbuckets:
log.warning("vbucket state changed from active to {0}".format(c))
del ready_vbuckets[i]
client.close()
return len(ready_vbuckets) == vbucket_count
def wait_for_vbuckets_ready_state(node,
bucket,
timeout_in_seconds=300,
log_msg=''):
log = logger.Logger.get_logger()
start_time = time.time()
end_time = start_time + timeout_in_seconds
ready_vbuckets = {}
rest = RestConnection(node)
servers = rest.get_nodes()
RestHelper(rest).vbucket_map_ready(bucket, 60)
vbucket_count = len(rest.get_vbuckets(bucket))
vbuckets = rest.get_vbuckets(bucket)
obj = VBucketAwareMemcached(rest, bucket)
memcacheds, vbucket_map, vbucket_map_replica = obj.request_map(
rest, bucket)
#Create dictionary with key:"ip:port" and value: a list of vbuckets
server_dict = defaultdict(list)
for everyID in range(0, vbucket_count):
memcached_ip_port = str(vbucket_map[everyID])
server_dict[memcached_ip_port].append(everyID)
while time.time() < end_time and len(ready_vbuckets) < vbucket_count:
for every_ip_port in server_dict:
#Retrieve memcached ip and port
ip, port = every_ip_port.split(":")
client = MemcachedClient(ip, int(port), timeout=30)
client.vbucket_count = len(vbuckets)
bucket_info = rest.get_bucket(bucket)
client.sasl_auth_plain(
bucket_info.name.encode('ascii'),
bucket_info.saslPassword.encode('ascii'))
for i in server_dict[every_ip_port]:
try:
(a, b, c) = client.get_vbucket_state(i)
except mc_bin_client.MemcachedError as e:
log.error("%s: %s" % (log_msg, e))
continue
if c.find("\x01") > 0 or c.find("\x02") > 0:
ready_vbuckets[i] = True
elif i in ready_vbuckets:
log.warning(
"vbucket state changed from active to {0}".format(
c))
del ready_vbuckets[i]
client.close()
return len(ready_vbuckets) == vbucket_count
def wait_for_vbuckets_ready_state(node, bucket, timeout_in_seconds=300, log_msg=''):
log = logger.Logger.get_logger()
start_time = time.time()
end_time = start_time + timeout_in_seconds
ready_vbuckets = {}
rest = RestConnection(node)
servers = rest.get_nodes()
RestHelper(rest).vbucket_map_ready(bucket, 60)
vbucket_count = len(rest.get_vbuckets(bucket))
vbuckets = rest.get_vbuckets(bucket)
obj = VBucketAwareMemcached(rest, bucket)
memcacheds, vbucket_map, vbucket_map_replica = obj.request_map(rest, bucket)
#Create dictionary with key:"ip:port" and value: a list of vbuckets
server_dict = defaultdict(list)
for everyID in range(0, vbucket_count):
memcached_ip_port = str(vbucket_map[everyID])
server_dict[memcached_ip_port].append(everyID)
while time.time() < end_time and len(ready_vbuckets) < vbucket_count:
for every_ip_port in server_dict:
#Retrieve memcached ip and port
ip, port = every_ip_port.split(":")
client = MemcachedClient(ip, int(port), timeout=30)
client.vbucket_count = len(vbuckets)
bucket_info = rest.get_bucket(bucket)
client.sasl_auth_plain(bucket_info.name.encode('ascii'),
bucket_info.saslPassword.encode('ascii'))
for i in server_dict[every_ip_port]:
try:
(a, b, c) = client.get_vbucket_state(i)
except mc_bin_client.MemcachedError as e:
ex_msg = str(e)
if "Not my vbucket" in log_msg:
log_msg = log_msg[:log_msg.find("vBucketMap") + 12] + "..."
if "Not my vbucket" in ex_msg:
#reduce output
ex_msg = str(e)[:str(e).find('Not my vbucket') + 14] + "..."
log.error("%s: %s" % (log_msg, ex_msg))
continue
if c.find("\x01") > 0 or c.find("\x02") > 0:
ready_vbuckets[i] = True
elif i in ready_vbuckets:
log.warning("vbucket state changed from active to {0}".format(c))
del ready_vbuckets[i]
client.close()
return len(ready_vbuckets) == vbucket_count
def direct_client(server, bucket, timeout=30):
rest = RestConnection(server)
node = rest.get_nodes_self()
log = logger.Logger.get_logger()
if isinstance(server, dict):
log.info("dict:{0}".format(server))
log.info("creating direct client {0}:{1} {2}".format(server["ip"], node.memcached, bucket))
else:
log.info("creating direct client {0}:{1} {2}".format(server.ip, node.memcached, bucket))
RestHelper(rest).vbucket_map_ready(bucket, 60)
vBuckets = RestConnection(server).get_vbuckets(bucket)
if isinstance(server, dict):
client = MemcachedClient(server["ip"], node.memcached, timeout=timeout)
else:
client = MemcachedClient(server.ip, node.memcached, timeout=timeout)
client.vbucket_count = len(vBuckets)
bucket_info = rest.get_bucket(bucket)
#todo raise exception for not bucket_info
client.sasl_auth_plain(bucket_info.name.encode('ascii'),
bucket_info.saslPassword.encode('ascii'))
return client
def getDirectMC(key, ip, port = 8091, bucket = "default", password = ""):
real_mc_client = None
# get initial mc client
client = MemcachedClient(ip, int(port))
# get vbucket map
rest = create_rest(ip, port)
vbuckets = rest.get_vbuckets(bucket)
vbId = (((zlib.crc32(key)) >> 16) & 0x7fff) & (len(vbuckets) - 1)
# find vbucket responsible to this key and mapping host
if vbuckets is not None:
vbucket = [vbucket for vbucket in vbuckets if vbucket.id == vbId]
if len(vbucket) == 1:
mc_ip, mc_port = vbucket[0].master.split(":")
real_mc_client = MemcachedClient(mc_ip, int(mc_port))
real_mc_client.sasl_auth_plain(bucket, password)
real_mc_client.vbucket_count = len(vbuckets)
return real_mc_client
def getDirectMC(key, ip, port = 8091, bucket = "default", password = ""):
real_mc_client = None
# get initial mc client
client = MemcachedClient(ip, int(port))
# get vbucket map
rest = create_rest(ip, port)
vbuckets = rest.get_vbuckets(bucket)
vbId = (((zlib.crc32(key)) >> 16) & 0x7fff) & (len(vbuckets) - 1)
# find vbucket responsible to this key and mapping host
if vbuckets is not None:
vbucket = [vbucket for vbucket in vbuckets if vbucket.id == vbId]
if len(vbucket) == 1:
mc_ip, mc_port = vbucket[0].master.split(":")
real_mc_client = MemcachedClient(mc_ip, int(mc_port))
real_mc_client.sasl_auth_plain(bucket, password)
real_mc_client.vbucket_count = len(vbuckets)
return real_mc_client
def wait_for_vbuckets_ready_state(node,
bucket,
timeout_in_seconds=300,
log_msg='',
admin_user='******',
admin_pass='******'):
start_time = time.time()
end_time = start_time + timeout_in_seconds
ready_vbuckets = {}
log = logger.get("infra")
rest = RestConnection(node)
# servers = rest.get_nodes()
bucket_conn = BucketHelper(node)
bucket_conn.vbucket_map_ready(bucket, 60)
vbucket_count = len(bucket_conn.get_vbuckets(bucket))
vbuckets = bucket_conn.get_vbuckets(bucket)
obj = VBucketAwareMemcached(rest, bucket, info=node)
memcacheds, vbucket_map, vbucket_map_replica = obj.request_map(
rest, bucket)
#Create dictionary with key:"ip:port" and value: a list of vbuckets
server_dict = defaultdict(list)
for everyID in range(0, vbucket_count):
memcached_ip_port = str(vbucket_map[everyID])
server_dict[memcached_ip_port].append(everyID)
while time.time() < end_time and len(ready_vbuckets) < vbucket_count:
for every_ip_port in server_dict:
#Retrieve memcached ip and port
ip, port = every_ip_port.split(":")
client = MemcachedClient(ip, int(port), timeout=30)
client.vbucket_count = len(vbuckets)
bucket_info = bucket_conn.get_bucket(bucket)
versions = rest.get_nodes_versions(logging=False)
pre_spock = False
for version in versions:
if "5" > version:
pre_spock = True
if pre_spock:
log.info("Atleast 1 of the server is on pre-spock "
"version. Using the old ssl auth to connect to "
"bucket.")
client.sasl_auth_plain(
bucket_info.name.encode('ascii'),
bucket_info.saslPassword.encode('ascii'))
else:
client.sasl_auth_plain(admin_user, admin_pass)
bucket = bucket.encode('ascii')
client.bucket_select(bucket)
for i in server_dict[every_ip_port]:
try:
(a, b, c) = client.get_vbucket_state(i)
except mc_bin_client.MemcachedError as e:
ex_msg = str(e)
if "Not my vbucket" in log_msg:
log_msg = log_msg[:log_msg.find("vBucketMap") +
12] + "..."
if e.status == memcacheConstants.ERR_NOT_MY_VBUCKET:
# May receive this while waiting for vbuckets, continue and retry...S
continue
log.error("%s: %s" % (log_msg, ex_msg))
continue
except exceptions.EOFError:
# The client was disconnected for some reason. This can
# happen just after the bucket REST API is returned (before
# the buckets are created in each of the memcached processes.)
# See here for some details: http://review.couchbase.org/#/c/49781/
# Longer term when we don't disconnect clients in this state we
# should probably remove this code.
log.error(
"got disconnected from the server, reconnecting")
continue
if c.find("\x01") > 0 or c.find("\x02") > 0:
ready_vbuckets[i] = True
elif i in ready_vbuckets:
log.warning(
"vbucket state changed from active to {0}".format(
c))
del ready_vbuckets[i]
client.close()
return len(ready_vbuckets) == vbucket_count
