def verify_single_node(self, server, kv_store=1):
"""This is the verification function for single node backup.
Args:
server: the master server in the cluster as self.master.
kv_store: default value is 1. This is the key of the kv_store of each bucket.
If --single-node flag appears in backup commad line, we just backup all the items
from a single node (the master node in this case). For each bucket, we request for the vBucketMap. For every key
in the kvstore of that bucket, we use hash function to get the vBucketId corresponding to that
key. By using the vBucketMap, we can know whether that key is in master node or not.
If yes, keep it. Otherwise delete it."""
rest = RestConnection(server)
for bucket in self.buckets:
VBucketAware = VBucketAwareMemcached(rest, bucket.name)
memcacheds, vBucketMap, vBucketMapReplica = VBucketAware.request_map(rest, bucket.name)
valid_keys, deleted_keys = bucket.kvs[kv_store].key_set()
for key in valid_keys:
vBucketId = VBucketAware._get_vBucket_id(key)
which_server = vBucketMap[vBucketId]
sub = which_server.find(":")
which_server_ip = which_server[:sub]
if which_server_ip != server.ip:
partition = bucket.kvs[kv_store].acquire_partition(key)
partition.delete(key)
bucket.kvs[kv_store].release_partition(key)
self._verify_all_buckets(server, kv_store, self.wait_timeout * 50, self.max_verify, True, 1)
def verify_single_node(self, server, kv_store=1):
"""This is the verification function for single node backup.
Args:
server: the master server in the cluster as self.master.
kv_store: default value is 1. This is the key of the kv_store of each bucket.
If --single-node flag appears in backup commad line, we just backup all the items
from a single node (the master node in this case). For each bucket, we request for the vBucketMap. For every key
in the kvstore of that bucket, we use hash function to get the vBucketId corresponding to that
key. By using the vBucketMap, we can know whether that key is in master node or not.
If yes, keep it. Otherwise delete it."""
rest = RestConnection(server)
for bucket in self.buckets:
VBucketAware = VBucketAwareMemcached(rest, bucket.name)
memcacheds, vBucketMap, vBucketMapReplica = VBucketAware.request_map(rest, bucket.name)
valid_keys, deleted_keys = bucket.kvs[kv_store].key_set()
for key in valid_keys:
vBucketId = VBucketAware._get_vBucket_id(key)
which_server = vBucketMap[vBucketId]
sub = which_server.find(":")
which_server_ip = which_server[:sub]
if which_server_ip != server.ip:
partition = bucket.kvs[kv_store].acquire_partition(key)
partition.delete(key)
bucket.kvs[kv_store].release_partition(key)
self._verify_all_buckets(server, kv_store, self.wait_timeout * 50, self.max_verify, True, 1)
def wait_for_vbuckets_ready_state(node, bucket, timeout_in_seconds=300, log_msg=''):
log = logger.Logger.get_logger()
start_time = time.time()
end_time = start_time + timeout_in_seconds
ready_vbuckets = {}
rest = RestConnection(node)
servers = rest.get_nodes()
RestHelper(rest).vbucket_map_ready(bucket, 60)
vbucket_count = len(rest.get_vbuckets(bucket))
vbuckets = rest.get_vbuckets(bucket)
obj = VBucketAwareMemcached(rest, bucket)
memcacheds, vbucket_map, vbucket_map_replica = obj.request_map(rest, bucket)
#Create dictionary with key:"ip:port" and value: a list of vbuckets
server_dict = defaultdict(list)
for everyID in range(0, vbucket_count):
memcached_ip_port = str(vbucket_map[everyID])
server_dict[memcached_ip_port].append(everyID)
while time.time() < end_time and len(ready_vbuckets) < vbucket_count:
for every_ip_port in server_dict:
#Retrieve memcached ip and port
ip, port = every_ip_port.split(":")
client = MemcachedClient(ip, int(port), timeout=30)
client.vbucket_count = len(vbuckets)
bucket_info = rest.get_bucket(bucket)
client.sasl_auth_plain(bucket_info.name.encode('ascii'),
bucket_info.saslPassword.encode('ascii'))
for i in server_dict[every_ip_port]:
try:
(a, b, c) = client.get_vbucket_state(i)
except mc_bin_client.MemcachedError as e:
ex_msg = str(e)
if "Not my vbucket" in log_msg:
log_msg = log_msg[:log_msg.find("vBucketMap") + 12] + "..."
if e.status == memcacheConstants.ERR_NOT_MY_VBUCKET:
# May receive this while waiting for vbuckets, continue and retry...S
continue
log.error("%s: %s" % (log_msg, ex_msg))
continue
except exceptions.EOFError:
# The client was disconnected for some reason. This can
# happen just after the bucket REST API is returned (before
# the buckets are created in each of the memcached processes.)
# See here for some details: http://review.couchbase.org/#/c/49781/
# Longer term when we don't disconnect clients in this state we
# should probably remove this code.
log.error("got disconnected from the server, reconnecting")
client.reconnect()
client.sasl_auth_plain(bucket_info.name.encode('ascii'),
bucket_info.saslPassword.encode('ascii'))
continue
if c.find("\x01") > 0 or c.find("\x02") > 0:
ready_vbuckets[i] = True
elif i in ready_vbuckets:
log.warning("vbucket state changed from active to {0}".format(c))
del ready_vbuckets[i]
client.close()
return len(ready_vbuckets) == vbucket_count
def wait_for_vbuckets_ready_state(node,
bucket,
timeout_in_seconds=300,
log_msg=''):
log = logger.Logger.get_logger()
start_time = time.time()
end_time = start_time + timeout_in_seconds
ready_vbuckets = {}
rest = RestConnection(node)
servers = rest.get_nodes()
RestHelper(rest).vbucket_map_ready(bucket, 60)
vbucket_count = len(rest.get_vbuckets(bucket))
vbuckets = rest.get_vbuckets(bucket)
obj = VBucketAwareMemcached(rest, bucket)
memcacheds, vbucket_map, vbucket_map_replica = obj.request_map(
rest, bucket)
#Create dictionary with key:"ip:port" and value: a list of vbuckets
server_dict = defaultdict(list)
for everyID in range(0, vbucket_count):
memcached_ip_port = str(vbucket_map[everyID])
server_dict[memcached_ip_port].append(everyID)
while time.time() < end_time and len(ready_vbuckets) < vbucket_count:
for every_ip_port in server_dict:
#Retrieve memcached ip and port
ip, port = every_ip_port.split(":")
client = MemcachedClient(ip, int(port), timeout=30)
client.vbucket_count = len(vbuckets)
bucket_info = rest.get_bucket(bucket)
client.sasl_auth_plain(
bucket_info.name.encode('ascii'),
bucket_info.saslPassword.encode('ascii'))
for i in server_dict[every_ip_port]:
try:
(a, b, c) = client.get_vbucket_state(i)
except mc_bin_client.MemcachedError as e:
log.error("%s: %s" % (log_msg, e))
continue
if c.find("\x01") > 0 or c.find("\x02") > 0:
ready_vbuckets[i] = True
elif i in ready_vbuckets:
log.warning(
"vbucket state changed from active to {0}".format(
c))
del ready_vbuckets[i]
client.close()
return len(ready_vbuckets) == vbucket_count
def wait_for_vbuckets_ready_state(node, bucket, timeout_in_seconds=300, log_msg=''):
log = logger.Logger.get_logger()
start_time = time.time()
end_time = start_time + timeout_in_seconds
ready_vbuckets = {}
rest = RestConnection(node)
servers = rest.get_nodes()
RestHelper(rest).vbucket_map_ready(bucket, 60)
vbucket_count = len(rest.get_vbuckets(bucket))
vbuckets = rest.get_vbuckets(bucket)
obj = VBucketAwareMemcached(rest, bucket)
memcacheds, vbucket_map, vbucket_map_replica = obj.request_map(rest, bucket)
#Create dictionary with key:"ip:port" and value: a list of vbuckets
server_dict = defaultdict(list)
for everyID in range(0, vbucket_count):
memcached_ip_port = str(vbucket_map[everyID])
server_dict[memcached_ip_port].append(everyID)
while time.time() < end_time and len(ready_vbuckets) < vbucket_count:
for every_ip_port in server_dict:
#Retrieve memcached ip and port
ip, port = every_ip_port.split(":")
client = MemcachedClient(ip, int(port), timeout=30)
client.vbucket_count = len(vbuckets)
bucket_info = rest.get_bucket(bucket)
client.sasl_auth_plain(bucket_info.name.encode('ascii'),
bucket_info.saslPassword.encode('ascii'))
for i in server_dict[every_ip_port]:
try:
(a, b, c) = client.get_vbucket_state(i)
except mc_bin_client.MemcachedError as e:
ex_msg = str(e)
if "Not my vbucket" in log_msg:
log_msg = log_msg[:log_msg.find("vBucketMap") + 12] + "..."
if "Not my vbucket" in ex_msg:
#reduce output
ex_msg = str(e)[:str(e).find('Not my vbucket') + 14] + "..."
log.error("%s: %s" % (log_msg, ex_msg))
continue
if c.find("\x01") > 0 or c.find("\x02") > 0:
ready_vbuckets[i] = True
elif i in ready_vbuckets:
log.warning("vbucket state changed from active to {0}".format(c))
del ready_vbuckets[i]
client.close()
return len(ready_vbuckets) == vbucket_count
def wait_for_vbuckets_ready_state(node,
bucket,
timeout_in_seconds=300,
log_msg='',
admin_user='******',
admin_pass='******'):
start_time = time.time()
end_time = start_time + timeout_in_seconds
ready_vbuckets = {}
log = logger.get("infra")
rest = RestConnection(node)
# servers = rest.get_nodes()
bucket_conn = BucketHelper(node)
bucket_conn.vbucket_map_ready(bucket, 60)
vbucket_count = len(bucket_conn.get_vbuckets(bucket))
vbuckets = bucket_conn.get_vbuckets(bucket)
obj = VBucketAwareMemcached(rest, bucket, info=node)
memcacheds, vbucket_map, vbucket_map_replica = obj.request_map(
rest, bucket)
#Create dictionary with key:"ip:port" and value: a list of vbuckets
server_dict = defaultdict(list)
for everyID in range(0, vbucket_count):
memcached_ip_port = str(vbucket_map[everyID])
server_dict[memcached_ip_port].append(everyID)
while time.time() < end_time and len(ready_vbuckets) < vbucket_count:
for every_ip_port in server_dict:
#Retrieve memcached ip and port
ip, port = every_ip_port.split(":")
client = MemcachedClient(ip, int(port), timeout=30)
client.vbucket_count = len(vbuckets)
bucket_info = bucket_conn.get_bucket(bucket)
versions = rest.get_nodes_versions(logging=False)
pre_spock = False
for version in versions:
if "5" > version:
pre_spock = True
if pre_spock:
log.info("Atleast 1 of the server is on pre-spock "
"version. Using the old ssl auth to connect to "
"bucket.")
client.sasl_auth_plain(
bucket_info.name.encode('ascii'),
bucket_info.saslPassword.encode('ascii'))
else:
client.sasl_auth_plain(admin_user, admin_pass)
bucket = bucket.encode('ascii')
client.bucket_select(bucket)
for i in server_dict[every_ip_port]:
try:
(a, b, c) = client.get_vbucket_state(i)
except mc_bin_client.MemcachedError as e:
ex_msg = str(e)
if "Not my vbucket" in log_msg:
log_msg = log_msg[:log_msg.find("vBucketMap") +
12] + "..."
if e.status == memcacheConstants.ERR_NOT_MY_VBUCKET:
# May receive this while waiting for vbuckets, continue and retry...S
continue
log.error("%s: %s" % (log_msg, ex_msg))
continue
except exceptions.EOFError:
# The client was disconnected for some reason. This can
# happen just after the bucket REST API is returned (before
# the buckets are created in each of the memcached processes.)
# See here for some details: http://review.couchbase.org/#/c/49781/
# Longer term when we don't disconnect clients in this state we
# should probably remove this code.
log.error(
"got disconnected from the server, reconnecting")
continue
if c.find("\x01") > 0 or c.find("\x02") > 0:
ready_vbuckets[i] = True
elif i in ready_vbuckets:
log.warning(
"vbucket state changed from active to {0}".format(
c))
del ready_vbuckets[i]
client.close()
return len(ready_vbuckets) == vbucket_count
