def test_sdk_client(self):
"""
Test SDK Client Calls
"""
scheme = "couchbase"
host = self.master.ip
if self.master.ip == "127.0.0.1":
scheme = "http"
host = "{0}:{1}".format(self.master.ip, self.master.port)
client = SDKClient(scheme=scheme, hosts=[host], bucket="default")
client.remove("1", quiet=True)
client.insert("1", "{1:2}")
flag, cas, val = client.get("1")
self.log.info("val={0},flag={1},cas={2}".format(val, flag, cas))
client.upsert("1", "{1:3}")
client.touch("1", ttl=100)
flag, cas, val = client.get("1")
self.assertTrue(val == "{1:3}", val)
self.log.info("val={0},flag={1},cas={2}".format(val, flag, cas))
client.remove("1", cas=cas)
client.incr("key", delta=20, initial=5)
flag, cas, val = client.get("key")
self.assertTrue(val == 5)
self.log.info("val={0},flag={1},cas={2}".format(val, flag, cas))
client.incr("key", delta=20, initial=5)
flag, cas, val = client.get("key")
self.assertTrue(val == 25)
self.log.info("val={0},flag={1},cas={2}".format(val, flag, cas))
client.decr("key", delta=20, initial=5)
flag, cas, val = client.get("key")
self.assertTrue(val == 5)
print flag, cas, val
client.upsert("key1", "document1")
client.upsert("key2", "document2")
client.upsert("key3", "document3")
set = client.get_multi(["key1", "key2"])
self.log.info(set)
client.upsert_multi({"key1": "{1:2}", "key2": "{3:2}"})
set = client.get_multi(["key1", "key2"])
self.log.info(set)
client.touch_multi(["key1", "key2"], ttl=200)
set = client.get_multi(["key1", "key2"])
self.log.info(set)
data = client.observe("key1")
self.log.info(data)
data = client.observe_multi(["key1", "key2"])
self.log.info(data)
stats = client.stats(["key1"])
self.log.info(stats)
client.n1ql_request(
client.n1ql_query('create primary index on default')).execute()
query = client.n1ql_query('select * from default')
request = client.n1ql_request(query)
obj = request.get_single_result()._jsobj
self.log.info(obj)
client.close()
def test_sdk_client(self):
"""
Test SDK Client Calls
"""
scheme = "couchbase"
host=self.master.ip
if self.master.ip == "127.0.0.1":
scheme = "http"
host="{0}:{1}".format(self.master.ip,self.master.port)
client = SDKClient(scheme=scheme,hosts = [host], bucket = "default")
client.remove("1",quiet=True)
client.insert("1","{1:2}")
flag, cas, val = client.get("1")
self.log.info("val={0},flag={1},cas={2}".format(val, flag, cas))
client.upsert("1","{1:3}")
client.touch("1",ttl=100)
flag, cas, val = client.get("1")
self.assertTrue(val == "{1:3}", val)
self.log.info("val={0},flag={1},cas={2}".format(val, flag, cas))
client.remove("1",cas = cas)
client.incr("key", delta=20, initial=5)
flag, cas, val = client.get("key")
self.assertTrue(val == 5)
self.log.info("val={0},flag={1},cas={2}".format(val, flag, cas))
client.incr("key", delta=20, initial=5)
flag, cas, val = client.get("key")
self.assertTrue(val == 25)
self.log.info("val={0},flag={1},cas={2}".format(val, flag, cas))
client.decr("key", delta=20, initial=5)
flag, cas, val = client.get("key")
self.assertTrue(val == 5)
print flag, cas, val
client.upsert("key1","document1")
client.upsert("key2","document2")
client.upsert("key3","document3")
set = client.get_multi(["key1","key2"])
self.log.info(set)
client.upsert_multi({"key1":"{1:2}","key2":"{3:2}"})
set = client.get_multi(["key1","key2"])
self.log.info(set)
client.touch_multi(["key1","key2"],ttl=200)
set = client.get_multi(["key1","key2"])
self.log.info(set)
data = client.observe("key1")
self.log.info(data)
data = client.observe_multi(["key1","key2"])
self.log.info(data)
stats = client.stats(["key1"])
self.log.info(stats)
client.n1ql_request(client.n1ql_query('create primary index on default')).execute()
query = client.n1ql_query('select * from default')
request = client.n1ql_request(query)
obj = request.get_single_result()._jsobj
self.log.info(obj)
client.close()
def test_ephemeral_bucket_NRU_eviction_access_in_the_delete_range(self):
"""
generate_load = BlobGenerator(EvictionKV.KEY_ROOT, 'param2', self.value_size, start=0, end=self.num_items)
self._load_all_ephemeral_buckets_until_no_more_memory(self.servers[0], generate_load, "create", 0, self.num_items)
# figure out how many items were loaded and load 10% more
rest = RestConnection(self.servers[0])
itemCount = rest.get_bucket(self.buckets[0]).stats.itemCount
self.log.info( 'Reached OOM, the number of items is {0}'.format( itemCount))
"""
# select some keys which we expect to be adjacent to the kvs which will be deleted
# and how many KVs should we select, maybe that is a parameter
itemCount = 50000
max_delete_value = itemCount / 10
NUM_OF_ACCESSES = 50
keys_to_access = set()
for i in range(NUM_OF_ACCESSES):
keys_to_access.add(random.randint(0, max_delete_value))
# and then do accesses on the key set
client = SDKClient(hosts=[self.master.ip], bucket=self.buckets[0])
for i in keys_to_access:
# and we may want to parameterize the get at some point
rc = client.get(EvictionKV.KEY_ROOT + str(i), no_format=True)
# and then do puts to delete out stuff
PERCENTAGE_TO_ADD = 10
incremental_kv_population = BlobGenerator(EvictionKV.KEY_ROOT,
'param2',
self.value_size,
start=itemCount,
end=itemCount *
PERCENTAGE_TO_ADD / 100)
self._load_bucket(self.buckets[0],
self.master,
incremental_kv_population,
"create",
exp=0,
kv_store=1)
# and verify that the touched kvs are still there
for i in keys_to_access:
# and we may want to parameterize the get at some point
rc = client.get(EvictionKV.KEY_ROOT + str(i), no_format=True)
self.assertFalse(
rc is None,
'Key {0} was incorrectly deleted'.format(EvictionKV.KEY_ROOT +
str(i)))
def test_add_concurrent(self):
DOCID = 'subdoc_doc_id'
SERVER_IP = self.servers[0].ip
ITERATIONS = 200
THREADS = 20
main_bucket = SDKClient(scheme="couchbase", hosts=[self.servers[0].ip],
bucket='default').cb
main_bucket.upsert(DOCID, {'recs':[]})
class Runner(Thread):
def run(self, *args, **kw):
cb = SDKClient(scheme="couchbase", hosts=[SERVER_IP],
bucket='default').cb
for x in range(ITERATIONS):
cb.mutate_in(DOCID, SD.array_append('recs', 1))
thrs = [Runner() for x in range(THREADS)]
[t.start() for t in thrs]
[t.join() for t in thrs]
obj = main_bucket.get(DOCID)
array_entry_count = len(obj.value['recs'])
self.assertTrue(array_entry_count == ITERATIONS * THREADS,
'Incorrect number of array entries. Expected {0} actual {1}'.format(ITERATIONS * THREADS,
array_entry_count))
def __load_chain(self, start_num=0):
for i, cluster in enumerate(self.get_cb_clusters()):
if self._rdirection == REPLICATION_DIRECTION.BIDIRECTION:
if i > len(self.get_cb_clusters()) - 1:
break
else:
if i >= len(self.get_cb_clusters()) - 1:
break
if not self._dgm_run:
for bucket in cluster.get_buckets():
client = SDKClient(scheme="couchbase", hosts=[cluster.get_master_node().ip],
bucket=bucket.name).cb
for i in range(start_num, start_num + self._num_items):
key = 'k_%s_%s' % (i, str(cluster).replace(' ', '_').
replace('.', '_').replace(',', '_').replace(':', '_'))
value = {'xattr_%s' % i:'value%s' % i}
client.upsert(key, value)
client.mutate_in(key, SD.upsert('xattr_%s' % i, 'value%s' % i,
xattr=True,
create_parents=True))
partition = bucket.kvs[1].acquire_partition(key)#["partition"]
if self.only_store_hash:
value = str(crc32.crc32_hash(value))
res = client.get(key)
partition.set(key, json.dumps(value), 0, res.flags)
bucket.kvs[1].release_partition(key)
else:
cluster.load_all_buckets_till_dgm(
active_resident_threshold=self._active_resident_threshold,
items=self._num_items)
def __load_chain(self, start_num=0):
for i, cluster in enumerate(self.get_cb_clusters()):
if self._rdirection == REPLICATION_DIRECTION.BIDIRECTION:
if i > len(self.get_cb_clusters()) - 1:
break
else:
if i >= len(self.get_cb_clusters()) - 1:
break
if not self._dgm_run:
for bucket in cluster.get_buckets():
client = SDKClient(scheme="couchbase", hosts=[cluster.get_master_node().ip],
bucket=bucket.name).cb
for i in xrange(start_num, start_num + self._num_items):
key = 'k_%s_%s' % (i, str(cluster).replace(' ','_').
replace('.','_').replace(',','_').replace(':','_'))
value = {'xattr_%s' % i:'value%s' % i}
client.upsert(key, value)
client.mutate_in(key, SD.upsert('xattr_%s' % i, 'value%s' % i,
xattr=True,
create_parents=True))
partition = bucket.kvs[1].acquire_partition(key)#["partition"]
if self.only_store_hash:
value = str(crc32.crc32_hash(value))
res = client.get(key)
partition.set(key, json.dumps(value), 0, res.flags)
bucket.kvs[1].release_partition(key)
else:
cluster.load_all_buckets_till_dgm(
active_resident_threshold=self._active_resident_threshold,
items=self._num_items)
def test_add_concurrent(self):
DOCID = 'subdoc_doc_id'
SERVER_IP = self.servers[0].ip
ITERATIONS = 200
THREADS = 20
main_bucket = SDKClient(scheme="couchbase", hosts=[self.servers[0].ip],
bucket='default').cb
main_bucket.upsert(DOCID, {'recs':[]})
class Runner(Thread):
def run(self, *args, **kw):
cb = SDKClient(scheme="couchbase", hosts=[SERVER_IP],
bucket='default').cb
for x in range(ITERATIONS):
cb.mutate_in(DOCID, SD.array_append('recs', 1))
thrs = [Runner() for x in range(THREADS)]
[t.start() for t in thrs]
[t.join() for t in thrs]
obj = main_bucket.get(DOCID)
array_entry_count = len(obj.value['recs'])
self.assertTrue(array_entry_count == ITERATIONS * THREADS,
'Incorrect number of array entries. Expected {0} actual {1}'.format(ITERATIONS * THREADS,
array_entry_count))
def test_nru_eviction_impact_on_cbas(self):
self.log.info("Create dataset")
self.cbas_util.create_dataset_on_bucket(self.cb_bucket_name,
self.cbas_dataset_name)
self.log.info("Connect to Local link")
self.cbas_util.connect_link()
self.log.info("Add documents until ram percentage")
self.load_document_until_ram_percentage()
self.log.info("Fetch current document count")
bucket_helper = BucketHelper(self.master)
item_count = bucket_helper.get_bucket(
self.cb_bucket_name).stats.itemCount
self.log.info("Completed base load with %s items" % item_count)
self.log.info(
"Fetch initial inserted 100 documents, so they are not removed")
client = SDKClient(hosts=[self.master.ip],
bucket=self.cb_bucket_name,
password=self.master.rest_password)
for i in range(100):
client.get("test_docs-" + str(i))
self.log.info("Add 20% more items to trigger NRU")
for i in range(item_count, int(item_count * 1.2)):
client.insert_document("key-id" + str(i), '{"name":"dave"}')
self.log.info("Validate document count on CBAS")
count_n1ql = self.rest.query_tool(
'select count(*) from %s' %
(self.cb_bucket_name))['results'][0]['$1']
if self.cbas_util.validate_cbas_dataset_items_count(
self.cbas_dataset_name, count_n1ql):
pass
else:
self.log.info(
"Document count mismatch might be due to ejection of documents on KV. Retry again"
)
count_n1ql = self.rest.query_tool(
'select count(*) from %s' %
(self.cb_bucket_name))['results'][0]['$1']
self.assertTrue(self.cbas_util.validate_cbas_dataset_items_count(
self.cbas_dataset_name, count_n1ql),
msg="Count mismatch on CBAS")
def test_sdk_client(self):
"""
Test SDK Client Calls
"""
client = SDKClient(scheme="couchbase",hosts = [self.master], bucket = "default")
client.remove("1",quiet=True)
client.insert("1","{1:2}")
val, flag, cas = client.get("1")
self.log.info("val={0},flag={1},cas={2}".format(val, flag, cas))
client.upsert("1","{1:3}")
client.touch("1",ttl=100)
val, flag, cas = client.get("1")
self.assertTrue(val == "{1:3}")
self.log.info("val={0},flag={1},cas={2}".format(val, flag, cas))
client.remove("1",cas = cas)
client.incr("key", delta=20, initial=5)
val, flag, cas = client.get("key")
self.assertTrue(val == 5)
self.log.info("val={0},flag={1},cas={2}".format(val, flag, cas))
client.incr("key", delta=20, initial=5)
val, flag, cas = client.get("key")
self.assertTrue(val == 25)
self.log.info("val={0},flag={1},cas={2}".format(val, flag, cas))
client.decr("key", delta=20, initial=5)
val, flag, cas = client.get("key")
self.assertTrue(val == 5)
print val, flag, cas
client.upsert("key1","document1")
client.upsert("key2","document2")
client.upsert("key3","document3")
set = client.get_multi(["key1","key2"])
self.log.info(set)
client.upsert_multi({"key1":"{1:2}","key2":"{3:2}"})
set = client.get_multi(["key1","key2","key3"])
self.log.info(set)
client.touch_multi(["key1","key2","key3"],ttl=200)
set = client.get_multi(["key1","key2","key3"])
self.log.info(set)
set = client.get_multi(["key1","key2","key3"],replica=True)
self.log.info(set)
data = client.observe("key1")
self.log.info(data)
data = client.observe_multi(["key1","key2"])
self.log.info(data)
stats = client.stats(["key1"])
self.log.info(stats)
client.close()
def test_ephemeral_bucket_NRU_eviction(self):
generate_load = BlobGenerator(EvictionKV.KEY_ROOT,
'param2',
self.value_size,
start=0,
end=self.num_items)
self._load_all_ephemeral_buckets_until_no_more_memory(
self.servers[0], generate_load, "create", 0, self.num_items)
# figure out how many items were loaded and load 10% more
rest = RestConnection(self.servers[0])
itemCount = rest.get_bucket(self.buckets[0]).stats.itemCount
self.log.info(
'Reached OOM, the number of items is {0}'.format(itemCount))
incremental_kv_population = BlobGenerator(EvictionKV.KEY_ROOT,
'param2',
self.value_size,
start=itemCount,
end=itemCount * 1.1)
self._load_bucket(self.buckets[0],
self.master,
incremental_kv_population,
"create",
exp=0,
kv_store=1)
# and then probe the keys that are left. For now print out a distribution but later apply some heuristic
client = SDKClient(hosts=[self.master.ip], bucket=self.buckets[0])
NUMBER_OF_CHUNKS = 11
items_in_chunk = int(1.1 * itemCount / NUMBER_OF_CHUNKS)
for i in range(NUMBER_OF_CHUNKS):
keys_still_present = 0
for j in range(items_in_chunk):
rc = client.get(EvictionKV.KEY_ROOT +
str(i * items_in_chunk + j),
no_format=True)
if rc[2] is not None:
keys_still_present = keys_still_present + 1
self.log.info(
'Chunk {0} has {1:.2f} percent items still present'.format(
i, 100 * keys_still_present /
(itemCount * 1.1 / NUMBER_OF_CHUNKS)))
def test_sdk_subddoc(self):
"""
Test SDK Client Calls
"""
scheme = "couchbase"
host=self.master.ip
if self.master.ip == "127.0.0.1":
scheme = "http"
host="{0}:{1}".format(self.master.ip,self.master.port)
client = SDKClient(scheme=scheme,hosts = [host], bucket = "default")
json_document = {"1":1, "2":2, "array": [1]}
document_key = "1"
client.insert("1",json_document)
client.insert_in(document_key, "3", 3)
client.upsert_in(document_key, "4", 4)
client.upsert_in(document_key, "4", "change_4")
client.replace_in(document_key, "4", "crap_4")
client.arrayprepend_in(document_key, "array", "0")
client.arrayappend_in(document_key, "array", "2")
client.arrayinsert_in(document_key, "array[1]", "INSERT_VALUE_AT_INDEX_1")
client.arrayaddunique_in(document_key, "array", "INSERT_UNIQUE_VALUE")
print json.dumps(client.get(document_key))
def test_sdk_subddoc(self):
"""
Test SDK Client Calls
"""
scheme = "couchbase"
host = self.master.ip
if self.master.ip == "127.0.0.1":
scheme = "http"
host = "{0}:{1}".format(self.master.ip, self.master.port)
client = SDKClient(scheme=scheme, hosts=[host], bucket="default")
json_document = {"1": 1, "2": 2, "array": [1]}
document_key = "1"
client.insert("1", json_document)
client.insert_in(document_key, "3", 3)
client.upsert_in(document_key, "4", 4)
client.upsert_in(document_key, "4", "change_4")
client.replace_in(document_key, "4", "crap_4")
client.arrayprepend_in(document_key, "array", "0")
client.arrayappend_in(document_key, "array", "2")
client.arrayinsert_in(document_key, "array[1]",
"INSERT_VALUE_AT_INDEX_1")
client.arrayaddunique_in(document_key, "array", "INSERT_UNIQUE_VALUE")
print json.dumps(client.get(document_key))
def key_not_exists_test(self):
client = SDKClient(hosts = [self.master.ip], bucket = "default")
KEY_NAME = 'key'
for i in range(1500):
client.set(KEY_NAME, "x")
#For some reason could not get delete to work
client.remove(KEY_NAME)
rc = client.get(KEY_NAME)
#.get is automatically set to quiet for the sdk_client, therefore I look for
#none to indicate an error, otherwise the sdk_client spends 10 seconds trying
#to retry the commands and is very slow
if rc[2] == None:
pass
else:
assert False
#cas errors do not sleep the test for 10 seconds, plus we need to check that the correct
#error is being thrown
try:
#For some reason replace instead of cas would not reproduce the bug
client.cas(KEY_NAME, "value", cas = 10)
except NotFoundError:
pass
assert True
def verify_results(self,
skip_verify_data=[],
skip_verify_revid=[],
sg_run=False):
"""Verify data between each couchbase and remote clusters.
Run below steps for each source and destination cluster..
1. Run expiry pager.
2. Wait for disk queue size to 0 on each nodes.
3. Wait for Outbound mutations to 0.
4. Wait for Items counts equal to kv_store size of buckets.
5. Verify items value on each bucket.
6. Verify Revision id of each item.
"""
skip_key_validation = self._input.param("skip_key_validation", False)
self.__merge_all_buckets()
for cb_cluster in self.get_cb_clusters():
for remote_cluster_ref in cb_cluster.get_remote_clusters():
try:
src_cluster = remote_cluster_ref.get_src_cluster()
dest_cluster = remote_cluster_ref.get_dest_cluster()
if self._evict_with_compactor:
for b in src_cluster.get_buckets():
# only need to do compaction on the source cluster, evictions are propagated to the remote
# cluster
src_cluster.get_cluster().compact_bucket(
src_cluster.get_master_node(), b)
else:
src_cluster.run_expiry_pager()
dest_cluster.run_expiry_pager()
src_cluster.wait_for_flusher_empty()
dest_cluster.wait_for_flusher_empty()
src_dcp_queue_drained = src_cluster.wait_for_dcp_queue_drain(
)
dest_dcp_queue_drained = dest_cluster.wait_for_dcp_queue_drain(
)
src_cluster.wait_for_outbound_mutations()
dest_cluster.wait_for_outbound_mutations()
except Exception as e:
# just log any exception thrown, do not fail test
self.log.error(e)
if not skip_key_validation:
try:
if not sg_run:
src_active_passed, src_replica_passed = \
src_cluster.verify_items_count(timeout=self._item_count_timeout)
dest_active_passed, dest_replica_passed = \
dest_cluster.verify_items_count(timeout=self._item_count_timeout)
src_cluster.verify_data(
max_verify=self._max_verify,
skip=skip_verify_data,
only_store_hash=self.only_store_hash)
dest_cluster.verify_data(
max_verify=self._max_verify,
skip=skip_verify_data,
only_store_hash=self.only_store_hash)
for _, cluster in enumerate(self.get_cb_clusters()):
for bucket in cluster.get_buckets():
h = httplib2.Http(".cache")
resp, content = h.request(
"http://{0}:4984/db/_all_docs".format(
cluster.get_master_node().ip))
self.assertEqual(
json.loads(content)['total_rows'],
self._num_items)
client = SDKClient(
scheme="couchbase",
hosts=[cluster.get_master_node().ip],
bucket=bucket.name).cb
for i in range(self._num_items):
key = 'k_%s_%s' % (i, str(cluster).replace(
' ', '_').replace('.', '_').replace(
',', '_').replace(':', '_'))
res = client.get(key)
for xk, xv in res.value.items():
rv = client.mutate_in(
key, SD.get(xk, xattr=True))
self.assertTrue(rv.exists(xk))
self.assertEqual(xv, rv[xk])
if sg_run:
resp, content = h.request(
"http://{0}:4984/db/{1}".format(
cluster.get_master_node().ip,
key))
self.assertEqual(
json.loads(content)['_id'], key)
self.assertEqual(
json.loads(content)[xk], xv)
self.assertTrue('2-' in json.loads(
content)['_rev'])
except Exception as e:
self.log.error(e)
finally:
if not sg_run:
rev_err_count = self.verify_rev_ids(
remote_cluster_ref.get_replications(),
skip=skip_verify_revid)
# we're done with the test, now report specific errors
if (not (src_active_passed and dest_active_passed)) and \
(not (src_dcp_queue_drained and dest_dcp_queue_drained)):
self.fail(
"Incomplete replication: Keys stuck in dcp queue"
)
if not (src_active_passed and dest_active_passed):
self.fail(
"Incomplete replication: Active key count is incorrect"
)
if not (src_replica_passed
and dest_replica_passed):
self.fail(
"Incomplete intra-cluster replication: "
"replica count did not match active count")
if rev_err_count > 0:
self.fail(
"RevID verification failed for remote-cluster: {0}"
.format(remote_cluster_ref))
# treat errors in self.__report_error_list as failures
if len(self.get_report_error_list()) > 0:
error_logger = self.check_errors_in_goxdcr_logs()
if error_logger:
self.fail("Errors found in logs : {0}".format(error_logger))
def verify_results(self, skip_verify_data=[], skip_verify_revid=[], sg_run=False):
"""Verify data between each couchbase and remote clusters.
Run below steps for each source and destination cluster..
1. Run expiry pager.
2. Wait for disk queue size to 0 on each nodes.
3. Wait for Outbound mutations to 0.
4. Wait for Items counts equal to kv_store size of buckets.
5. Verify items value on each bucket.
6. Verify Revision id of each item.
"""
skip_key_validation = self._input.param("skip_key_validation", False)
self.__merge_all_buckets()
for cb_cluster in self.get_cb_clusters():
for remote_cluster_ref in cb_cluster.get_remote_clusters():
try:
src_cluster = remote_cluster_ref.get_src_cluster()
dest_cluster = remote_cluster_ref.get_dest_cluster()
if self._evict_with_compactor:
for b in src_cluster.get_buckets():
# only need to do compaction on the source cluster, evictions are propagated to the remote
# cluster
src_cluster.get_cluster().compact_bucket(src_cluster.get_master_node(), b)
else:
src_cluster.run_expiry_pager()
dest_cluster.run_expiry_pager()
src_cluster.wait_for_flusher_empty()
dest_cluster.wait_for_flusher_empty()
src_dcp_queue_drained = src_cluster.wait_for_dcp_queue_drain()
dest_dcp_queue_drained = dest_cluster.wait_for_dcp_queue_drain()
src_cluster.wait_for_outbound_mutations()
dest_cluster.wait_for_outbound_mutations()
except Exception as e:
# just log any exception thrown, do not fail test
self.log.error(e)
if not skip_key_validation:
try:
if not sg_run:
src_active_passed, src_replica_passed = \
src_cluster.verify_items_count(timeout=self._item_count_timeout)
dest_active_passed, dest_replica_passed = \
dest_cluster.verify_items_count(timeout=self._item_count_timeout)
src_cluster.verify_data(max_verify=self._max_verify, skip=skip_verify_data,
only_store_hash=self.only_store_hash)
dest_cluster.verify_data(max_verify=self._max_verify, skip=skip_verify_data,
only_store_hash=self.only_store_hash)
for _, cluster in enumerate(self.get_cb_clusters()):
for bucket in cluster.get_buckets():
h = httplib2.Http(".cache")
resp, content = h.request(
"http://{0}:4984/db/_all_docs".format(cluster.get_master_node().ip))
self.assertEqual(json.loads(content)['total_rows'], self._num_items)
client = SDKClient(scheme="couchbase", hosts=[cluster.get_master_node().ip],
bucket=bucket.name).cb
for i in xrange(self._num_items):
key = 'k_%s_%s' % (i, str(cluster).replace(' ', '_').
replace('.', '_').replace(',', '_').replace(':', '_'))
res = client.get(key)
for xk, xv in res.value.iteritems():
rv = client.mutate_in(key, SD.get(xk, xattr=True))
self.assertTrue(rv.exists(xk))
self.assertEqual(xv, rv[xk])
if sg_run:
resp, content = h.request("http://{0}:4984/db/{1}".format(cluster.get_master_node().ip, key))
self.assertEqual(json.loads(content)['_id'], key)
self.assertEqual(json.loads(content)[xk], xv)
self.assertTrue('2-' in json.loads(content)['_rev'])
except Exception as e:
self.log.error(e)
finally:
if not sg_run:
rev_err_count = self.verify_rev_ids(remote_cluster_ref.get_replications(),
skip=skip_verify_revid)
# we're done with the test, now report specific errors
if (not (src_active_passed and dest_active_passed)) and \
(not (src_dcp_queue_drained and dest_dcp_queue_drained)):
self.fail("Incomplete replication: Keys stuck in dcp queue")
if not (src_active_passed and dest_active_passed):
self.fail("Incomplete replication: Active key count is incorrect")
if not (src_replica_passed and dest_replica_passed):
self.fail("Incomplete intra-cluster replication: "
"replica count did not match active count")
if rev_err_count > 0:
self.fail("RevID verification failed for remote-cluster: {0}".
format(remote_cluster_ref))
# treat errors in self.__report_error_list as failures
if len(self.get_report_error_list()) > 0:
error_logger = self.check_errors_in_goxdcr_logs()
if error_logger:
self.fail("Errors found in logs : {0}".format(error_logger))
