def verify_es_num_docs(self,
src_server,
dest_server,
kv_store=1,
retry=10,
verification_count=10000):
cb_rest = RestConnection(src_server)
es_rest = RestConnection(dest_server)
buckets = self.xd_ref._get_cluster_buckets(src_server)
wait = 20
for bucket in buckets:
all_cb_docs = cb_rest.all_docs(bucket.name)
cb_valid = [str(row['id']) for row in all_cb_docs['rows']]
cb_num_items = cb_rest.get_bucket_stats(bucket.name)['curr_items']
es_num_items = es_rest.get_bucket(bucket.name).stats.itemCount
_retry = retry
while _retry > 0 and cb_num_items != es_num_items:
self._log.info("elasticsearch items %s, expected: %s....retry after %s seconds" % \
(es_num_items, cb_num_items, wait))
time.sleep(wait)
last_es_items = es_num_items
es_num_items = es_rest.get_bucket(bucket.name).stats.itemCount
if es_num_items == last_es_items:
_retry = _retry - 1
# if index doesn't change reduce retry count
elif es_num_items <= last_es_items:
self._log.info("%s items removed from index " %
(es_num_items - last_es_items))
_retry = retry
elif es_num_items >= last_es_items:
self._log.info("%s items added to index" %
(es_num_items - last_es_items))
_retry = retry
if es_num_items != cb_num_items:
self.xd_ref.fail("Error: Couchbase has %s docs, ElasticSearch has %s docs " % \
(cb_num_items, es_num_items))
# query for all es keys
es_valid = es_rest.all_docs(keys_only=True,
indices=[bucket.name],
size=cb_num_items)
if len(es_valid) != cb_num_items:
self._log.info("WARNING: Couchbase has %s docs, ElasticSearch all_docs returned %s docs " % \
(cb_num_items, len(es_valid)))
for _id in cb_valid[:verification_count]: # match at most 10k keys
id_found = _id in es_valid
if id_found == False:
# document missing from all_docs query do manual term search
doc = es_rest.search_all_nodes(_id, indices=[bucket.name])
if doc is None:
self.xd_ref.fail(
"Document %s Missing from ES Index (%s)" %
(_id, bucket.name))
self._log.info("Verified couchbase bucket (%s) replicated (%s) docs to elasticSearch with matching keys" % \
(bucket.name, cb_num_items))
def verify_es_num_docs(self, src_server, dest_server, kv_store=1, retry=10, verification_count=10000):
cb_rest = RestConnection(src_server)
es_rest = RestConnection(dest_server)
buckets = self.xd_ref._get_cluster_buckets(src_server)
wait = 20
for bucket in buckets:
all_cb_docs = cb_rest.all_docs(bucket.name)
cb_valid = [str(row["id"]) for row in all_cb_docs["rows"]]
cb_num_items = cb_rest.get_bucket_stats(bucket.name)["curr_items"]
es_num_items = es_rest.get_bucket(bucket.name).stats.itemCount
_retry = retry
while _retry > 0 and cb_num_items != es_num_items:
self._log.info(
"elasticsearch items %s, expected: %s....retry after %s seconds"
% (es_num_items, cb_num_items, wait)
)
time.sleep(wait)
last_es_items = es_num_items
es_num_items = es_rest.get_bucket(bucket.name).stats.itemCount
if es_num_items == last_es_items:
_retry = _retry - 1
# if index doesn't change reduce retry count
elif es_num_items <= last_es_items:
self._log.info("%s items removed from index " % (es_num_items - last_es_items))
_retry = retry
elif es_num_items >= last_es_items:
self._log.info("%s items added to index" % (es_num_items - last_es_items))
_retry = retry
if es_num_items != cb_num_items:
self.xd_ref.fail(
"Error: Couchbase has %s docs, ElasticSearch has %s docs " % (cb_num_items, es_num_items)
)
# query for all es keys
es_valid = es_rest.all_docs(keys_only=True, indices=[bucket.name], size=cb_num_items)
if len(es_valid) != cb_num_items:
self._log.info(
"WARNING: Couchbase has %s docs, ElasticSearch all_docs returned %s docs "
% (cb_num_items, len(es_valid))
)
for _id in cb_valid[:verification_count]: # match at most 10k keys
id_found = _id in es_valid
if id_found == False:
# document missing from all_docs query do manual term search
doc = es_rest.search_all_nodes(_id, indices=[bucket.name])
if doc is None:
self.xd_ref.fail("Document %s Missing from ES Index (%s)" % (_id, bucket.name))
self._log.info(
"Verified couchbase bucket (%s) replicated (%s) docs to elasticSearch with matching keys"
% (bucket.name, cb_num_items)
)
def _verify_es_revIds(self,
src_server,
dest_server,
kv_store=1,
verification_count=10000):
cb_rest = RestConnection(src_server)
es_rest = RestConnection(dest_server)
buckets = self.xd_ref._get_cluster_buckets(src_server)
for bucket in buckets:
# retrieve all docs from couchbase and es
# then represent doc lists from couchbase
# and elastic search in following format
# [ (id, rev), (id, rev) ... ]
all_cb_docs = cb_rest.all_docs(bucket.name)
cb_id_rev_list = self.get_cb_id_rev_list(all_cb_docs)
es_valid = es_rest.all_docs(indices=[bucket.name],
size=len(cb_id_rev_list))
es_id_rev_list = self.get_es_id_rev_list(es_valid)
# verify each (id, rev) pair returned from couchbase exists in elastic search
for cb_id_rev_pair in cb_id_rev_list:
try:
# lookup es document with matching _id and revid
# if no exception thrown then doc was properly indexed
es_list_pos = es_id_rev_list.index(cb_id_rev_pair)
except ValueError:
# attempt manual lookup by search term
es_doc = es_rest.search_all_nodes(cb_id_rev_pair[0],
indices=[bucket.name])
if es_doc is None:
self.xd_ref.fail(
"Error during verification: %s does not exist in ES index %s"
% (cb_id_rev_pair, bucket.name))
# compare
es_id_rev_pair = (es_doc['_source']['meta']['id'],
es_doc['_source']['meta']['rev'])
if cb_id_rev_pair != es_id_rev_pair:
self.xd_ref.fail("ES document %s Missmatch Couchbase doc (%s). bucket (%s)" % \
(es_id_rev_pair, cb_id_rev_pair, bucket.name))
self._log.info("Verified doc rev-ids in couchbase bucket (%s) match meta rev-ids elastic search" % \
(bucket.name))
def _verify_es_values(self,
src_server,
dest_server,
kv_store=1,
verification_count=10000):
cb_rest = RestConnection(src_server)
es_rest = RestConnection(dest_server)
buckets = self.xd_ref._get_cluster_buckets(src_server)
for bucket in buckets:
mc = VBucketAwareMemcached(cb_rest, bucket)
es_valid = es_rest.all_docs(indices=[bucket.name],
size=verification_count)
# compare values of es documents to documents in couchbase
for row in es_valid[:verification_count]:
key = str(row['meta']['id'])
try:
_, _, doc = mc.get(key)
val_src = str(json.loads(doc)['site_name'])
val_dest = str(row['doc']['site_name'])
if val_src != val_dest:
self.xd_ref.fail("Document %s has unexpected value (%s) expected (%s)" % \
(key, val_src, val_dest))
except MemcachedError as e:
self.xd_ref.fail(
"Error during verification. Index contains invalid key: %s"
% key)
self._log.info("Verified doc values in couchbase bucket (%s) match values in elastic search" % \
(bucket.name))
def _verify_es_values(self, src_server, dest_server, kv_store=1, verification_count=10000):
cb_rest = RestConnection(src_server)
es_rest = RestConnection(dest_server)
buckets = self.xd_ref._get_cluster_buckets(src_server)
for bucket in buckets:
mc = VBucketAwareMemcached(cb_rest, bucket)
es_valid = es_rest.all_docs(indices=[bucket.name], size=verification_count)
# compare values of es documents to documents in couchbase
for row in es_valid[:verification_count]:
key = str(row["meta"]["id"])
try:
_, _, doc = mc.get(key)
val_src = str(json.loads(doc)["site_name"])
val_dest = str(row["doc"]["site_name"])
if val_src != val_dest:
self.xd_ref.fail(
"Document %s has unexpected value (%s) expected (%s)" % (key, val_src, val_dest)
)
except MemcachedError as e:
self.xd_ref.fail("Error during verification. Index contains invalid key: %s" % key)
self._log.info(
"Verified doc values in couchbase bucket (%s) match values in elastic search" % (bucket.name)
)
def _verify_es_revIds(self, src_server, dest_server, kv_store=1, verification_count=10000):
cb_rest = RestConnection(src_server)
es_rest = RestConnection(dest_server)
buckets = self.xd_ref._get_cluster_buckets(src_server)
for bucket in buckets:
# retrieve all docs from couchbase and es
# then represent doc lists from couchbase
# and elastic search in following format
# [ (id, rev), (id, rev) ... ]
all_cb_docs = cb_rest.all_docs(bucket.name)
cb_id_rev_list = self.get_cb_id_rev_list(all_cb_docs)
es_valid = es_rest.all_docs(indices=[bucket.name], size=len(cb_id_rev_list))
es_id_rev_list = self.get_es_id_rev_list(es_valid)
# verify each (id, rev) pair returned from couchbase exists in elastic search
for cb_id_rev_pair in cb_id_rev_list:
try:
# lookup es document with matching _id and revid
# if no exception thrown then doc was properly indexed
es_list_pos = es_id_rev_list.index(cb_id_rev_pair)
except ValueError:
# attempt manual lookup by search term
es_doc = es_rest.search_all_nodes(cb_id_rev_pair[0], indices=[bucket.name])
if es_doc is None:
self.xd_ref.fail(
"Error during verification: %s does not exist in ES index %s"
% (cb_id_rev_pair, bucket.name)
)
# compare
es_id_rev_pair = (es_doc["_source"]["meta"]["id"], es_doc["_source"]["meta"]["rev"])
if cb_id_rev_pair != es_id_rev_pair:
self.xd_ref.fail(
"ES document %s Missmatch Couchbase doc (%s). bucket (%s)"
% (es_id_rev_pair, cb_id_rev_pair, bucket.name)
)
self._log.info(
"Verified doc rev-ids in couchbase bucket (%s) match meta rev-ids elastic search" % (bucket.name)
)
def _verify_keys(self, doc_names, prefix):
master = self.servers[0]
rest = RestConnection(master)
bucket = "default"
results = rest.all_docs(bucket)
result_keys = ViewBaseTests._get_keys(self, results)
# result_keys also contains the design document, but this doesn't
# matter if we compare like this
if set(doc_names) - set(result_keys):
self.fail("returned doc names have different values than expected")
def test_all_docs_startkey_endkey_validation(self):
"""Regression test for MB-6591
This tests makes sure that the validation of startkey/endkey
works with _all_docs which uses raw collation. Return results
when startkey is smaller than endkey. When endkey is smaller than
the startkey and exception should be raised. With raw collation
"Foo" < "foo", with Unicode collation "foo" < "Foo".
"""
bucket = "default"
master = self.servers[0]
rest = RestConnection(master)
ViewBaseTests._setup_cluster(self)
startkey = '"Foo"'
endkey = '"foo"'
params = {"startkey": startkey, "endkey": endkey}
results = rest.all_docs(bucket, params)
self.assertTrue('rows' in results, "Results were returned")
# Flip startkey and endkey
params = {"startkey": endkey, "endkey": startkey}
self.assertRaises(Exception, rest.all_docs, bucket, params)