def key_not_exists_test(self):
client = SDKClient([self.cluster.master], self.bucket)
collections = BucketUtils.get_random_collections([self.bucket], 1, 1,
1)
scope_dict = collections[self.bucket.name]["scopes"]
scope_name = scope_dict.keys()[0]
collection_name = scope_dict[scope_name]["collections"].keys()[0]
client.select_collection(scope_name, collection_name)
self.log.info("CAS test on collection %s: %s" %
(scope_name, collection_name))
load_gen = doc_generator(self.key, 0, self.num_items, doc_size=256)
key, val = load_gen.next()
for _ in range(1500):
result = client.crud("create",
key,
val,
durability=self.durability_level,
timeout=self.sdk_timeout)
if result["status"] is False:
self.log_failure("Create failed: %s" % result)
create_cas = result["cas"]
# Delete and verify get fails
result = client.crud("delete",
key,
durability=self.durability_level,
timeout=self.sdk_timeout)
if result["status"] is False:
self.log_failure("Delete failed: %s" % result)
elif result["cas"] <= create_cas:
self.log_failure("Delete returned invalid cas: %s" % result)
result = client.crud("read", key, timeout=self.sdk_timeout)
if result["status"] is True:
self.log_failure("Read succeeded after delete: %s" % result)
elif SDKException.DocumentNotFoundException \
not in str(result["error"]):
self.log_failure("Invalid exception during read "
"for non-exists key: %s" % result)
# cas errors do not sleep the test for 10 seconds,
# plus we need to check that the correct error is being thrown
result = client.crud("replace",
key,
val,
exp=60,
timeout=self.sdk_timeout,
cas=create_cas)
if result["status"] is True:
self.log_failure("Replace succeeded after delete: %s" % result)
if SDKException.DocumentNotFoundException \
not in str(result["error"]):
self.log_failure("Invalid exception during read "
"for non-exists key: %s" % result)
# Validate doc count as per bucket collections
self.bucket_util.validate_docs_per_collections_all_buckets()
self.validate_test_failure()
def test_load_duplicate_key_within_same_collection(self):
client = SDKClient([self.cluster.master], self.bucket,
scope=CbServer.default_scope,
collection=CbServer.default_collection,
compression_settings=self.sdk_compression)
result = client.crud("create", "test_key-1", "TestValue")
if result["status"] is True:
self.log.info("CRUD succeeded first time")
result = client.crud("create", "test_key-1", "TestValue")
if result["status"] is True:
self.fail("CRUD succeeded second time when it should have not")
elif result["status"] is False:
self.log.info("CRUD didn't succeed for duplicate key as expected")
def key_not_exists_test(self):
client = SDKClient(self.rest, self.bucket)
load_gen = doc_generator(self.key, 0, 1, doc_size=256)
key, val = load_gen.next()
for _ in range(1500):
result = client.crud("create",
key,
val,
durability=self.durability_level,
timeout=self.sdk_timeout)
if result["status"] is False:
self.log_failure("Create failed: %s" % result)
create_cas = result["cas"]
# Delete and verify get fails
result = client.crud("delete",
key,
durability=self.durability_level,
timeout=self.sdk_timeout)
if result["status"] is False:
self.log_failure("Delete failed: %s" % result)
elif result["cas"] <= create_cas:
self.log_failure("Delete returned invalid cas: %s" % result)
result = client.crud("read", key, timeout=self.sdk_timeout)
if result["status"] is True:
self.log_failure("Read succeeded after delete: %s" % result)
elif DurableExceptions.KeyNotFoundException \
not in str(result["error"]):
self.log_failure("Invalid exception during read "
"for non-exists key: %s" % result)
# cas errors do not sleep the test for 10 seconds,
# plus we need to check that the correct error is being thrown
result = client.crud("replace",
key,
val,
exp=60,
timeout=self.sdk_timeout,
cas=create_cas)
if result["status"] is True:
self.log_failure("Replace succeeded after delete: %s" % result)
if DurableExceptions.KeyNotFoundException \
not in str(result["error"]):
self.log_failure("Invalid exception during read "
"for non-exists key: %s" % result)
self.validate_test_failure()
def touch_test(self):
self.log.info("1. Loading initial set of documents")
load_gen = doc_generator(self.key, 0, self.num_items,
doc_size=self.doc_size)
self._load_all_buckets(load_gen, "create")
self.bucket_util.verify_stats_all_buckets(self.num_items)
self.bucket_util._wait_for_stats_all_buckets()
self.log.info("2. Loading bucket into DGM")
dgm_gen = doc_generator(
self.key, self.num_items, self.num_items+1)
dgm_task = self.task.async_load_gen_docs(
self.cluster, self.bucket_util.buckets[0], dgm_gen, "create", 0,
persist_to=self.persist_to,
replicate_to=self.replicate_to,
durability=self.durability_level,
timeout_secs=self.sdk_timeout,
batch_size=10,
process_concurrency=4,
active_resident_threshold=self.active_resident_threshold)
self.task_manager.get_task_result(dgm_task)
self.log.info("3. Touch intial self.num_items docs which are "
"residing on disk due to DGM")
client = SDKClient([self.cluster.master],
self.bucket_util.buckets[0])
while load_gen.has_next():
key, _ = load_gen.next()
result = client.crud("touch", key,
durability=self.durability_level,
timeout=self.sdk_timeout)
if result["status"] is not True:
self.log_failure("Touch on %s failed: %s" % (key, result))
client.close()
self.validate_test_failure()
def perform_crud_ops():
old_cas = 0
client = SDKClient([self.cluster.master], bucket_obj)
for op_type in ["create", "update", "read", "replace", "delete"]:
crud_desc = "Key %s, doc_op: %s" % (key, op_type)
self.log.info(crud_desc)
result = client.crud(op_type, key, value,
replicate_to=self.replicate_to,
persist_to=self.persist_to)
if op_type != "read":
if op_type != "replace":
dict_key = "ops_%s" % op_type
else:
dict_key = "ops_update"
verification_dict[dict_key] += 1
verification_dict["sync_write_committed_count"] += 1
if result["cas"] == old_cas:
self.log_failure("CAS didn't get updated: %s"
% result["cas"])
elif op_type == "read":
if result["cas"] != old_cas:
self.log_failure("CAS updated for read operation: %s"
% result["cas"])
self.summary.add_step(crud_desc)
old_cas = result["cas"]
client.close()
def test_doc_key_size(self):
"""
Insert document key with min and max key size on each available
collection and validate
:return:
"""
min_doc_gen = doc_generator("test_min_key_size",
0,
self.num_items,
key_size=1,
doc_size=self.doc_size,
mix_key_size=False,
randomize_doc_size=False)
max_doc_gen = doc_generator("test_max_key_size",
0,
self.num_items,
key_size=245,
doc_size=self.doc_size,
mix_key_size=False,
randomize_doc_size=False)
# Set to keep track of all inserted CAS values
known_cas = set()
# Client to insert docs under different collections
client = SDKClient([self.cluster.master],
self.bucket,
compression_settings=self.sdk_compression)
for doc_gen in [min_doc_gen, max_doc_gen]:
while doc_gen.has_next():
key, value = doc_gen.next()
for _, scope in self.bucket.scopes.items():
for _, collection in scope.collections.items():
client.select_collection(scope.name, collection.name)
result = client.crud("create",
key,
value,
self.maxttl,
durability=self.durability_level,
timeout=self.sdk_timeout,
time_unit="seconds")
if result["status"] is False:
self.log_failure(
"Doc create failed for key '%s' "
"collection::scope %s::%s - %s" %
(key, scope.name, collection.name, result))
else:
if result["cas"] in known_cas:
self.log_failure(
"Same CAS exists under different "
"collection: %s" % result)
collection.num_items += 1
known_cas.add(result["cas"])
# Close SDK connection
client.close()
# Validate doc count as per bucket collections
self.bucket_util.validate_docs_per_collections_all_buckets()
self.validate_test_failure()
def test_nru_eviction_impact_on_cbas(self):
self.log.info("Create dataset")
self.cbas_util.create_dataset_on_bucket(self.cb_bucket_name,
self.cbas_dataset_name)
self.log.info("Connect to Local link")
self.cbas_util.connect_link()
self.log.info("Add documents until ram percentage")
self.load_document_until_ram_percentage()
self.log.info("Fetch current document count")
target_bucket = None
self.bucket_util.get_all_buckets()
for tem_bucket in self.bucket_util.buckets:
if tem_bucket.name == self.cb_bucket_name:
target_bucket = tem_bucket
break
item_count = target_bucket.stats.itemCount
self.log.info("Completed base load with %s items" % item_count)
self.log.info("Get initial inserted 100 docs, so they aren't removed")
client = SDKClient([self.cluster.master], target_bucket)
for doc_index in range(100):
doc_key = "test_docs-" + str(doc_index)
client.read(doc_key)
self.log.info("Add 20% more items to trigger NRU")
for doc_index in range(item_count, int(item_count * 1.2)):
doc_key = "key_id-" + str(doc_index)
op_result = client.crud("create",
doc_key,
'{"name":"dave"}',
durability=self.durability_level)
if op_result["status"] is False:
self.log.warning("Insert failed for %s: %s" %
(doc_key, op_result))
# Disconnect the SDK client
client.close()
self.log.info("Validate document count on CBAS")
count_n1ql = self.rest.query_tool(
'select count(*) from %s' %
self.cb_bucket_name)['results'][0]['$1']
if self.cbas_util.validate_cbas_dataset_items_count(
self.cbas_dataset_name, count_n1ql):
pass
else:
self.log.info("Document count mismatch might be due to ejection "
"of documents on KV. Retry again")
count_n1ql = self.rest.query_tool(
'select count(*) from %s' %
self.cb_bucket_name)['results'][0]['$1']
self.assertTrue(self.cbas_util.validate_cbas_dataset_items_count(
self.cbas_dataset_name, count_n1ql),
msg="Count mismatch on CBAS")
def test_no_eviction_impact_on_cbas(self):
self.log.info("Add documents until ram percentage")
self.load_document_until_ram_percentage()
self.log.info("Fetch current document count")
target_bucket = None
buckets = self.bucket_util.get_all_buckets(self.cluster)
for tem_bucket in buckets:
if tem_bucket.name == self.bucket_name:
target_bucket = tem_bucket
break
item_count = target_bucket.stats.itemCount
self.log.info("Completed base load with %s items" % item_count)
self.log.info("Load more until we are out of memory")
client = SDKClient([self.cluster.master], target_bucket)
i = item_count
op_result = {"status": True}
while op_result["status"] is True:
op_result = client.crud("create",
"key-id" + str(i),
'{"name":"dave"}',
durability=self.durability_level)
i += 1
if SDKException.AmbiguousTimeoutException not in op_result["error"] \
or SDKException.RetryReason.KV_TEMPORARY_FAILURE \
not in op_result["error"]:
client.close()
self.fail("Invalid exception for OOM insert: %s" % op_result)
self.log.info('Memory is full at {0} items'.format(i))
self.log.info("As a result added more %s items" % (i - item_count))
self.log.info("Fetch item count")
target_bucket = None
buckets = self.bucket_util.get_all_buckets(self.cluster)
for tem_bucket in buckets:
if tem_bucket.name == self.cb_bucket_name:
target_bucket = tem_bucket
break
item_count_when_oom = target_bucket.stats.itemCount
mem_when_oom = target_bucket.stats.memUsed
self.log.info('Item count when OOM {0} and memory used {1}'.format(
item_count_when_oom, mem_when_oom))
self.log.info("Validate document count on CBAS")
count_n1ql = self.rest.query_tool('select count(*) from %s' %
self.bucket_name)['results'][0]['$1']
dataset = self.cbas_util.list_all_dataset_objs()[0]
if not self.cbas_util.validate_cbas_dataset_items_count(
self.cluster, dataset.full_name, count_n1ql):
self.fail("No. of items in CBAS dataset do not match "
"that in the KV bucket")
def perform_create_deletes():
index = 0
client = SDKClient([self.cluster.master], self.cluster.buckets[0])
self.log.info("Starting ops to create tomb_stones")
while not self.stop_thread:
key = "temp_key--%s" % index
result = client.crud("create", key, "")
if result["status"] is False:
self.log_failure("Key %s create failed: %s" %
(key, result))
break
result = client.crud("delete", key)
if result["status"] is False:
self.log_failure("Key %s delete failed: %s" %
(key, result))
break
index += 1
client.close()
self.log.info("Total keys deleted: %s" % index)
def test_doc_size(self):
"""
Insert document with empty content and max size on each available
collection and validate
:return:
"""
# Empty docs
min_doc_size_gen = doc_generator("test_min_doc_size",
0, self.num_items,
key_size=self.key_size,
doc_size=0,
mix_key_size=False,
randomize_doc_size=False)
# 20 MB docs
max_doc_size_gen = doc_generator("test_max_doc_size",
0, self.num_items,
key_size=self.key_size,
doc_size=1024 * 1024 * 20,
mix_key_size=False,
randomize_doc_size=False)
# Set to keep track of all inserted CAS values
# Format know_cas[CAS] = list(vb_lists)
known_cas = dict()
# Client to insert docs under different collections
client = SDKClient([self.cluster.master], self.bucket,
compression_settings=self.sdk_compression)
for doc_gen in [min_doc_size_gen, max_doc_size_gen]:
while doc_gen.has_next():
key, value = doc_gen.next()
for _, scope in self.bucket.scopes.items():
for _, collection in scope.collections.items():
client.select_collection(scope.name, collection.name)
result = client.crud("create", key, value, self.maxttl,
durability=self.durability_level,
timeout=self.sdk_timeout,
time_unit="seconds")
if result["status"] is False:
self.log_failure("Doc create failed for key '%s' "
"collection::scope %s::%s - %s"
% (key,
scope.name,
collection.name,
result))
else:
self.__validate_cas_for_key(key, result, known_cas)
collection.num_items += 1
# Close SDK connection
client.close()
# Validate doc count as per bucket collections
self.bucket_util.validate_docs_per_collections_all_buckets()
self.validate_test_failure()
def MB36948(self):
node_to_stop = self.servers[0]
self.log.info("Adding index/query node")
self.task.rebalance([self.cluster.master], [self.servers[2]], [],
services=["n1ql,index"])
self.log.info("Creating SDK client connection")
client = SDKClient([self.cluster.master],
self.bucket_util.buckets[0],
compression_settings=self.sdk_compression)
self.log.info("Stopping memcached on: %s" % node_to_stop)
ssh_conn = RemoteMachineShellConnection(node_to_stop)
err_sim = CouchbaseError(self.log, ssh_conn)
err_sim.create(CouchbaseError.STOP_MEMCACHED)
result = client.crud("create", "abort1", "abort1_val")
if not result["status"]:
self.log_failure("Async SET failed")
result = client.crud("update",
"abort1",
"abort1_val",
durability=self.durability_level,
timeout=3,
time_unit="seconds")
if result["status"]:
self.log_failure("Sync write succeeded")
if SDKException.DurabilityAmbiguousException not in result["error"]:
self.log_failure("Invalid exception for sync_write: %s" % result)
self.log.info("Resuming memcached on: %s" % node_to_stop)
err_sim.revert(CouchbaseError.STOP_MEMCACHED)
self.bucket_util._wait_for_stats_all_buckets()
self.bucket_util.verify_stats_all_buckets(1)
self.log.info("Closing ssh & SDK connections")
ssh_conn.disconnect()
client.close()
self.validate_test_failure()
def test_ops_on_same_key(self):
"""
1. Set key
2. Delete a key
3. Set the same key
4. Validate the rev_id for the key is maintained
Ref: MB-48179
"""
if self.durability_level in ["", Bucket.DurabilityLevel.NONE]:
self.fail("Test supported only for sync_write scenarios")
crud_pattern = self.input.param("crud_pattern", "async:sync:async")
crud_pattern = crud_pattern.split(":")
rev_ids = dict()
client = SDKClient([self.cluster.master], self.cluster.buckets[0])
# Async create of keys
for i in range(self.num_items):
key = self.key + str(i)
durability = ""
if crud_pattern[0] == "sync":
durability = self.durability_level
client.crud(DocLoading.Bucket.DocOps.CREATE,
key, {},
durability=durability)
# Sync delete of keys
for i in range(self.num_items):
key = self.key + str(i)
durability = ""
if crud_pattern[1] == "sync":
durability = self.durability_level
client.crud(DocLoading.Bucket.DocOps.DELETE,
key,
durability=durability)
# Async create of keys
for i in range(self.num_items):
key = self.key + str(i)
durability = ""
if crud_pattern[2] == "sync":
durability = self.durability_level
client.crud(DocLoading.Bucket.DocOps.CREATE,
key, {},
durability=durability)
result = client.collection.lookupIn(
key,
Collections.singletonList(
LookupInSpec.get(LookupInMacro.REV_ID).xattr()))
rev_ids[key] = int(result.contentAs(0, String))
client.close()
# Rev_id validation
for i in range(self.num_items):
key = self.key + str(i)
if rev_ids[key] != 3:
self.fail("Rev id mismatch for key '%s'. RevId: %s" %
(key, rev_ids[key]))
def touch_test(self):
self.log.info("Loading bucket into DGM")
load_gen = doc_generator(self.key,
0,
self.num_items,
doc_size=self.doc_size)
dgm_gen = doc_generator(self.key, self.num_items, self.num_items + 1)
dgm_task = self.task.async_load_gen_docs(
self.cluster,
self.bucket_util.buckets[0],
dgm_gen,
"create",
0,
persist_to=self.persist_to,
replicate_to=self.replicate_to,
durability=self.durability_level,
timeout_secs=self.sdk_timeout,
batch_size=10,
process_concurrency=4,
active_resident_threshold=self.active_resident_threshold)
self.task_manager.get_task_result(dgm_task)
self.log.info("Touch intial self.num_items docs which are "
"residing on disk due to DGM")
client = SDKClient([self.cluster.master], self.bucket_util.buckets[0])
collections = BucketUtils.get_random_collections(
self.bucket_util.buckets, 2, 2, 1)
for self.bucket_name, scope_dict in collections.iteritems():
bucket = BucketUtils.get_bucket_obj(self.bucket_util.buckets,
self.bucket_name)
scope_dict = scope_dict["scopes"]
for scope_name, collection_dict in scope_dict.items():
collection_dict = collection_dict["collections"]
for c_name, c_data in collection_dict.items():
self.log.info("CAS test on collection %s: %s" %
(scope_name, c_name))
client.select_collection(scope_name, c_name)
while load_gen.has_next():
key, _ = load_gen.next()
result = client.crud("touch",
key,
durability=self.durability_level,
timeout=self.sdk_timeout)
if result["status"] is not True:
self.log_failure("Touch on %s failed: %s" %
(key, result))
client.close()
self.bucket_util._wait_for_stats_all_buckets()
# Validate doc count as per bucket collections
self.bucket_util.validate_docs_per_collections_all_buckets()
self.validate_test_failure()
def test_durability_impossible(self):
"""
Create bucket with replica > num_kv_nodes.
Perform doc insert to make sure we get TimeoutException due to
durability_impossible from the server.
"""
verification_dict = self.get_cb_stat_verification_dict()
key, value = doc_generator("test_key", 0, 1).next()
for d_level in self.possible_d_levels[self.bucket_type]:
if d_level == Bucket.DurabilityLevel.NONE:
continue
bucket_dict = self.get_bucket_dict(self.bucket_type, d_level)
# Object to support performing CRUDs
bucket_obj = Bucket(bucket_dict)
self.bucket_util.create_bucket(self.cluster,
bucket_obj,
wait_for_warmup=True)
self.summary.add_step("Create bucket with durability %s" % d_level)
client = SDKClient([self.cluster.master], bucket_obj)
result = client.crud("create", key, value, timeout=3)
if result["status"] is True \
or SDKException.DurabilityImpossibleException \
not in result["error"]:
self.log_failure("Indirect sync_write succeeded "
"without enough nodes")
client.close()
# Cbstats vbucket-details validation
self.cb_stat_verify(verification_dict)
# Delete the created bucket
self.bucket_util.delete_bucket(self.cluster, bucket_obj)
self.summary.add_step("Delete bucket with d_level %s" % d_level)
def test_stop_process(self):
"""
1. Starting loading docs into the default bucket
2. Stop the requested process, which will impact the
memcached operations
3. Wait for load bucket task to complete
4. Validate the docs for durability
"""
error_to_simulate = self.input.param("simulate_error", None)
def_bucket = self.bucket_util.buckets[0]
target_node = self.getTargetNode()
remote = RemoteMachineShellConnection(target_node)
error_sim = CouchbaseError(self.log, remote)
target_vbuckets = self.getVbucketNumbers(remote, def_bucket.name,
self.target_node)
if len(target_vbuckets) == 0:
self.log.error("No target vbucket list generated to load data")
remote.disconnect()
return
# Create doc_generator targeting only the active/replica vbuckets
# present in the target_node
gen_load = doc_generator(self.key,
self.num_items,
self.new_docs_to_add,
key_size=self.key_size,
doc_size=self.doc_size,
doc_type=self.doc_type,
target_vbucket=target_vbuckets,
vbuckets=self.cluster_util.vbuckets)
if self.atomicity:
task = self.task.async_load_gen_docs_atomicity(
self.cluster,
self.bucket_util.buckets,
gen_load,
"create",
exp=0,
batch_size=10,
process_concurrency=self.process_concurrency,
replicate_to=self.replicate_to,
persist_to=self.persist_to,
durability=self.durability_level,
timeout_secs=self.sdk_timeout,
update_count=self.update_count,
transaction_timeout=self.transaction_timeout,
commit=True,
sync=self.sync)
else:
task = self.task.async_load_gen_docs(
self.cluster,
def_bucket,
gen_load,
"create",
exp=0,
batch_size=1,
process_concurrency=8,
replicate_to=self.replicate_to,
persist_to=self.persist_to,
durability=self.durability_level,
timeout_secs=self.sdk_timeout,
skip_read_on_error=True)
# Induce the error condition
error_sim.create(error_to_simulate)
self.sleep(20, "Wait before reverting the error condition")
# Revert the simulated error condition and close the ssh session
error_sim.revert(error_to_simulate)
remote.disconnect()
# Wait for doc loading task to complete
self.task.jython_task_manager.get_task_result(task)
if not self.atomicity:
if len(task.fail.keys()) != 0:
if self.target_node == "active" or self.num_replicas in [2, 3]:
self.log_failure("Unwanted failures for keys: %s" %
task.fail.keys())
validate_passed = \
self.durability_helper.validate_durability_exception(
task.fail,
SDKException.DurabilityAmbiguousException)
if not validate_passed:
self.log_failure("Unwanted exception seen during validation")
# Create SDK connection for CRUD retries
sdk_client = SDKClient([self.cluster.master], def_bucket)
for doc_key, crud_result in task.fail.items():
result = sdk_client.crud("create",
doc_key,
crud_result["value"],
replicate_to=self.replicate_to,
persist_to=self.persist_to,
durability=self.durability_level,
timeout=self.sdk_timeout)
if result["status"] is False:
self.log_failure("Retry of doc_key %s failed: %s" %
(doc_key, result["error"]))
# Close the SDK connection
sdk_client.close()
# Update self.num_items
self.num_items += self.new_docs_to_add
if not self.atomicity:
# Validate doc count
self.bucket_util._wait_for_stats_all_buckets()
self.bucket_util.verify_stats_all_buckets(self.num_items)
self.validate_test_failure()
def test_create_scopes(self):
"""
1. Load data into '_default' collection (if required by test)
2. Create scope(s) under the bucket
3. Validate the scopes are created properly
4. Validate '_default' collection is intact
"""
num_scopes = self.input.param("num_scopes", 1)
if self.action_phase == "before_default_load":
BucketUtils.create_scopes(self.cluster, self.bucket, num_scopes)
create_gen = doc_generator("scope_create_key",
0,
self.num_items,
doc_size=self.doc_size,
doc_type=self.doc_type,
target_vbucket=self.target_vbucket,
mutation_type="ADD",
mutate=1,
key_size=self.key_size)
update_gen = doc_generator("scope_create_key",
0,
self.num_items,
doc_size=self.doc_size,
doc_type=self.doc_type,
target_vbucket=self.target_vbucket,
mutation_type="SET",
mutate=2,
key_size=self.key_size)
self.log.info("Loading %d docs into '_default' collection" %
self.num_items)
client = SDKClient([self.cluster.master],
self.bucket,
compression_settings=self.sdk_compression)
while create_gen.has_next():
key, val = create_gen.next()
result = client.crud("create",
key,
val,
exp=self.maxttl,
durability=self.durability_level,
timeout=self.sdk_timeout)
if result["status"] is False:
self.log_failure("Doc create failed for '_default' collection")
break
client.close()
# Update num_items for default collection
self.bucket.scopes[CbServer.default_scope] \
.collections[CbServer.default_collection] \
.num_items += self.num_items
# Doc count validation
self.bucket_util._wait_for_stats_all_buckets()
# Prints bucket stats after doc_ops
self.bucket_util.print_bucket_stats()
self.bucket_util.validate_doc_count_as_per_collections(self.bucket)
# Perform update mutation
task = self.task.async_load_gen_docs(
self.cluster,
self.bucket,
update_gen,
"update",
self.maxttl,
batch_size=10,
process_concurrency=8,
replicate_to=self.replicate_to,
persist_to=self.persist_to,
durability=self.durability_level,
compression=self.sdk_compression,
timeout_secs=self.sdk_timeout,
scope=CbServer.default_scope,
collection=CbServer.default_collection)
# Create scope(s) while CRUDs are running in background
if self.action_phase == "during_default_load":
BucketUtils.create_scopes(self.cluster, self.bucket, num_scopes)
# Validate drop collection using cbstats
for node in self.cluster_util.get_kv_nodes():
shell_conn = RemoteMachineShellConnection(node)
cbstats = Cbstats(shell_conn)
c_data = cbstats.get_collections(self.bucket)
if c_data["count"] != 1:
self.log_failure("%s - Expected scope count is '1'."
"Actual: %s" % (node.ip, c_data["count"]))
if "_default" not in c_data:
self.log_failure("%s: _default collection missing in cbstats" %
node.ip)
# Wait for doc_loading to complete
self.task_manager.get_task_result(task)
self.validate_test_failure()
def verify_cas(self, ops, generator):
"""
Verify CAS value manipulation.
For update we use the latest CAS value return by set()
to do the mutation again to see if there is any exceptions.
We should be able to mutate that item with the latest CAS value.
For delete(), after it is called, we try to mutate that item with the
cas value returned by delete(). We should see SDK Error.
Otherwise the test should fail.
For expire, We want to verify using the latest CAS value of that item
can not mutate it because it is expired already.
"""
for bucket in self.bucket_util.buckets:
client = SDKClient(RestConnection(self.cluster.master), bucket)
gen = generator
while gen.has_next():
key, value = gen.next()
vb_of_key = self.bucket_util.get_vbucket_num_for_key(key)
active_node_ip = None
for node_ip in self.shell_conn.keys():
if vb_of_key in self.vb_details[node_ip]["active"]:
active_node_ip = node_ip
break
self.log.info("Performing %s on key %s" % (ops, key))
if ops in ["update", "touch"]:
for x in range(self.mutate_times):
old_cas = client.crud("read", key, timeout=10)["cas"]
# value = {"val": "mysql-new-value-%s" % x}
if ops == 'update':
result = client.crud(
"replace",
key,
value,
durability=self.durability_level,
cas=old_cas)
else:
result = client.touch(
key,
0,
durability=self.durability_level,
timeout=self.sdk_timeout)
if result["status"] is False:
client.close()
self.log_failure("Touch / replace with cas failed")
return
new_cas = result["cas"]
if old_cas == new_cas:
self.log_failure("CAS old (%s) == new (%s)" %
(old_cas, new_cas))
if ops == 'update':
if json.loads(str(result["value"])) \
!= json.loads(value):
self.log_failure("Value mismatch. "
"%s != %s" %
(result["value"], value))
else:
self.log.debug(
"Mutate %s with CAS %s successfully! "
"Current CAS: %s" %
(old_cas, key, new_cas))
active_read = client.crud("read",
key,
timeout=self.sdk_timeout)
active_cas = active_read["cas"]
replica_cas = -1
cas_in_active_node = \
self.cb_stat[active_node_ip].vbucket_details(
bucket.name)[str(vb_of_key)]["max_cas"]
if str(cas_in_active_node) != str(new_cas):
self.log_failure("CbStats CAS mismatch. %s != %s" %
(cas_in_active_node, new_cas))
poll_count = 0
max_retry = 5
while poll_count < max_retry:
replica_read = client.getFromReplica(
key, ReplicaMode.FIRST)[0]
replica_cas = replica_read["cas"]
if active_cas == replica_cas \
or self.durability_level:
break
poll_count = poll_count + 1
self.sleep(1, "Retry read CAS from replica..")
if active_cas != replica_cas:
self.log_failure("Replica cas mismatch. %s != %s" %
(new_cas, replica_cas))
elif ops == "delete":
old_cas = client.crud("read", key, timeout=10)["cas"]
result = client.crud("delete",
key,
durability=self.durability_level,
timeout=self.sdk_timeout)
self.log.info("CAS after delete of key %s: %s" %
(key, result["cas"]))
result = client.crud("replace",
key,
"test",
durability=self.durability_level,
timeout=self.sdk_timeout,
cas=old_cas)
if result["status"] is True:
self.log_failure("The item should already be deleted")
if DurableExceptions.KeyNotFoundException \
not in result["error"]:
self.log_failure("Invalid Excepetion: %s" % result)
if result["cas"] != 0:
self.log_failure("Delete returned invalid cas: %s, "
"Expected 0" % result["cas"])
if result["cas"] == old_cas:
self.log_failure("Deleted doc returned old cas: %s " %
old_cas)
elif ops == "expire":
old_cas = client.crud("read", key, timeout=10)["cas"]
result = client.crud("touch", key, exp=self.expire_time)
if result["status"] is True:
if result["cas"] == old_cas:
self.log_failure("Touch failed to update CAS")
else:
self.log_failure("Touch operation failed")
self.sleep(self.expire_time + 1, "Wait for item to expire")
result = client.crud("replace",
key,
"test",
durability=self.durability_level,
timeout=self.sdk_timeout,
cas=old_cas)
if result["status"] is True:
self.log_failure("Able to mutate %s with old cas: %s" %
(key, old_cas))
if ClientException.KeyNotFoundException \
not in result["error"]:
self.log_failure("Invalid error after expiry: %s" %
result)
def test_sync_write_in_progress(self):
doc_ops = self.input.param("doc_ops", "create;create").split(';')
shell_conn = dict()
cbstat_obj = dict()
error_sim = dict()
vb_info = dict()
active_vbs = dict()
replica_vbs = dict()
# Override d_level, error_simulation type based on d_level
self.__get_d_level_and_error_to_simulate()
# Acquire SDK client from the pool for performing doc_ops locally
client = SDKClient([self.cluster.master], self.bucket)
target_nodes = DurabilityHelper.getTargetNodes(self.cluster,
self.nodes_init,
self.num_nodes_affected)
for node in target_nodes:
shell_conn[node.ip] = RemoteMachineShellConnection(node)
cbstat_obj[node.ip] = Cbstats(node)
vb_info["init"] = dict()
vb_info["init"][node.ip] = cbstat_obj[node.ip].vbucket_seqno(
self.bucket.name)
error_sim[node.ip] = CouchbaseError(self.log, shell_conn[node.ip])
# Fetch affected nodes' vb_num which are of type=replica
active_vbs[node.ip] = cbstat_obj[node.ip].vbucket_list(
self.bucket.name, vbucket_type="active")
replica_vbs[node.ip] = cbstat_obj[node.ip].vbucket_list(
self.bucket.name, vbucket_type="replica")
if self.durability_level \
== Bucket.DurabilityLevel.MAJORITY_AND_PERSIST_TO_ACTIVE:
target_vbs = active_vbs
target_vbuckets = list()
for target_node in target_nodes:
target_vbuckets += target_vbs[target_node.ip]
else:
target_vbuckets = replica_vbs[target_nodes[0].ip]
if len(target_nodes) > 1:
index = 1
while index < len(target_nodes):
target_vbuckets = list(
set(target_vbuckets).intersection(
set(replica_vbs[target_nodes[index].ip])))
index += 1
doc_load_spec = dict()
doc_load_spec["doc_crud"] = dict()
doc_load_spec["doc_crud"][MetaCrudParams.DocCrud.COMMON_DOC_KEY] \
= "test_collections"
doc_load_spec[MetaCrudParams.TARGET_VBUCKETS] = target_vbuckets
doc_load_spec[MetaCrudParams.COLLECTIONS_CONSIDERED_FOR_CRUD] = 5
doc_load_spec[MetaCrudParams.SCOPES_CONSIDERED_FOR_CRUD] = "all"
doc_load_spec[MetaCrudParams.DURABILITY_LEVEL] = self.durability_level
doc_load_spec[MetaCrudParams.SDK_TIMEOUT] = 60
if doc_ops[0] == DocLoading.Bucket.DocOps.CREATE:
doc_load_spec["doc_crud"][
MetaCrudParams.DocCrud.CREATE_PERCENTAGE_PER_COLLECTION] = 1
elif doc_ops[0] == DocLoading.Bucket.DocOps.UPDATE:
doc_load_spec["doc_crud"][
MetaCrudParams.DocCrud.UPDATE_PERCENTAGE_PER_COLLECTION] = 1
elif doc_ops[0] == DocLoading.Bucket.DocOps.REPLACE:
doc_load_spec["doc_crud"][
MetaCrudParams.DocCrud.REPLACE_PERCENTAGE_PER_COLLECTION] = 1
elif doc_ops[0] == DocLoading.Bucket.DocOps.DELETE:
doc_load_spec["doc_crud"][
MetaCrudParams.DocCrud.DELETE_PERCENTAGE_PER_COLLECTION] = 1
# Induce error condition for testing
for node in target_nodes:
error_sim[node.ip].create(self.simulate_error,
bucket_name=self.bucket.name)
self.sleep(3, "Wait for error simulation to take effect")
doc_loading_task = \
self.bucket_util.run_scenario_from_spec(
self.task,
self.cluster,
self.cluster.buckets,
doc_load_spec,
async_load=True)
self.sleep(5, "Wait for doc ops to reach server")
for bucket, s_dict in doc_loading_task.loader_spec.items():
for s_name, c_dict in s_dict["scopes"].items():
for c_name, c_meta in c_dict["collections"].items():
client.select_collection(s_name, c_name)
for op_type in c_meta:
key, value = c_meta[op_type]["doc_gen"].next()
if self.with_non_sync_writes:
fail = client.crud(doc_ops[1],
key,
value,
exp=0,
timeout=2,
time_unit="seconds")
else:
fail = client.crud(
doc_ops[1],
key,
value,
exp=0,
durability=self.durability_level,
timeout=2,
time_unit="seconds")
expected_exception = \
SDKException.AmbiguousTimeoutException
retry_reason = \
SDKException.RetryReason.KV_SYNC_WRITE_IN_PROGRESS
if doc_ops[0] == DocLoading.Bucket.DocOps.CREATE \
and doc_ops[1] in \
[DocLoading.Bucket.DocOps.DELETE,
DocLoading.Bucket.DocOps.REPLACE]:
expected_exception = \
SDKException.DocumentNotFoundException
retry_reason = None
# Validate the returned error from the SDK
if expected_exception not in str(fail["error"]):
self.log_failure("Invalid exception for %s: %s" %
(key, fail["error"]))
if retry_reason \
and retry_reason not in str(fail["error"]):
self.log_failure(
"Invalid retry reason for %s: %s" %
(key, fail["error"]))
# Try reading the value in SyncWrite state
fail = client.crud("read", key)
if doc_ops[0] == "create":
# Expected KeyNotFound in case of CREATE op
if fail["status"] is True:
self.log_failure(
"%s returned value during SyncWrite %s" %
(key, fail))
else:
# Expects prev val in case of other operations
if fail["status"] is False:
self.log_failure(
"Key %s read failed for prev value: %s" %
(key, fail))
# Revert the introduced error condition
for node in target_nodes:
error_sim[node.ip].revert(self.simulate_error,
bucket_name=self.bucket.name)
# Wait for doc_loading to complete
self.task_manager.get_task_result(doc_loading_task)
self.bucket_util.validate_doc_loading_results(doc_loading_task)
if doc_loading_task.result is False:
self.log_failure("Doc CRUDs failed")
# Release the acquired SDK client
client.close()
self.validate_test_failure()
class basic_ops(BaseTestCase):
def setUp(self):
super(basic_ops, self).setUp()
self.test_log = logging.getLogger("test")
self.key = 'test_docs'.rjust(self.key_size, '0')
nodes_init = self.cluster.servers[1:self.nodes_init] if self.nodes_init != 1 else []
self.task.rebalance([self.cluster.master], nodes_init, [])
self.cluster.nodes_in_cluster.extend([self.cluster.master] + nodes_init)
self.bucket_util.add_rbac_user()
if self.default_bucket:
self.bucket_util.create_default_bucket(replica=self.num_replicas,
compression_mode=self.compression_mode, ram_quota=100)
time.sleep(10)
self.def_bucket= self.bucket_util.get_all_buckets()
self.client = VBucketAwareMemcached(RestConnection(self.cluster.master), self.def_bucket[0])
self.__durability_level()
self.create_Transaction()
self._stop = threading.Event()
self.log.info("==========Finished Basic_ops base setup========")
def tearDown(self):
self.client.close()
super(basic_ops, self).tearDown()
def __durability_level(self):
if self.durability_level == "MAJORITY":
self.durability = 1
elif self.durability_level == "MAJORITY_AND_PERSIST_ON_MASTER":
self.durability = 2
elif self.durability_level == "PERSIST_TO_MAJORITY":
self.durability = 3
elif self.durability_level == "ONLY_NONE":
self.durability = 4
else:
self.durability = 0
def get_doc_generator(self, start, end):
age = range(5)
first = ['james', 'sharon']
body = [''.rjust(self.doc_size - 10, 'a')]
template = '{{ "age": {0}, "first_name": "{1}", "body": "{2}"}}'
generator = DocumentGenerator(self.key, template, age, first, body, start=start,
end=end)
return generator
def set_exception(self, exception):
self.exception = exception
raise BaseException("Got an exception {}".format(self.exception))
def __chunks(self, l, n):
"""Yield successive n-sized chunks from l."""
for i in range(0, len(l), n):
yield l[i:i + n]
def create_Transaction(self, client=None):
if not client:
client = self.client
transaction_config = Transaction().createTransactionConfig(self.transaction_timeout, self.durability)
try:
self.transaction = Transaction().createTansaction(client.cluster, transaction_config)
except Exception as e:
self.set_exception(e)
def __thread_to_transaction(self, transaction, op_type, doc, txn_commit, update_count=1, sync=True, set_exception=True, client=None):
if not client:
client = self.client
if op_type == "create":
exception = Transaction().RunTransaction(transaction, [client.collection], doc, [], [], txn_commit, sync, update_count)
elif op_type == "update":
self.test_log.info("updating all the keys through threads")
exception = Transaction().RunTransaction(transaction, [client.collection], [], doc, [], txn_commit, sync, update_count)
elif op_type == "delete":
exception = Transaction().RunTransaction(transaction, [client.collection], [], [], doc, txn_commit, sync, update_count)
if set_exception:
if exception:
self.set_exception("Failed")
def doc_gen(self, num_items, start=0, value={'value':'value1'}):
self.docs = []
self.keys = []
self.content = self.client.translate_to_json_object(value)
for i in range(start, self.num_items):
key = "test_docs-" + str(i)
doc = Tuples.of(key, self.content)
self.keys.append(key)
self.docs.append(doc)
def verify_doc(self, num_items, client):
for i in range(num_items):
key = "test_docs-" + str(i)
result = client.read(key)
actual_val = self.client.translate_to_json_object(result['value'])
self.assertEquals(self.content, actual_val)
def test_MultiThreadTxnLoad(self):
# Atomicity.basic_retry.basic_ops.test_MultiThreadTxnLoad,num_items=1000
''' Load data through txn, update half the items through different threads
and delete half the items through different threads. if update_retry then update and delete
the same key in two different transaction and make sure update fails '''
self.num_txn = self.input.param("num_txn", 9)
self.update_retry = self.input.param("update_retry", False)
self.doc_gen(self.num_items)
threads = []
# create the docs
exception = Transaction().RunTransaction(self.transaction, [self.client.collection], self.docs, [], [], self.transaction_commit, True, self.update_count)
if exception:
self.set_exception("Failed")
if self.update_retry:
threads.append(threading.Thread(target=self.__thread_to_transaction, args=(self.transaction, "delete", self.keys, self.transaction_commit, self.update_count)))
threads.append(threading.Thread(target=self.__thread_to_transaction, args=(self.transaction, "update", self.keys, self.transaction_commit, self.update_count)))
else:
update_docs = self.__chunks(self.keys[:self.num_items/2], self.num_txn)
delete_docs = self.__chunks(self.keys[self.num_items/2:], self.num_txn)
for keys in update_docs:
threads.append(threading.Thread(target=self.__thread_to_transaction, args=(self.transaction, "update", keys, self.transaction_commit, self.update_count)))
for keys in delete_docs:
threads.append(threading.Thread(target=self.__thread_to_transaction, args=(self.transaction, "delete", keys, self.transaction_commit, self.update_count)))
for thread in threads:
thread.start()
for thread in threads:
thread.join()
self.sleep(60)
if self.update_retry:
for key in self.keys:
result = self.client.read(key)
self.assertEquals(result['status'], False)
else:
self.value = {'mutated':1, 'value':'value1'}
self.content = self.client.translate_to_json_object(self.value)
self.verify_doc(self.num_items/2, self.client)
for key in self.keys[self.num_items/2:]:
result = self.client.read(key)
self.assertEquals(result['status'], False)
def test_basic_retry(self):
''' Load set of data to the cluster, update through 2 different threads, make sure transaction maintains the order of update'''
self.write_conflict = self.input.param("write_conflict", 2)
self.test_log.info("going to create and execute the task")
self.gen_create = self.get_doc_generator(0, self.num_items)
task = self.task.async_load_gen_docs_atomicity(self.cluster, self.def_bucket,
self.gen_create, "create" , exp=0,
batch_size=10,
process_concurrency=8,
replicate_to=self.replicate_to,
persist_to=self.persist_to, timeout_secs=self.sdk_timeout,
retries=self.sdk_retries,update_count=self.update_count, transaction_timeout=self.transaction_timeout,
commit=True,durability=self.durability_level,sync=self.sync)
self.task.jython_task_manager.get_task_result(task)
self.test_log.info("get all the keys in the cluster")
self.doc_gen(self.num_items)
threads = []
for update_count in [2, 4, 6]:
threads.append(threading.Thread(target=self.__thread_to_transaction, args=(self.transaction, "update", self.keys, self.transaction_commit, update_count)))
# Add verification task
if self.transaction_commit:
self.update_count = 6
else:
self.update_count = 0
for thread in threads:
thread.start()
self.sleep(2)
for thread in threads:
thread.join()
self.sleep(10)
task = self.task.async_load_gen_docs_atomicity(self.cluster, self.def_bucket,
self.gen_create, "verify" , exp=0,
batch_size=10,
process_concurrency=8,
replicate_to=self.replicate_to,
persist_to=self.persist_to, timeout_secs=self.sdk_timeout,
retries=self.sdk_retries,update_count=self.update_count, transaction_timeout=self.transaction_timeout,
commit=True,durability=self.durability_level)
self.task.jython_task_manager.get_task_result(task)
def test_basic_retry_async(self):
self.test_log.info("going to create and execute the task")
self.gen_create = self.get_doc_generator(0, self.num_items)
task = self.task.async_load_gen_docs_atomicity(self.cluster, self.def_bucket,
self.gen_create, "create" , exp=0,
batch_size=10,
process_concurrency=1,
replicate_to=self.replicate_to,
persist_to=self.persist_to, timeout_secs=self.sdk_timeout,
retries=self.sdk_retries,update_count=self.update_count, transaction_timeout=self.transaction_timeout,
commit=True,durability=self.durability_level,sync=True,num_threads=1)
self.task.jython_task_manager.get_task_result(task)
self.test_log.info("get all the keys in the cluster")
keys = ["test_docs-0"]*20
exception = Transaction().RunTransaction(self.transaction, [self.client.collection], [], keys, [], self.transaction_commit, False, 0)
if exception:
self.set_exception(Exception(exception))
def basic_concurrency(self):
self.crash = self.input.param("crash", False)
self.doc_gen(self.num_items)
# run transaction
thread = threading.Thread(target=self.__thread_to_transaction, args=(self.transaction, "create", self.docs, self.transaction_commit, self.update_count, True, False))
thread.start()
self.sleep(1)
if self.crash:
self.client.cluster.shutdown()
self.transaction.close()
print "going to create a new transaction"
self.client1 = VBucketAwareMemcached(RestConnection(self.cluster.master), self.def_bucket[0])
self.create_Transaction(self.client1)
self.sleep(self.transaction_timeout+60)
exception = Transaction().RunTransaction(self.transaction, [self.client1.collection], self.docs, [], [], self.transaction_commit, self.sync, self.update_count)
if exception:
time.sleep(60)
self.verify_doc(self.num_items, self.client1)
self.client1.close()
else:
key = "test_docs-0"
# insert will fail
result = self.client.insert(key, "value")
self.assertEqual(result["status"], False)
# Update should pass
result = self.client.upsert(key,"value")
self.assertEqual(result["status"], True)
# delete should pass
result = self.client.delete(key)
self.assertEqual(result["status"], True)
thread.join()
def test_stop_loading(self):
''' Load through transactions and close the transaction abruptly, create a new transaction sleep for 60 seconds and
perform create on the same set of docs '''
self.num_txn = self.input.param("num_txn", 9)
self.doc_gen(self.num_items)
threads = []
docs = list(self.__chunks(self.docs, len(self.docs)/self.num_txn))
for doc in docs:
threads.append(threading.Thread(target=self.__thread_to_transaction, args=(self.transaction, "create", doc, self.transaction_commit, self.update_count, True, False)))
for thread in threads:
thread.start()
self.client.cluster.shutdown()
self.transaction.close()
self.client1 = VBucketAwareMemcached(RestConnection(self.cluster.master), self.def_bucket[0])
self.create_Transaction(self.client1)
self.sleep(self.transaction_timeout+60) # sleep for 60 seconds so that transaction cleanup can happen
self.test_log.info("going to start the load")
for doc in docs:
exception = Transaction().RunTransaction(self.transaction, [self.client1.collection], doc, [], [], self.transaction_commit, self.sync, self.update_count)
if exception:
time.sleep(60)
self.verify_doc(self.num_items, self.client1)
self.client1.close()
def __insert_sub_doc_and_validate(self, doc_id, op_type, key, value):
_, failed_items = self.client.crud(
op_type,
doc_id,
[key, value],
durability=self.durability_level,
timeout=self.sdk_timeout,
time_unit="seconds",
create_path=True,
xattr=True)
self.assertFalse(failed_items, "Subdoc Xattr insert failed")
def __read_doc_and_validate(self, doc_id, expected_val, subdoc_key=None):
if subdoc_key:
success, failed_items = self.client.crud("subdoc_read",
doc_id,
subdoc_key,
xattr=True)
self.assertFalse(failed_items, "Xattr read failed")
self.assertEqual(expected_val,
str(success[doc_id]["value"][0]),
"Sub_doc value mismatch: %s != %s"
% (success[doc_id]["value"][0],
expected_val))
def test_TxnWithXattr(self):
self.system_xattr = self.input.param("system_xattr", False)
if self.system_xattr:
xattr_key = "my._attr"
else:
xattr_key="my.attr"
val = "v" * self.doc_size
self.doc_gen(self.num_items)
threads = threading.Thread(target=self.__thread_to_transaction, args=(self.transaction, "create", self.docs, self.transaction_commit, self.update_count))
threads.start()
self.sleep(1)
self.__insert_sub_doc_and_validate("test_docs-0", "subdoc_insert",
xattr_key, val)
threads.join()
if self.transaction_commit:
self.__read_doc_and_validate("test_docs-0", val, xattr_key)
self.sleep(60)
self.verify_doc(self.num_items, self.client)
def test_TxnWithMultipleXattr(self):
xattrs_to_insert = [["my.attr", "value"],
["new_my.attr", "new_value"]]
self.doc_gen(self.num_items)
threads = threading.Thread(target=self.__thread_to_transaction, args=(self.transaction, "create", self.docs, self.transaction_commit, self.update_count))
threads.start()
self.sleep(1)
for key, val in xattrs_to_insert:
self.__insert_sub_doc_and_validate("test_docs-0", "subdoc_insert",
key, val)
threads.join()
if self.transaction_commit:
for key, val in xattrs_to_insert:
self.__read_doc_and_validate("test_docs-0", val, key)
self.sleep(60)
self.verify_doc(self.num_items, self.client)
class OpsChangeCasTests(CasBaseTest):
def setUp(self):
super(OpsChangeCasTests, self).setUp()
self.key = "test_cas"
self.expire_time = self.input.param("expire_time", 35)
self.item_flag = self.input.param("item_flag", 0)
self.load_gen = doc_generator(self.key, 0, self.num_items,
doc_size=self.doc_size)
self.node_data = dict()
for node in self.cluster_util.get_kv_nodes():
shell = RemoteMachineShellConnection(node)
cb_stat = Cbstats(shell)
self.node_data[node.ip] = dict()
self.node_data[node.ip]["shell"] = shell
self.node_data[node.ip]["cb_stat"] = Cbstats(shell)
self.node_data[node.ip]["active"] = cb_stat.vbucket_list(
self.bucket,
"active")
self.node_data[node.ip]["replica"] = cb_stat.vbucket_list(
self.bucket,
"replica")
if self.sdk_client_pool:
self.client = self.sdk_client_pool.get_client_for_bucket(
self.bucket)
else:
self.client = SDKClient([self.cluster.master], self.bucket)
def tearDown(self):
# Close opened shell connections
for node_ip in self.node_data.keys():
self.node_data[node_ip]["shell"].disconnect()
# Close SDK client connection
self.client.close()
super(OpsChangeCasTests, self).tearDown()
def test_meta_rebalance_out(self):
KEY_NAME = 'key1'
for i in range(10):
# set a key
value = 'value' + str(i)
self.client.memcached(KEY_NAME).set(KEY_NAME, 0, 0, json.dumps({'value':value}))
vbucket_id = self.client._get_vBucket_id(KEY_NAME)
mc_active = self.client.memcached(KEY_NAME)
mc_replica = self.client.memcached_for_replica_vbucket(vbucket_id)
cas_active = mc_active.getMeta(KEY_NAME)[4]
max_cas = int(mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(KEY_NAME)) + ':max_cas'] )
self.assertTrue(cas_active == max_cas, '[ERROR]Max cas is not 0 it is {0}'.format(cas_active))
# remove that node
self.log.info('Remove the node with active data')
rebalance = self.cluster.async_rebalance(self.servers[-1:],
[],
[self.master])
rebalance.result()
replica_cas = mc_replica.getMeta(KEY_NAME)[4]
get_meta_resp = mc_replica.getMeta(KEY_NAME,
request_extended_meta_data=False)
# Add the node back
self.log.info('Add the node back, the max_cas should be healed')
rebalance = self.cluster.async_rebalance(self.servers[-1:],
[self.master], [])
if rebalance.result is False:
self.log_failure("Node add-back rebalance failed")
# verify the CAS is good
mc_active = self.client.memcached(KEY_NAME)
active_cas = mc_active.getMeta(KEY_NAME)[4]
if replica_cas == active_cas:
self.log_failure("CAS mismatch. Active: %s, Replica: %s"
% (active_cas, replica_cas))
self.validate_test_failure()
def test_meta_failover(self):
KEY_NAME = 'key2'
for i in range(10):
# set a key
value = 'value' + str(i)
client.memcached(KEY_NAME).set(KEY_NAME, 0, 0,json.dumps({'value':value}))
vbucket_id = client._get_vBucket_id(KEY_NAME)
#print 'vbucket_id is {0}'.format(vbucket_id)
mc_active = client.memcached(KEY_NAME)
mc_master = client.memcached_for_vbucket( vbucket_id )
mc_replica = client.memcached_for_replica_vbucket(vbucket_id)
cas_active = mc_active.getMeta(KEY_NAME)[4]
#print 'cas_a {0} '.format(cas_active)
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(client._get_vBucket_id(KEY_NAME)) + ':max_cas'] )
self.assertTrue(cas_active == max_cas, '[ERROR]Max cas is not 0 it is {0}'.format(cas_active))
# failover that node
self.log.info('Failing over node with active data {0}'.format(self.master))
self.cluster.failover(self.servers, [self.master])
self.log.info('Remove the node with active data {0}'.format(self.master))
rebalance = self.cluster.async_rebalance(self.servers[:], [] ,[self.master])
rebalance.result()
self.sleep(60)
replica_CAS = mc_replica.getMeta(KEY_NAME)[4]
#print 'replica CAS {0}'.format(replica_CAS)
# add the node back
self.log.info('Add the node back, the max_cas should be healed')
rebalance = self.cluster.async_rebalance(self.servers[-1:], [self.master], [])
rebalance.result()
# verify the CAS is good
mc_active = client.memcached(KEY_NAME)
active_CAS = mc_active.getMeta(KEY_NAME)[4]
#print 'active cas {0}'.format(active_CAS)
get_meta_resp = mc_active.getMeta(KEY_NAME,request_extended_meta_data=False)
#print 'replica CAS {0}'.format(replica_CAS)
#print 'replica ext meta {0}'.format(get_meta_resp)
self.assertTrue(replica_CAS == active_CAS, 'cas mismatch active: {0} replica {1}'.format(active_CAS,replica_CAS))
self.assertTrue( get_meta_resp[5] == 1, msg='Metadata indicate conflict resolution is not set')
def test_meta_soft_restart(self):
KEY_NAME = 'key2'
for i in range(10):
# set a key
value = 'value' + str(i)
client.memcached(KEY_NAME).set(KEY_NAME, 0, 0,json.dumps({'value':value}))
vbucket_id = client._get_vBucket_id(KEY_NAME)
#print 'vbucket_id is {0}'.format(vbucket_id)
mc_active = client.memcached(KEY_NAME)
mc_master = client.memcached_for_vbucket( vbucket_id )
mc_replica = client.memcached_for_replica_vbucket(vbucket_id)
cas_pre = mc_active.getMeta(KEY_NAME)[4]
#print 'cas_a {0} '.format(cas_pre)
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(client._get_vBucket_id(KEY_NAME)) + ':max_cas'] )
self.assertTrue(cas_pre == max_cas, '[ERROR]Max cas is not 0 it is {0}'.format(cas_pre))
# restart nodes
self._restart_server(self.servers[:])
# verify the CAS is good
mc_active = client.memcached(KEY_NAME)
cas_post = mc_active.getMeta(KEY_NAME)[4]
#print 'post cas {0}'.format(cas_post)
get_meta_resp = mc_active.getMeta(KEY_NAME,request_extended_meta_data=False)
#print 'post CAS {0}'.format(cas_post)
#print 'post ext meta {0}'.format(get_meta_resp)
self.assertTrue(cas_pre == cas_post, 'cas mismatch active: {0} replica {1}'.format(cas_pre, cas_post))
# extended meta is not supported self.assertTrue( get_meta_resp[5] == 1, msg='Metadata indicate conflict resolution is not set')
def test_meta_hard_restart(self):
KEY_NAME = 'key2'
for i in range(10):
# set a key
value = 'value' + str(i)
client.memcached(KEY_NAME).set(KEY_NAME, 0, 0,json.dumps({'value':value}))
vbucket_id = client._get_vBucket_id(KEY_NAME)
#print 'vbucket_id is {0}'.format(vbucket_id)
mc_active = client.memcached(KEY_NAME)
mc_master = client.memcached_for_vbucket( vbucket_id )
mc_replica = client.memcached_for_replica_vbucket(vbucket_id)
cas_pre = mc_active.getMeta(KEY_NAME)[4]
#print 'cas_a {0} '.format(cas_pre)
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(client._get_vBucket_id(KEY_NAME)) + ':max_cas'] )
self.assertTrue(cas_pre == max_cas, '[ERROR]Max cas is not 0 it is {0}'.format(cas_pre))
# reboot nodes
self._reboot_server()
self.sleep(60)
# verify the CAS is good
mc_active = client.memcached(KEY_NAME)
cas_post = mc_active.getMeta(KEY_NAME)[4]
#print 'post cas {0}'.format(cas_post)
get_meta_resp = mc_active.getMeta(KEY_NAME,request_extended_meta_data=False)
#print 'post CAS {0}'.format(cas_post)
#print 'post ext meta {0}'.format(get_meta_resp)
self.assertTrue(cas_pre == cas_post, 'cas mismatch active: {0} replica {1}'.format(cas_pre, cas_post))
# extended meta is not supported self.assertTrue( get_meta_resp[5] == 1, msg='Metadata indicate conflict resolution is not set')
def test_cas_set(self):
"""
Test Incremental sets on cas and max cas values for keys
"""
self._load_ops(self.load_gen, 'update', mutations=20)
self._check_cas(self.load_gen, check_conflict_resolution=False)
self.validate_test_failure()
def test_cas_updates(self):
"""
Test Incremental updates on cas and max cas values for keys
"""
self._load_ops(self.load_gen, 'update', mutations=20)
self._load_ops(self.load_gen, 'replace', mutations=20)
self._check_cas(self.load_gen, check_conflict_resolution=False)
self.validate_test_failure()
def test_cas_deletes(self):
"""
Test Incremental deletes on cas and max cas values for keys
"""
self._load_ops(self.load_gen, 'create')
self._load_ops(self.load_gen, 'replace', mutations=20)
self._check_cas(self.load_gen, check_conflict_resolution=False)
self._load_ops(self.load_gen, 'delete')
self._check_cas(self.load_gen, check_conflict_resolution=False,
docs_deleted=True)
self.validate_test_failure()
''' Test expiry on cas and max cas values for keys
'''
def test_cas_expiry(self):
self._load_ops(self.load_gen, 'create')
self._load_ops(self.load_gen, 'expiry')
self._check_cas(self.load_gen, check_conflict_resolution=False)
self._check_expiry(self.load_gen)
self.validate_test_failure()
def test_cas_touch(self):
"""
Test touch on cas and max cas values for keys
"""
self.log.info('Starting test-touch')
self._load_ops(self.load_gen, 'update', mutations=20)
self._load_ops(self.load_gen, 'touch')
self._check_cas(self.load_gen, check_conflict_resolution=False)
self.validate_test_failure()
def test_cas_getMeta(self):
"""
Test getMeta on cas and max cas values for keys
"""
self.log.info('Starting test-getMeta')
self._load_ops(ops='set', mutations=20)
self._check_cas(check_conflict_resolution=False)
#self._load_ops(ops='add')
self._load_ops(ops='replace',mutations=20)
self._check_cas(check_conflict_resolution=False)
self._load_ops(ops='delete')
self._check_cas(check_conflict_resolution=False)
def test_cas_setMeta_lower(self):
self.log.info(' Starting test-getMeta')
# set some kv
self._load_ops(ops='set', mutations=1)
#self._check_cas(check_conflict_resolution=False)
k=0
while k<10:
key = "{0}{1}".format(self.prefix, k)
k += 1
vbucket_id = self.client._get_vBucket_id(key)
self.log.info('For key {0} the vbucket is {1}'.format( key,vbucket_id ))
#print 'vbucket_id is {0}'.format(vbucket_id)
mc_active = self.client.memcached(key)
mc_master = self.client.memcached_for_vbucket( vbucket_id )
#mc_replica = self.client.memcached_for_replica_vbucket(vbucket_id)
TEST_SEQNO = 123
TEST_CAS = k
rc = mc_active.getMeta(key)
cas = rc[4] + 1
self.log.info('Key {0} retrieved CAS is {1} and will set CAS to {2}'.format(key, rc[4], cas))
rev_seqno = rc[3]
# do a set meta based on the existing CAS
set_with_meta_resp = mc_active.setWithMeta(key, '123456789', 0, 0, 123, cas)
# check what get meta say
rc = mc_active.getMeta(key)
cas_post_meta = rc[4]
self.log.info('Getmeta CAS is {0}'.format(cas_post_meta))
self.assertTrue( cas_post_meta == cas, 'Meta expected {0} actual {1}'.format( cas, cas_post_meta))
# and what stats says
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.log.info('Max CAS for key {0} vbucket is {1}'.format( key, max_cas))
self.assertTrue(cas_post_meta >= max_cas, '[ERROR]Max cas is not higher it is lower than {0}'.format(cas_post_meta))
self.assertTrue(max_cas == cas, '[ERROR]Max cas is not equal to original cas {0}'.format(cas))
# do another mutation and compare
mc_active.set(key, 0, 0,json.dumps({'value':'value3'}))
cas = mc_active.getMeta(key)[4]
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.assertTrue(max_cas == cas, '[ERROR]Max cas is not equal to cas {0}'.format(cas))
# and then mix in a set with meta
set_with_meta_resp = mc_active.setWithMeta(key, '123456789', 0, 0, 225, max_cas+1)
cas_post_meta = mc_active.getMeta(key)[4]
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.assertTrue(cas_post_meta == max_cas, '[ERROR]Max cas is not higher it is lower than {0}'.format(cas_post_meta))
# and one more mutation for good measure
mc_active.set(key, 0, 0,json.dumps({'value':'value3'}))
cas = mc_active.getMeta(key)[4]
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.assertTrue(max_cas == cas, '[ERROR]Max cas is not equal to cas {0}'.format(cas))
def test_cas_setMeta_higher(self):
self.log.info(' Starting test-getMeta')
self._load_ops(ops='set', mutations=20)
self._check_cas(check_conflict_resolution=False)
k=0
while k<10:
key = "{0}{1}".format(self.prefix, k)
k += 1
vbucket_id = self.client._get_vBucket_id(key)
#print 'vbucket_id is {0}'.format(vbucket_id)
mc_active = self.client.memcached(key)
mc_master = self.client.memcached_for_vbucket( vbucket_id )
#mc_replica = self.client.memcached_for_replica_vbucket(vbucket_id)
get_meta_1 = mc_active.getMeta(key,request_extended_meta_data=False)
#print 'cr {0}'.format(get_meta_1)
#print '-'*100
TEST_SEQNO = 123
TEST_CAS = 9966180844186042368
cas = mc_active.getMeta(key)[4]
#set_with_meta_resp = mc_active.set_with_meta(key, 0, 0, TEST_SEQNO, TEST_CAS, '123456789',vbucket_id,
# add_extended_meta_data=True, conflict_resolution_mode=1)
set_with_meta_resp = mc_active.setWithMeta(key, '123456789', 0, 0, TEST_SEQNO, TEST_CAS)
cas_post_meta = mc_active.getMeta(key)[4]
get_meta_2 = mc_active.getMeta(key,request_extended_meta_data=False)
#print 'cr2 {0}'.format(get_meta_2)
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.assertTrue(cas_post_meta == max_cas, '[ERROR]Max cas is not equal it is {0}'.format(cas_post_meta))
self.assertTrue(max_cas > cas, '[ERROR]Max cas is not higher than original cas {0}'.format(cas))
mc_active.set(key, 0, 0,json.dumps({'value':'value3'}))
cas = mc_active.getMeta(key)[4]
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.assertTrue(max_cas == cas, '[ERROR]Max cas is not equal to cas {0}'.format(cas))
set_with_meta_resp = mc_active.setWithMeta(key, '123456789', 0, 0, TEST_SEQNO, max_cas+1)
#set_with_meta_resp = mc_active.set_with_meta(key, 0, 0, 125, TEST_CAS+1, '223456789',vbucket_id,
# add_extended_meta_data=True, conflict_resolution_mode=1)
cas_post_meta = mc_active.getMeta(key)[4]
get_meta_3 = mc_active.getMeta(key,request_extended_meta_data=False)
#print 'cr3 {0}'.format(get_meta_3)
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.assertTrue(cas_post_meta == max_cas, '[ERROR]Max cas is not lower it is higher than {0}'.format(cas_post_meta))
#self.assertTrue(max_cas == cas, '[ERROR]Max cas is not equal to original cas {0}'.format(cas))
mc_active.set(key, 0, 0,json.dumps({'value':'value3'}))
cas = mc_active.getMeta(key)[4]
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.assertTrue(max_cas == cas, '[ERROR]Max cas is not equal to cas {0}'.format(cas))
''' Test deleteMeta on cas and max cas values for keys
'''
def test_cas_deleteMeta(self):
self.log.info(' Starting test-deleteMeta')
# load 20 kvs and check the CAS
self._load_ops(ops='set', mutations=20)
self.sleep(60)
self._check_cas(check_conflict_resolution=False)
k = 0
test_cas = 456
while k < 1:
key = "{0}{1}".format(self.prefix, k)
k += 1
vbucket_id = self.client._get_vBucket_id(key)
mc_active = self.client.memcached(key)
mc_master = self.client.memcached_for_vbucket( vbucket_id )
mc_replica = self.client.memcached_for_replica_vbucket(vbucket_id)
TEST_SEQNO = 123
test_cas = test_cas + 1
# get the meta data
cas = mc_active.getMeta(key)[4] + 1
set_with_meta_resp = mc_active.setWithMeta(key, '123456789', 0, 0, TEST_SEQNO, cas)
cas_post_meta = mc_active.getMeta(key)[4]
# verify the observed CAS is as set
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.assertTrue(max_cas == cas, '[ERROR]Max cas {0} is not equal to original cas {1}'.format(max_cas, cas))
mc_active.set(key, 0, 0,json.dumps({'value':'value3'}))
cas = mc_active.getMeta(key)[4]
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.assertTrue(max_cas == cas, '[ERROR]Max cas is not equal to cas {0}'.format(cas))
# what is test cas for? Commenting out for now
"""
set_with_meta_resp = mc_active.setWithMeta(key, '123456789', 0, 0, TEST_SEQNO, test_cas)
cas_post_meta = mc_active.getMeta(key)[4]
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.assertTrue(cas_post_meta < max_cas, '[ERROR]Max cas is not higher it is lower than {0}'.format(cas_post_meta))
self.assertTrue(max_cas == cas, '[ERROR]Max cas is not equal to original cas {0}'.format(cas))
"""
# test the delete
mc_active.set(key, 0, 0,json.dumps({'value':'value3'}))
cas = mc_active.getMeta(key)[4]
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.assertTrue(max_cas == cas, '[ERROR]Max cas is not equal to cas {0}'.format(cas))
#
self.log.info('Doing delete with meta, using a lower CAS value')
get_meta_pre = mc_active.getMeta(key)[4]
del_with_meta_resp = mc_active.del_with_meta(key, 0, 0, TEST_SEQNO, test_cas, test_cas+1)
get_meta_post = mc_active.getMeta(key)[4]
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.assertTrue(max_cas > test_cas+1, '[ERROR]Max cas {0} is not greater than delete cas {1}'.format(max_cas, test_cas))
''' Testing skipping conflict resolution, whereby the last write wins, and it does neither cas CR nor rev id CR
'''
def test_cas_skip_conflict_resolution(self):
self.log.info(' Starting test_cas_skip_conflict_resolution ..')
self._load_ops(ops='set', mutations=20)
self._check_cas(check_conflict_resolution=False)
k=0
#Check for first 20 keys
while k<20:
key = "{0}{1}".format(self.prefix, k)
k += 1
vbucket_id = self.client._get_vBucket_id(key)
mc_active = self.client.memcached(key)
mc_master = self.client.memcached_for_vbucket( vbucket_id )
mc_replica = self.client.memcached_for_replica_vbucket(vbucket_id)
low_seq=12
cas = mc_active.getMeta(key)[4]
pre_seq = mc_active.getMeta(key)[3]
all = mc_active.getMeta(key)
self.log.info('all meta data before set_meta_force {0}'.format(all))
self.log.info('Forcing conflict_resolution to allow insertion of lower Seq Number')
lower_cas = int(cas)-1
#import pdb;pdb.set_trace()
#set_with_meta_resp = mc_active.set_with_meta(key, 0, 0, low_seq, lower_cas, '123456789',vbucket_id)
set_with_meta_resp = mc_active.setWithMeta(key, '123456789', 0, 0, low_seq, lower_cas, 3)
cas_post_meta = mc_active.getMeta(key)[4]
all_post_meta = mc_active.getMeta(key)
post_seq = mc_active.getMeta(key)[3]
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.log.info('Expect No conflict_resolution to occur, and the last updated mutation to be the winner..')
#print 'cas meta data after set_meta_force {0}'.format(cas_post_meta)
#print 'all meta data after set_meta_force {0}'.format(all_post_meta)
self.log.info('all meta data after set_meta_force {0}'.format(all_post_meta))
self.assertTrue(max_cas == cas, '[ERROR]Max cas {0} is not equal to original cas {1}'.format(max_cas, cas))
self.assertTrue(pre_seq > post_seq, '[ERROR]Pre rev id {0} is not greater than post rev id {1}'.format(pre_seq, post_seq))
''' Testing revid based conflict resolution with timeSync enabled, where cas on either mutations match and it does rev id CR
'''
def test_revid_conflict_resolution(self):
self.log.info(' Starting test_cas_revid_conflict_resolution ..')
self._load_ops(ops='set', mutations=20)
self._check_cas(check_conflict_resolution=False)
k=0
#Check for first 20 keys
while k<20:
key = "{0}{1}".format(self.prefix, k)
k += 1
vbucket_id = self.client._get_vBucket_id(key)
mc_active = self.client.memcached(key)
mc_master = self.client.memcached_for_vbucket( vbucket_id )
mc_replica = self.client.memcached_for_replica_vbucket(vbucket_id)
new_seq=121
cas = mc_active.getMeta(key)[4]
pre_seq = mc_active.getMeta(key)[3]
all = mc_active.getMeta(key)
self.log.info('all meta data before set_meta_force {0}'.format(all))
self.log.info('Forcing conflict_resolution to rev-id by matching inserting cas ')
set_with_meta_resp = mc_active.set_with_meta(key, 0, 0, new_seq, cas, '123456789',vbucket_id,
add_extended_meta_data=True, conflict_resolution_mode=1)
cas_post_meta = mc_active.getMeta(key)[4]
all_post_meta = mc_active.getMeta(key)
post_seq = mc_active.getMeta(key)[3]
get_meta_2 = mc_active.getMeta(key,request_extended_meta_data=False)
#print 'cr2 {0}'.format(get_meta_2)
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.log.info('Expect No conflict_resolution to occur, and the last updated mutation to be the winner..')
self.log.info('all meta data after set_meta_force {0}'.format(all_post_meta))
self.assertTrue(max_cas == cas, '[ERROR]Max cas {0} is not equal to original cas {1}'.format(max_cas, cas))
self.assertTrue(pre_seq < post_seq, '[ERROR]Pre rev id {0} is not greater than post rev id {1}'.format(pre_seq, post_seq))
''' Testing conflict resolution, where timeSync is enabled and cas is lower but higher revid, expect Higher Cas to Win
'''
def test_cas_conflict_resolution(self):
self.log.info(' Starting test_cas_conflict_resolution ..')
self._load_ops(ops='set', mutations=20)
self._check_cas(check_conflict_resolution=False)
k=0
#Check for first 20 keys
while k<20:
key = "{0}{1}".format(self.prefix, k)
k += 1
vbucket_id = self.client._get_vBucket_id(key)
mc_active = self.client.memcached(key)
mc_master = self.client.memcached_for_vbucket( vbucket_id )
mc_replica = self.client.memcached_for_replica_vbucket(vbucket_id)
new_seq=121
cas = mc_active.getMeta(key)[4]
pre_seq = mc_active.getMeta(key)[3]
all = mc_active.getMeta(key)
self.log.info('all meta data before set_meta_force {0}'.format(all))
lower_cas = int(cas)-100
self.log.info('Forcing lower rev-id to win with higher CAS value, instead of higher rev-id with Lower Cas ')
#set_with_meta_resp = mc_active.set_with_meta(key, 0, 0, new_seq, lower_cas, '123456789',vbucket_id)
try:
set_with_meta_resp = mc_active.setWithMeta(key, '123456789', 0, 0, new_seq, lower_cas)
except mc_bin_client.MemcachedError as e:
# this is expected
pass
cas_post_meta = mc_active.getMeta(key)[4]
all_post_meta = mc_active.getMeta(key)
post_seq = mc_active.getMeta(key)[3]
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.log.info('Expect CAS conflict_resolution to occur, and the first mutation to be the winner..')
self.log.info('all meta data after set_meta_force {0}'.format(all_post_meta))
self.assertTrue(max_cas == cas, '[ERROR]Max cas {0} is not equal to original cas {1}'.format(max_cas, cas))
#self.assertTrue(pre_seq < post_seq, '[ERROR]Pre rev id {0} is not greater than post rev id {1}'.format(pre_seq, post_seq))
''' Testing revid based conflict resolution with timeSync enabled, where cas on either mutations match and it does rev id CR
and retains it after a restart server'''
def test_restart_revid_conflict_resolution(self):
self.log.info(' Starting test_restart_revid_conflict_resolution ..')
self._load_ops(ops='set', mutations=20)
k = 0
key = "{0}{1}".format(self.prefix, k)
vbucket_id = self.client._get_vBucket_id(key)
mc_active = self.client.memcached(key)
mc_master = self.client.memcached_for_vbucket( vbucket_id )
mc_replica = self.client.memcached_for_replica_vbucket(vbucket_id)
# set a key
value = 'value0'
client.memcached(key).set(key, 0, 0,json.dumps({'value':value}))
vbucket_id = client._get_vBucket_id(key)
#print 'vbucket_id is {0}'.format(vbucket_id)
mc_active = client.memcached(key)
mc_master = client.memcached_for_vbucket( vbucket_id )
mc_replica = client.memcached_for_replica_vbucket(vbucket_id)
new_seq=121
pre_cas = mc_active.getMeta(key)[4]
pre_seq = mc_active.getMeta(key)[3]
pre_max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
all = mc_active.getMeta(key)
get_meta_1 = mc_active.getMeta(key,request_extended_meta_data=False)
#print 'cr {0}'.format(get_meta_1)
self.log.info('all meta data before set_meta_force {0}'.format(all))
self.log.info('max_cas before set_meta_force {0}'.format(pre_max_cas))
self.log.info('Forcing conflict_resolution to rev-id by matching inserting cas ')
try:
set_with_meta_resp = mc_active.set_with_meta(key, 0, 0, new_seq, pre_cas, '123456789',vbucket_id)
except mc_bin_client.MemcachedError as e:
# this is expected
pass
cas_post = mc_active.getMeta(key)[4]
all_post_meta = mc_active.getMeta(key)
post_seq = mc_active.getMeta(key)[3]
get_meta_2 = mc_active.getMeta(key,request_extended_meta_data=False)
#print 'cr {0}'.format(get_meta_2)
max_cas_post = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.log.info('Expect RevId conflict_resolution to occur, and the last updated mutation to be the winner..')
self.log.info('all meta data after set_meta_force {0}'.format(all_post_meta))
#self.assertTrue(max_cas_post == pre_cas, '[ERROR]Max cas {0} is not equal to original cas {1}'.format(max_cas_post, pre_cas))
#self.assertTrue(pre_seq < post_seq, '[ERROR]Pre rev id {0} is not greater than post rev id {1}'.format(pre_seq, post_seq))
# Restart Nodes
self._restart_server(self.servers[:])
# verify the CAS is good
mc_active = client.memcached(key)
cas_restart = mc_active.getMeta(key)[4]
#print 'post cas {0}'.format(cas_post)
get_meta_resp = mc_active.getMeta(key,request_extended_meta_data=False)
#print 'post CAS {0}'.format(cas_post)
#print 'post ext meta {0}'.format(get_meta_resp)
self.assertTrue(pre_cas == cas_post, 'cas mismatch active: {0} replica {1}'.format(pre_cas, cas_post))
#self.assertTrue( get_meta_resp[5] == 1, msg='Metadata indicate conflict resolution is not set')
''' Testing revid based conflict resolution with timeSync enabled, where cas on either mutations match and it does rev id CR
and retains it after a rebalance server'''
def test_rebalance_revid_conflict_resolution(self):
self.log.info(' Starting test_rebalance_revid_conflict_resolution ..')
self._load_ops(ops='set', mutations=20)
key = 'key1'
value = 'value'
client.memcached(key).set(key, 0, 0,json.dumps({'value':value}))
vbucket_id = client._get_vBucket_id(key)
#print 'vbucket_id is {0}'.format(vbucket_id)
mc_active = client.memcached(key)
mc_master = client.memcached_for_vbucket( vbucket_id )
mc_replica = client.memcached_for_replica_vbucket(vbucket_id)
new_seq=121
pre_cas = mc_active.getMeta(key)[4]
pre_seq = mc_active.getMeta(key)[3]
pre_max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
all = mc_active.getMeta(key)
get_meta_1 = mc_active.getMeta(key,request_extended_meta_data=False)
#print 'cr {0}'.format(get_meta_1)
self.log.info('all meta data before set_meta_force {0}'.format(all))
self.log.info('max_cas before set_meta_force {0}'.format(pre_max_cas))
self.log.info('Forcing conflict_resolution to rev-id by matching inserting cas ')
#set_with_meta_resp = mc_active.set_with_meta(key, 0, 0, new_seq, pre_cas, '123456789',vbucket_id)
set_with_meta_resp = mc_active.setWithMeta(key, '123456789', 0, 0, new_seq, pre_cas)
cas_post = mc_active.getMeta(key)[4]
all_post_meta = mc_active.getMeta(key)
post_seq = mc_active.getMeta(key)[3]
get_meta_2 = mc_active.getMeta(key,request_extended_meta_data=False)
#print 'cr {0}'.format(get_meta_2)
max_cas_post = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.log.info('Expect RevId conflict_resolution to occur, and the last updated mutation to be the winner..')
self.log.info('all meta data after set_meta_force {0}'.format(all_post_meta))
self.assertTrue(max_cas_post == pre_cas, '[ERROR]Max cas {0} is not equal to original cas {1}'.format(max_cas_post, pre_cas))
self.assertTrue(pre_seq < post_seq, '[ERROR]Pre rev id {0} is not greater than post rev id {1}'.format(pre_seq, post_seq))
# remove that node
self.log.info('Remove the node with active data')
rebalance = self.cluster.async_rebalance(self.servers[-1:], [] ,[self.master])
rebalance.result()
self.sleep(120)
replica_CAS = mc_replica.getMeta(key)[4]
get_meta_resp = mc_replica.getMeta(key,request_extended_meta_data=False)
#print 'replica CAS {0}'.format(replica_CAS)
#print 'replica ext meta {0}'.format(get_meta_resp)
# add the node back
self.log.info('Add the node back, the max_cas should be healed')
rebalance = self.cluster.async_rebalance(self.servers[-1:], [self.master], [])
rebalance.result()
# verify the CAS is good
mc_active = client.memcached(key)
active_CAS = mc_active.getMeta(key)[4]
print 'active cas {0}'.format(active_CAS)
self.assertTrue(replica_CAS == active_CAS, 'cas mismatch active: {0} replica {1}'.format(active_CAS,replica_CAS))
#self.assertTrue( get_meta_resp[5] == 1, msg='Metadata indicate conflict resolution is not set')
''' Testing revid based conflict resolution with timeSync enabled, where cas on either mutations match and it does rev id CR
and retains it after a failover server'''
def test_failover_revid_conflict_resolution(self):
self.log.info(' Starting test_rebalance_revid_conflict_resolution ..')
self._load_ops(ops='set', mutations=20)
key = 'key1'
value = 'value'
self.client.memcached(key).set(key, 0, 0,json.dumps({'value':value}))
vbucket_id = client._get_vBucket_id(key)
#print 'vbucket_id is {0}'.format(vbucket_id)
mc_active = client.memcached(key)
mc_master = client.memcached_for_vbucket( vbucket_id )
mc_replica = client.memcached_for_replica_vbucket(vbucket_id)
new_seq=121
pre_cas = mc_active.getMeta(key)[4]
pre_seq = mc_active.getMeta(key)[3]
pre_max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
all = mc_active.getMeta(key)
get_meta_1 = mc_active.getMeta(key,request_extended_meta_data=False)
#print 'cr {0}'.format(get_meta_1)
self.log.info('all meta data before set_meta_force {0}'.format(all))
self.log.info('max_cas before set_meta_force {0}'.format(pre_max_cas))
self.log.info('Forcing conflict_resolution to rev-id by matching inserting cas ')
#set_with_meta_resp = mc_active.set_with_meta(key, 0, 0, new_seq, pre_cas, '123456789',vbucket_id)
set_with_meta_resp = mc_active.setWithMeta(key, '123456789', 0, 0, new_seq, pre_cas)
cas_post = mc_active.getMeta(key)[4]
all_post_meta = mc_active.getMeta(key)
post_seq = mc_active.getMeta(key)[3]
get_meta_2 = mc_active.getMeta(key,request_extended_meta_data=False)
#print 'cr {0}'.format(get_meta_2)
max_cas_post = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.log.info('Expect RevId conflict_resolution to occur, and the last updated mutation to be the winner..')
self.log.info('all meta data after set_meta_force {0}'.format(all_post_meta))
self.assertTrue(max_cas_post == pre_cas, '[ERROR]Max cas {0} is not equal to original cas {1}'.format(max_cas_post, pre_cas))
self.assertTrue(pre_seq < post_seq, '[ERROR]Pre rev id {0} is not greater than post rev id {1}'.format(pre_seq, post_seq))
# failover that node
self.log.info('Failing over node with active data {0}'.format(self.master))
self.cluster.failover(self.servers, [self.master])
self.log.info('Remove the node with active data {0}'.format(self.master))
rebalance = self.cluster.async_rebalance(self.servers[:], [] ,[self.master])
rebalance.result()
self.sleep(120)
replica_CAS = mc_replica.getMeta(key)[4]
get_meta_resp = mc_replica.getMeta(key,request_extended_meta_data=False)
#print 'replica CAS {0}'.format(replica_CAS)
#print 'replica ext meta {0}'.format(get_meta_resp)
# add the node back
self.log.info('Add the node back, the max_cas should be healed')
rebalance = self.cluster.async_rebalance(self.servers[-1:], [self.master], [])
rebalance.result()
# verify the CAS is good
mc_active = client.memcached(key)
active_CAS = mc_active.getMeta(key)[4]
#print 'active cas {0}'.format(active_CAS)
self.assertTrue(replica_CAS == active_CAS, 'cas mismatch active: {0} replica {1}'.format(active_CAS,replica_CAS))
#self.assertTrue( get_meta_resp[5] == 1, msg='Metadata indicate conflict resolution is not set')
''' Test getMeta on cas and max cas values for empty vbucket
'''
def test_cas_getMeta_empty_vBucket(self):
self.log.info(' Starting test-getMeta')
self._load_ops(ops='set', mutations=20)
k=0
all_keys = []
while k<10:
k+=1
key = "{0}{1}".format(self.prefix, k)
all_keys.append(key)
vbucket_ids = self.client._get_vBucket_ids(all_keys)
print 'bucket_ids'
for v in vbucket_ids:
print v
print 'done'
i=1111
if i not in vbucket_ids and i <= 1023:
vb_non_existing=i
elif i>1023:
i +=1
else:
self.log.info('ERROR generating empty vbucket id')
vb_non_existing=vbucket_ids.pop()
print 'nominated vb_nonexisting is {0}'.format(vb_non_existing)
mc_active = self.client.memcached(all_keys[0]) #Taking a temp connection to the mc.
#max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(vb_non_existing) + ':max_cas'] )
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(all_keys[0])) + ':max_cas'] )
self.assertTrue( max_cas != 0, msg='[ERROR] Max cas is non-zero')
def test_meta_backup(self):
"""
Test addMeta on cas and max cas values for keys
"""
self.log.info('Starting test-getMeta')
self._load_ops(ops='set', mutations=20)
# Do the backup on the bucket
self.shell = RemoteMachineShellConnection(self.cluster.master)
self.buckets = self.bucket_util.buckets
self.couchbase_login_info = "%s:%s" % (self.input.membase_settings.rest_username,
self.input.membase_settings.rest_password)
self.backup_location = "/tmp/backup"
self.command_options = self.input.param("command_options", '')
try:
shell = RemoteMachineShellConnection(self.cluster.master)
self.shell.execute_cluster_backup(
self.couchbase_login_info,
self.backup_location,
self.command_options)
self.sleep(5, "Wait before restore backup")
shell.restore_backupFile(self.couchbase_login_info,
self.backup_location,
[bucket.name for bucket in self.buckets])
self.log.info("Done with restore")
finally:
self._check_cas(check_conflict_resolution=False)
def _check_cas(self, load_gen, check_conflict_resolution=False,
time_sync=None, docs_deleted=False):
"""
Common function to verify the expected values on cas
"""
load_gen = copy.deepcopy(load_gen)
self.log.info('Verifying CAS and max-cas for the keys')
while load_gen.has_next():
key, value = load_gen.next()
cas = self.client.crud("read", key)["cas"]
if docs_deleted:
if cas != 0:
self.log_failure("Max CAS mismatch. %s != 0" % cas)
else:
max_cas = None
vb_for_key = self.bucket_util.get_vbucket_num_for_key(key)
for _, data in self.node_data.items():
if vb_for_key in data["active"]:
vb_stat = data["cb_stat"].vbucket_details(
self.bucket.name)
max_cas = long(vb_stat[str(vb_for_key)]["max_cas"])
break
if cas != max_cas:
self.log_failure("Max CAS mismatch. %s != %s"
% (cas, max_cas))
if check_conflict_resolution:
get_meta_resp = mc_active.getMeta(
key,
request_extended_meta_data=False)
if time_sync == 'enabledWithoutDrift':
if get_meta_resp[5] != 1:
self.log_failure("Metadata indicates "
"conflict resolution is not set")
elif time_sync == 'disabled':
if get_meta_resp[5] != 0:
self.log_failure("Metadata indicates "
"conflict resolution is set")
def _load_ops(self, load_gen, op_type, mutations=1):
"""
Common function to add set delete etc operations on the bucket
"""
mutations = 1
self.log.info("Performing %s for %s times" % (op_type, mutations))
exp = 0
if op_type == "expiry":
op_type = "touch"
exp = self.expire_time
for _ in range(mutations):
tasks_info = self.bucket_util.sync_load_all_buckets(
self.cluster, load_gen, op_type, exp=exp,
persist_to=self.persist_to, replicate_to=self.replicate_to,
durability=self.durability_level,
timeout_secs=self.sdk_timeout,
batch_size=self.num_items,
sdk_client_pool=self.sdk_client_pool)
for task, _ in tasks_info.items():
if task.fail:
self.log_failure("Failures observed during %s" % op_type)
def _check_expiry(self, load_gen):
"""
Check if num_items are expired as expected
"""
self.sleep(self.expire_time, "Wait for docs to expire")
while load_gen.has_next():
key, value = load_gen.next()
vb_for_key = self.bucket_util.get_vbucket_num_for_key(key)
cas = None
for _, data in self.node_data.items():
if vb_for_key in data["active"]:
vb_stat = data["cb_stat"].vbucket_details(self.bucket.name)
cas = long(vb_stat[str(vb_for_key)]["max_cas"])
break
replace_result = self.client.crud(
"replace", key, value,
replicate_to=self.replicate_to,
persist_to=self.persist_to,
durability=self.durability_level,
timeout=self.sdk_timeout,
cas=cas)
if replace_result["status"] is True:
self.log_failure("Replace on %s succeeded using old CAS %s"
% (key, cas))
if SDKException.DocumentNotFoundException \
not in str(replace_result["error"]):
self.log_failure("Invalid exception for %s: %s"
% (key, replace_result))
def test_bucket_durability_upgrade(self):
update_task = None
self.sdk_timeout = 60
create_batch_size = 10000
if self.atomicity:
create_batch_size = 10
# To make sure sync_write can we supported by initial cluster version
sync_write_support = True
if float(self.initial_version[0:3]) < 6.5:
sync_write_support = False
if sync_write_support:
self.verification_dict["rollback_item_count"] = 0
self.verification_dict["sync_write_aborted_count"] = 0
if self.upgrade_with_data_load:
self.log.info("Starting async doc updates")
update_task = self.task.async_continuous_doc_ops(
self.cluster, self.bucket, self.gen_load,
op_type=DocLoading.Bucket.DocOps.UPDATE,
process_concurrency=1,
persist_to=1,
replicate_to=1,
timeout_secs=30)
self.log.info("Upgrading cluster nodes to target version")
node_to_upgrade = self.fetch_node_to_upgrade()
while node_to_upgrade is not None:
self.log.info("Selected node for upgrade: %s"
% node_to_upgrade.ip)
self.upgrade_function[self.upgrade_type](node_to_upgrade,
self.upgrade_version)
try:
self.cluster.update_master_using_diag_eval(
self.cluster.servers[0])
except Exception:
self.cluster.update_master_using_diag_eval(
self.cluster.servers[self.nodes_init-1])
create_gen = doc_generator(self.key, self.num_items,
self.num_items+create_batch_size)
# Validate sync_write results after upgrade
if self.atomicity:
sync_write_task = self.task.async_load_gen_docs_atomicity(
self.cluster, self.bucket_util.buckets,
create_gen, DocLoading.Bucket.DocOps.CREATE,
process_concurrency=1,
transaction_timeout=self.transaction_timeout,
record_fail=True)
else:
sync_write_task = self.task.async_load_gen_docs(
self.cluster, self.bucket, create_gen,
DocLoading.Bucket.DocOps.CREATE,
timeout_secs=self.sdk_timeout,
process_concurrency=4,
sdk_client_pool=self.sdk_client_pool,
skip_read_on_error=True,
suppress_error_table=True)
self.task_manager.get_task_result(sync_write_task)
self.num_items += create_batch_size
retry_index = 0
while retry_index < 5:
self.sleep(3, "Wait for num_items to match")
current_items = self.bucket_util.get_bucket_current_item_count(
self.cluster, self.bucket)
if current_items == self.num_items:
break
self.log.debug("Num_items mismatch. Expected: %s, Actual: %s"
% (self.num_items, current_items))
# Doc count validation
self.cluster_util.print_cluster_stats()
self.verification_dict["ops_create"] += create_batch_size
self.summary.add_step("Upgrade %s" % node_to_upgrade.ip)
# Halt further upgrade if test has failed during current upgrade
if self.test_failure:
break
node_to_upgrade = self.fetch_node_to_upgrade()
if self.upgrade_with_data_load:
# Wait for update_task to complete
update_task.end_task()
self.task_manager.get_task_result(update_task)
else:
self.verification_dict["ops_update"] = 0
# Cb_stats vb-details validation
failed = self.durability_helper.verify_vbucket_details_stats(
self.bucket_util.buckets[0],
self.cluster_util.get_kv_nodes(),
vbuckets=self.cluster_util.vbuckets,
expected_val=self.verification_dict)
if failed:
self.log_failure("Cbstat vbucket-details validation failed")
self.summary.add_step("Cbstats vb-details verification")
self.validate_test_failure()
possible_d_levels = dict()
possible_d_levels[Bucket.Type.MEMBASE] = \
self.bucket_util.get_supported_durability_levels()
possible_d_levels[Bucket.Type.EPHEMERAL] = [
Bucket.DurabilityLevel.NONE,
Bucket.DurabilityLevel.MAJORITY]
len_possible_d_levels = len(possible_d_levels[self.bucket_type]) - 1
if not sync_write_support:
self.verification_dict["rollback_item_count"] = 0
self.verification_dict["sync_write_aborted_count"] = 0
# Perform bucket_durability update
key, value = doc_generator("b_durability_doc", 0, 1).next()
client = SDKClient([self.cluster.master], self.bucket_util.buckets[0])
for index, d_level in enumerate(possible_d_levels[self.bucket_type]):
self.log.info("Updating bucket_durability=%s" % d_level)
self.bucket_util.update_bucket_property(
self.bucket_util.buckets[0],
bucket_durability=BucketDurability[d_level])
self.bucket_util.print_bucket_stats()
buckets = self.bucket_util.get_all_buckets()
if buckets[0].durability_level != BucketDurability[d_level]:
self.log_failure("New bucket_durability not taken")
self.summary.add_step("Update bucket_durability=%s" % d_level)
self.sleep(10, "MB-39678: Bucket_d_level change to take effect")
if index == 0:
op_type = DocLoading.Bucket.DocOps.CREATE
self.verification_dict["ops_create"] += 1
elif index == len_possible_d_levels:
op_type = DocLoading.Bucket.DocOps.DELETE
self.verification_dict["ops_delete"] += 1
else:
op_type = DocLoading.Bucket.DocOps.UPDATE
if "ops_update" in self.verification_dict:
self.verification_dict["ops_update"] += 1
result = client.crud(op_type, key, value,
timeout=self.sdk_timeout)
if result["status"] is False:
self.log_failure("Doc_op %s failed on key %s: %s"
% (op_type, key, result["error"]))
self.summary.add_step("Doc_op %s" % op_type)
client.close()
# Cb_stats vb-details validation
failed = self.durability_helper.verify_vbucket_details_stats(
self.bucket_util.buckets[0],
self.cluster_util.get_kv_nodes(),
vbuckets=self.cluster_util.vbuckets,
expected_val=self.verification_dict)
if failed:
self.log_failure("Cbstat vbucket-details validation failed")
self.summary.add_step("Cbstats vb-details verification")
self.validate_test_failure()
def test_delete_default_collection(self):
"""
Test to delete '_default' collection under '_default' scope.
Params:
client_type: Supports collection deletion using REST/SDK client.
data_load: Load data into default collection based load_during_phase.
supports 'disabled / before_drop / during_drop'
"""
task = None
client_type = self.input.param("client_type", "sdk").lower()
data_load = self.input.param("load_data", "disabled")
load_gen = doc_generator('test_drop_default',
0,
self.num_items,
mutate=0,
target_vbucket=self.target_vbucket)
if data_load in ["before_drop", "during_drop"]:
self.log.info("Loading %s docs into '%s::%s' collection" %
(self.num_items, CbServer.default_scope,
CbServer.default_collection))
task = self.task.async_load_gen_docs(
self.cluster,
self.bucket,
load_gen,
"create",
self.maxttl,
batch_size=10,
process_concurrency=8,
replicate_to=self.replicate_to,
persist_to=self.persist_to,
durability=self.durability_level,
compression=self.sdk_compression,
timeout_secs=self.sdk_timeout,
scope=CbServer.default_scope,
collection=CbServer.default_collection,
suppress_error_table=True)
self.bucket.scopes[CbServer.default_scope] \
.collections[CbServer.default_collection] \
.num_items += self.num_items
# To make sure data_loading done before collection drop
if data_load == "before_drop":
self.task_manager.get_task_result(task)
if task.fail:
self.log_failure("Doc loading failed for keys: %s" %
task.fail.keys())
# Data validation
self.bucket_util._wait_for_stats_all_buckets()
# Prints bucket stats after doc_ops
self.bucket_util.print_bucket_stats()
self.bucket_util.validate_doc_count_as_per_collections(self.bucket)
# Drop collection phase
self.log.info("Deleting collection '%s::%s'" %
(CbServer.default_scope, CbServer.default_collection))
if client_type == "sdk":
client = SDKClient([self.cluster.master],
self.bucket,
compression_settings=self.sdk_compression)
client.drop_collection(CbServer.default_scope,
CbServer.default_collection)
client.close()
BucketUtils.mark_collection_as_dropped(self.bucket,
CbServer.default_scope,
CbServer.default_collection)
elif client_type == "rest":
self.bucket_util.drop_collection(self.cluster.master, self.bucket,
CbServer.default_scope,
CbServer.default_collection)
else:
self.log_failure("Invalid client_type '%s'" % client_type)
self.sleep(60)
# Wait for doc_loading task to complete
if data_load == "during_drop":
self.task_manager.get_task_result(task)
if task.fail:
self.log.info("Doc loading failed for keys: %s" % task.fail)
self.bucket_util._wait_for_stats_all_buckets()
# Prints bucket stats after doc_ops
self.bucket_util.print_bucket_stats()
# Validate drop collection using cbstats
for node in self.cluster_util.get_kv_nodes():
shell_conn = RemoteMachineShellConnection(node)
cbstats = Cbstats(shell_conn)
c_data = cbstats.get_collections(self.bucket)
expected_collection_count = \
len(self.bucket_util.get_active_collections(
self.bucket,
CbServer.default_scope,
only_names=True))
if c_data["count"] != expected_collection_count:
self.log_failure(
"%s - Expected collection count is '%s'. "
"Actual: %s" %
(node.ip, expected_collection_count, c_data["count"]))
if "_default" in c_data:
self.log_failure("%s: _default collection exists in cbstats" %
node.ip)
# SDK connection to default(dropped) collection to validate failure
try:
client = SDKClient([self.cluster.master],
self.bucket,
scope=CbServer.default_scope,
collection=CbServer.default_collection,
compression_settings=self.sdk_compression)
result = client.crud("create", "test_key-1", "TestValue")
if result["status"] is True:
self.log_failure("CRUD succeeded on deleted collection")
elif SDKException.RetryReason.KV_COLLECTION_OUTDATED \
not in result["error"]:
self.log_failure("Invalid error '%s'" % result["error"])
client.close()
except Exception as e:
self.log.info(e)
# Validate the bucket doc count is '0' after drop collection
self.bucket_util._wait_for_stats_all_buckets()
# Prints bucket stats after doc_ops
self.bucket_util.print_bucket_stats()
self.bucket_util.validate_doc_count_as_per_collections(self.bucket)
self.validate_test_failure()
def create_delete_collections(self):
"""
1. Create Scope-Collection
2. Validate '_default' collection values are intact
3. Load documents into created collection
4. Validate documents are loaded in new collection
5. Delete the collection and validate the '_default' collection
is unaffected
"""
use_scope_name_for_collection = \
self.input.param("use_scope_name_for_collection", False)
scope_name = BucketUtils.get_random_name()
collection_name = scope_name
if not use_scope_name_for_collection:
collection_name = BucketUtils.get_random_name()
gen_add = doc_generator(self.key, 0, self.num_items)
gen_set = doc_generator(self.key,
0,
self.num_items,
mutate=1,
mutation_type='SET')
self.log.info("Creating scope::collection '%s::%s'" %
(scope_name, collection_name))
self.bucket_util.create_scope(self.cluster.master, self.bucket,
scope_name)
self.bucket_util.create_collection(self.cluster.master, self.bucket,
scope_name, collection_name)
self.bucket_util.create_collection(self.cluster.master, self.bucket,
scope_name, "my_collection_2")
shell_conn = RemoteMachineShellConnection(self.cluster.master)
cbstats = Cbstats(shell_conn)
cbstats.get_collections(self.bucket_util.buckets[0])
self.log.info("Validating the documents in default collection")
self.bucket_util._wait_for_stats_all_buckets()
# Prints bucket stats after doc_ops
self.bucket_util.print_bucket_stats()
self.bucket_util.verify_stats_all_buckets(self.num_items)
self.log.info("Load documents into the created collection")
sdk_client = SDKClient([self.cluster.master],
self.bucket,
scope=scope_name,
collection=collection_name,
compression_settings=self.sdk_compression)
while gen_add.has_next():
key, value = gen_add.next()
result = sdk_client.crud("create",
key,
value,
replicate_to=self.replicate_to,
persist_to=self.persist_to,
durability=self.durability_level,
timeout=self.sdk_timeout)
if result["status"] is False:
self.log_failure("Doc create failed for collection: %s" %
result)
break
sdk_client.close()
self.validate_test_failure()
self.bucket.scopes[scope_name] \
.collections[collection_name] \
.num_items += self.num_items
self.bucket_util._wait_for_stats_all_buckets()
# Prints bucket stats after doc_ops
self.bucket_util.print_bucket_stats()
self.bucket_util.verify_stats_all_buckets(self.num_items * 2)
task = self.task.async_load_gen_docs(self.cluster,
self.bucket,
gen_set,
"update",
0,
batch_size=10,
process_concurrency=8,
replicate_to=self.replicate_to,
persist_to=self.persist_to,
durability=self.durability_level,
compression=self.sdk_compression,
timeout_secs=self.sdk_timeout,
scope=scope_name,
collection=collection_name)
self.task_manager.get_task_result(task)
self.bucket_util._wait_for_stats_all_buckets()
# Prints bucket stats after doc_ops
self.bucket_util.print_bucket_stats()
self.bucket_util.verify_stats_all_buckets(self.num_items * 2)
self.validate_test_failure()
def test_sub_doc_op_with_bucket_level_durability(self):
"""
Create Buckets with durability_levels set and perform
Sub_doc CRUDs from client without durability settings and
validate the ops to make sure respective durability is honored
"""
key, value = doc_generator("test_key", 0, 1).next()
sub_doc_key = "sub_doc_key"
sub_doc_vals = ["val_1", "val_2", "val_3", "val_4", "val_5"]
for d_level in self.get_supported_durability_for_bucket():
# Avoid creating bucket with durability=None
if d_level == Bucket.DurabilityLevel.NONE:
continue
step_desc = "Creating %s bucket with level '%s'" \
% (self.bucket_type, d_level)
verification_dict = self.get_cb_stat_verification_dict()
self.log.info(step_desc)
# Object to support performing CRUDs and create Bucket
bucket_dict = self.get_bucket_dict(self.bucket_type, d_level)
bucket_obj = Bucket(bucket_dict)
self.bucket_util.create_bucket(self.cluster, bucket_obj,
wait_for_warmup=True)
self.summary.add_step(step_desc)
# SDK client to perform sub_doc ops
client = SDKClient([self.cluster.master], bucket_obj)
result = client.crud("create", key, value)
verification_dict["ops_create"] += 1
verification_dict["sync_write_committed_count"] += 1
if result["status"] is False:
self.log_failure("Doc insert failed for key: %s" % key)
# Perform sub_doc CRUD
for sub_doc_op in ["subdoc_insert", "subdoc_upsert",
"subdoc_replace"]:
sub_doc_val = choice(sub_doc_vals)
_, fail = client.crud(sub_doc_op, key,
[sub_doc_key, sub_doc_val])
if fail:
self.log_failure("%s failure. Key %s, sub_doc (%s, %s): %s"
% (sub_doc_op, key,
sub_doc_key, sub_doc_val, result))
else:
verification_dict["ops_update"] += 1
verification_dict["sync_write_committed_count"] += 1
success, fail = client.crud("subdoc_read", key, sub_doc_key)
if fail or str(success[key]["value"].get(0)) != sub_doc_val:
self.log_failure("%s failed. Expected: %s, Actual: %s"
% (sub_doc_op, sub_doc_val,
success[key]["value"].get(0)))
self.summary.add_step("%s for key %s" % (sub_doc_op, key))
# Subdoc_delete and verify
sub_doc_op = "subdoc_delete"
_, fail = client.crud(sub_doc_op, key, sub_doc_key)
if fail:
self.log_failure("%s failure. Key %s, sub_doc (%s, %s): %s"
% (sub_doc_op, key,
sub_doc_key, sub_doc_val, result))
verification_dict["ops_update"] += 1
verification_dict["sync_write_committed_count"] += 1
_, fail = client.crud(sub_doc_op, key, sub_doc_key)
if SDKException.PathNotFoundException \
not in str(fail[key]["error"]):
self.log_failure("Invalid error after sub_doc_delete")
self.summary.add_step("%s for key %s" % (sub_doc_op, key))
# Validate doc_count
self.bucket_util._wait_for_stats_all_buckets(self.cluster,
self.cluster.buckets)
self.bucket_util.verify_stats_all_buckets(self.cluster, 1)
# Cbstats vbucket-details validation
self.cb_stat_verify(verification_dict)
# Close SDK client
client.close()
# Delete the bucket on server
self.bucket_util.delete_bucket(self.cluster, bucket_obj)
self.summary.add_step("Delete %s bucket" % self.bucket_type)
def test_scenario(bucket, doc_ops,
with_sync_write_val=None):
# Set crud_batch_size
crud_batch_size = 4
simulate_error = CouchbaseError.STOP_MEMCACHED
# Fetch target_vbs for CRUDs
node_vb_info = self.vbs_in_node
target_vbuckets = node_vb_info[target_nodes[0]]["replica"]
if len(target_nodes) > 1:
index = 1
while index < len(target_nodes):
target_vbuckets = list(
set(target_vbuckets).intersection(
set(node_vb_info[target_nodes[index]]["replica"]))
)
index += 1
# Variable to hold one of the doc_generator objects
gen_loader_1 = None
gen_loader_2 = None
# Initialize doc_generators to use for testing
self.log.info("Creating doc_generators")
gen_create = doc_generator(
self.key, self.num_items, crud_batch_size,
vbuckets=self.cluster.vbuckets,
target_vbucket=target_vbuckets)
gen_update = doc_generator(
self.key, 0, crud_batch_size,
vbuckets=self.cluster.vbuckets,
target_vbucket=target_vbuckets, mutate=1)
gen_delete = doc_generator(
self.key, 0, crud_batch_size,
vbuckets=self.cluster.vbuckets,
target_vbucket=target_vbuckets)
self.log.info("Done creating doc_generators")
# Start CRUD operation based on the given 'doc_op' type
if doc_ops[0] == "create":
self.num_items += crud_batch_size
gen_loader_1 = gen_create
elif doc_ops[0] in ["update", "replace", "touch"]:
gen_loader_1 = gen_update
elif doc_ops[0] == "delete":
gen_loader_1 = gen_delete
self.num_items -= crud_batch_size
if doc_ops[1] == "create":
gen_loader_2 = gen_create
elif doc_ops[1] in ["update", "replace", "touch"]:
gen_loader_2 = gen_update
elif doc_ops[1] == "delete":
gen_loader_2 = gen_delete
# Load required docs for doc_op_1 in case of type != create
if doc_op[2] == "load_initial_docs":
doc_loading_task = self.task.async_load_gen_docs(
self.cluster, bucket, gen_loader_1, "create", 0,
batch_size=crud_batch_size, process_concurrency=1,
timeout_secs=10,
print_ops_rate=False,
sdk_client_pool=self.sdk_client_pool)
self.task_manager.get_task_result(doc_loading_task)
if doc_loading_task.fail:
self.log_failure("Failure while loading initial docs")
self.summary.add_step("Create docs for %s" % doc_op[0])
verification_dict["ops_create"] += crud_batch_size
verification_dict["sync_write_committed_count"] \
+= crud_batch_size
# Initialize tasks and store the task objects
doc_loader_task = self.task.async_load_gen_docs(
self.cluster, bucket, gen_loader_1, doc_ops[0], 0,
batch_size=crud_batch_size, process_concurrency=8,
timeout_secs=60,
print_ops_rate=False,
start_task=False,
sdk_client_pool=self.sdk_client_pool)
# SDK client for performing individual ops
client = SDKClient([self.cluster.master], bucket)
# Perform specified action
for node in target_nodes:
error_sim = CouchbaseError(self.log,
self.vbs_in_node[node]["shell"])
error_sim.create(simulate_error,
bucket_name=bucket.name)
self.sleep(5, "Wait for error simulation to take effect")
self.task_manager.add_new_task(doc_loader_task)
self.sleep(5, "Wait for task_1 CRUDs to reach server")
# Perform specified CRUD operation on sync_write docs
tem_gen = deepcopy(gen_loader_2)
while tem_gen.has_next():
key, value = tem_gen.next()
for retry_strategy in [
SDKConstants.RetryStrategy.FAIL_FAST,
SDKConstants.RetryStrategy.BEST_EFFORT]:
if with_sync_write_val:
fail = client.crud(doc_ops[1], key, value=value,
exp=0,
durability=with_sync_write_val,
timeout=3, time_unit="seconds",
sdk_retry_strategy=retry_strategy)
else:
fail = client.crud(doc_ops[1], key, value=value,
exp=0,
timeout=3, time_unit="seconds",
sdk_retry_strategy=retry_strategy)
expected_exception = SDKException.AmbiguousTimeoutException
retry_reason = \
SDKException.RetryReason.KV_SYNC_WRITE_IN_PROGRESS
if retry_strategy == SDKConstants.RetryStrategy.FAIL_FAST:
expected_exception = \
SDKException.RequestCanceledException
retry_reason = \
SDKException.RetryReason \
.KV_SYNC_WRITE_IN_PROGRESS_NO_MORE_RETRIES
# Validate the returned error from the SDK
if expected_exception not in str(fail["error"]):
self.log_failure("Invalid exception for {0}: {1}"
.format(key, fail["error"]))
if retry_reason not in str(fail["error"]):
self.log_failure("Invalid retry reason for {0}: {1}"
.format(key, fail["error"]))
# Try reading the value in SyncWrite in-progress state
fail = client.crud("read", key)
if doc_ops[0] == "create":
# Expected KeyNotFound in case of CREATE operation
if fail["status"] is True:
self.log_failure(
"%s returned value during SyncWrite state: %s"
% (key, fail))
else:
# Expects prev value in case of other operations
if fail["status"] is False:
self.log_failure(
"Key %s read failed for previous value: %s"
% (key, fail))
# Revert the introduced error condition
for node in target_nodes:
error_sim = CouchbaseError(self.log,
self.vbs_in_node[node]["shell"])
error_sim.revert(simulate_error,
bucket_name=bucket.name)
# Wait for doc_loader_task to complete
self.task.jython_task_manager.get_task_result(doc_loader_task)
verification_dict["ops_%s" % doc_op[0]] += crud_batch_size
verification_dict["sync_write_committed_count"] \
+= crud_batch_size
# Disconnect the client
client.close()
def test_expired_sys_xattr_consumed_by_dcp(self):
"""
1. Create a empty doc with exp=5sec
2. Insert xattr to the same doc (preserve_expiry=True)
3. Wait for ep_queue_size to become zero
4. Insert few more docs to the bucket
5. Create a new secondary index to start DCP streaming from memory
6. Expect no crash after step#5
"""
doc_ttl = 10
key = "test_xattr_doc-1"
index_retry = 10
index_created = False
index_name = "test_index"
bucket = self.cluster.buckets[0]
vb_for_key = self.bucket_util.get_vbucket_num_for_key(key)
# Open SDK client
client = SDKClient([self.cluster.master], bucket)
# Create Sync_write doc with xattr + doc_ttl=10s
client.crud(DocLoading.Bucket.DocOps.CREATE, key, {},
durability=self.durability_level)
client.crud("subdoc_insert", key, ["_sdkey", "abc123"],
xattr=True, durability=self.durability_level)
client.crud(DocLoading.Bucket.DocOps.UPDATE, key, {},
durability=self.durability_level, exp=doc_ttl)
# Wait for all items to get persist
self.log.info("Waiting for ep_queue_size to become zero")
self.bucket_util._wait_for_stats_all_buckets(self.cluster,
self.cluster.buckets)
self.sleep(doc_ttl, "Wait for doc to expire")
self.log.info("Loading more docs into the targeted vb: %s"
% vb_for_key)
doc_gen = doc_generator(self.key, self.num_items, 1000,
target_vbucket=[vb_for_key])
doc_load_task = self.task.async_load_gen_docs(
self.cluster, bucket, doc_gen, DocLoading.Bucket.DocOps.UPDATE,
durability=self.durability_level)
self.task_manager.get_task_result(doc_load_task)
rest = RestConnection(self.cluster.master)
rest.set_indexer_storage_mode()
self.log.info("Creating 2i on the bucket")
client.cluster.query("CREATE PRIMARY INDEX %s ON %s"
% (index_name, bucket.name))
self.sleep(2, "Wait for primary index to be created")
while not index_created and index_retry != 0:
state = client.cluster \
.query("SELECT state FROM system:indexes "
"WHERE name='%s'" % index_name) \
.rowsAsObject()[0].get("state")
if state == "online":
index_created = True
else:
index_retry -= 1
self.sleep(1, "Retrying.. Index not yet online")
client.close()
def test_MB_41944(self):
num_index = self.input.param("num_index", 1)
# Create doc_gen for loading
doc_gen = doc_generator(self.key, 0, 1)
# Get key for delete op and reset the gen
key, v = doc_gen.next()
doc_gen.reset()
# Open SDK client connection
client = SDKClient([self.cluster.master], self.bucket_util.buckets[0])
query = list()
query.append("CREATE PRIMARY INDEX index_0 on %s USING GSI" %
self.bucket_util.buckets[0].name)
if num_index == 2:
query.append("CREATE INDEX index_1 on %s(name,age) "
"WHERE mutated=0 USING GSI" %
self.bucket_util.buckets[0].name)
# Create primary index on the bucket
for q in query:
client.cluster.query(q)
# Wait for index to become online`
for index, _ in enumerate(query):
query = "SELECT state FROM system:indexes WHERE name='index_%s'" \
% index
index = 0
state = None
while index < 30:
state = client.cluster.query(query) \
.rowsAsObject()[0].get("state")
if state == "online":
break
self.sleep(1)
if state != "online":
self.log_failure("Index 'index_%s' not yet online" % index)
# Start transaction to create the doc
trans_task = self.task.async_load_gen_docs_atomicity(
self.cluster, self.bucket_util.buckets, doc_gen,
DocLoading.Bucket.DocOps.CREATE)
self.task_manager.get_task_result(trans_task)
# Perform sub_doc operation on same key
_, fail = client.crud(DocLoading.Bucket.SubDocOps.INSERT,
key=key,
value=["_sysxattr", "sysxattr-payload"],
xattr=True)
if fail:
self.log_failure("Subdoc insert failed: %s" % fail)
else:
self.log.info("Subdoc insert success")
# Delete the created doc
result = client.crud(DocLoading.Bucket.DocOps.DELETE, key)
if result["status"] is False:
self.log_failure("Doc delete failed: %s" % result["error"])
else:
self.log.info("Document deleted")
# Re-insert same doc through transaction
trans_task = self.task.async_load_gen_docs_atomicity(
self.cluster, self.bucket_util.buckets, doc_gen,
DocLoading.Bucket.DocOps.CREATE)
self.task_manager.get_task_result(trans_task)
# Close SDK Client connection
client.close()
self.validate_test_failure()
def test_staged_doc_read(self):
self.verify = self.input.param("verify", True)
bucket = self.cluster.buckets[0]
expected_exception = SDKException.DocumentNotFoundException
# Create SDK client for transactions
client = SDKClient([self.cluster.master], bucket)
if self.doc_op in ["update", "delete"]:
for doc in self.docs:
result = client.crud("create",
doc.getT1(),
doc.getT2(),
durability=self.durability_level,
timeout=60)
if result["status"] is False:
self.log_failure("Key %s create failed: %s" %
(doc.getT1(), result))
break
expected_exception = None
read_thread = Thread(
target=self.__perform_read_on_doc_keys,
args=(bucket, self.keys),
kwargs=dict(expected_exception=expected_exception))
read_thread.start()
# Transaction load
exception = self.__run_mock_test(client, self.doc_op)
if SDKException.TransactionExpired not in str(exception):
self.log_failure("Expected exception not found")
self.log.info("Terminating reader thread")
self.stop_thread = True
read_thread.join()
self.transaction_fail_count = 2
exception = self.__run_mock_test(client, self.doc_op)
if exception:
self.log_failure(exception)
# verify the values
for key in self.keys:
result = client.read(key)
if "Remove" in self.operation \
or self.transaction_commit is False \
or self.verify is False:
if result['status']:
actual_val = client.translate_to_json_object(
result['value'])
self.log.info("Actual value for key %s is %s" %
(key, actual_val))
self.log_failure(
"Key '%s' should be deleted but present in the bucket"
% key)
else:
actual_val = client.translate_to_json_object(result['value'])
if self.doc_op == "update":
self.content.put("mutated", 1)
elif self.doc_op == "delete":
self.content.removeKey("value")
if self.content != actual_val:
self.log.info("Key %s Actual: %s, Expected: %s" %
(key, actual_val, self.content))
self.log_failure("Mismatch in doc content")
# Close SDK client
client.close()
if self.read_failed[self.cluster.buckets[0]] is True:
self.log_failure("Failure in read thread for bucket: %s" %
self.cluster.buckets[0].name)
self.validate_test_failure()
def test_staged_doc_query_from_index(self):
self.verify = self.input.param("verify", True)
expected_val = dict()
bucket = self.cluster.buckets[0]
# Create SDK client for transactions
client = SDKClient([self.cluster.master], bucket)
if self.doc_op in ["update", "delete"]:
for doc in self.docs:
result = client.crud("create",
doc.getT1(),
doc.getT2(),
durability=self.durability_level,
timeout=60)
if result["status"] is False:
self.log_failure("Key %s create failed: %s" %
(doc.getT1(), result))
break
expected_val[doc.getT1()] = json.loads(str(doc.getT2()))
# Create primary Index on all buckets
for t_bucket in self.cluster.buckets:
q_result = client.cluster.query("CREATE PRIMARY INDEX ON `%s`" %
t_bucket.name)
if q_result.metaData().status().toString() != "SUCCESS":
client.close()
self.fail("Create primary index failed for bucket %s" %
t_bucket.name)
self.sleep(10, "Wait for primary indexes to get warmed up")
query_thread = Thread(target=self.__perform_query_on_doc_keys,
args=(bucket, self.keys, expected_val))
query_thread.start()
# Transaction load
exception = self.__run_mock_test(client, self.doc_op)
if SDKException.TransactionExpired not in str(exception):
self.log_failure("Expected exception not found")
self.log.info("Terminating query thread")
self.stop_thread = True
query_thread.join()
self.transaction_fail_count = 2
exception = self.__run_mock_test(client, self.doc_op)
if exception:
self.log_failure(exception)
# verify the values
for key in self.keys:
result = client.read(key)
if "Remove" in self.operation \
or self.transaction_commit is False \
or self.verify is False:
if result['status']:
actual_val = client.translate_to_json_object(
result['value'])
self.log.info("Actual value for key %s is %s" %
(key, actual_val))
self.log_failure(
"Key '%s' should be deleted but present in the bucket"
% key)
else:
actual_val = client.translate_to_json_object(result['value'])
if self.doc_op == "update":
self.content.put("mutated", 1)
elif self.doc_op == "delete":
self.content.removeKey("value")
if self.content != actual_val:
self.log.info("Key %s Actual: %s, Expected: %s" %
(key, actual_val, self.content))
self.log_failure("Mismatch in doc content")
# Close SDK client
client.close()
if self.read_failed[self.cluster.buckets[0]] is True:
self.log_failure("Failure in read thread for bucket: %s" %
self.cluster.buckets[0].name)
self.validate_test_failure()
class basic_ops(ClusterSetup):
def setUp(self):
super(basic_ops, self).setUp()
if self.default_bucket:
# Over-ride bucket ram quota=100MB
self.bucket_size = 100
self.create_bucket(self.cluster)
self.sleep(10, "Wait for bucket to become ready for ops")
self.def_bucket = self.bucket_util.get_all_buckets(self.cluster)
self.client = SDKClient([self.cluster.master], self.def_bucket[0])
self.__durability_level()
self.create_Transaction()
self._stop = threading.Event()
self.log.info("==========Finished Basic_ops base setup========")
def tearDown(self):
self.client.close()
super(basic_ops, self).tearDown()
def __durability_level(self):
if self.durability_level == Bucket.DurabilityLevel.MAJORITY:
self.durability = 1
elif self.durability_level \
== Bucket.DurabilityLevel.MAJORITY_AND_PERSIST_TO_ACTIVE:
self.durability = 2
elif self.durability_level \
== Bucket.DurabilityLevel.PERSIST_TO_MAJORITY:
self.durability = 3
elif self.durability_level == "ONLY_NONE":
self.durability = 4
else:
self.durability = 0
def get_doc_generator(self, start, end):
age = range(5)
name = ['james', 'sharon']
body = [''.rjust(self.doc_size - 10, 'a')]
template = JsonObject.create()
template.put("age", age)
template.put("first_name", name)
template.put("body", body)
generator = DocumentGenerator(self.key,
template,
start=start,
end=end,
key_size=self.key_size,
doc_size=self.doc_size,
doc_type=self.doc_type,
randomize=True,
age=age,
first_name=name)
return generator
def set_exception(self, exception):
self.exception = exception
raise BaseException("Got an exception %s" % self.exception)
def __chunks(self, l, n):
"""Yield successive n-sized chunks from l."""
for i in range(0, len(l), n):
yield l[i:i + n]
def create_Transaction(self, client=None):
if not client:
client = self.client
transaction_config = Transaction().createTransactionConfig(
self.transaction_timeout, self.durability)
try:
self.transaction = Transaction().createTansaction(
client.cluster, transaction_config)
except Exception as e:
self.set_exception(e)
def __thread_to_transaction(self,
transaction,
op_type,
doc,
txn_commit,
update_count=1,
sync=True,
set_exception=True,
client=None):
exception = None
if client is None:
client = self.client
if op_type == "create":
exception = Transaction().RunTransaction(client.cluster,
transaction,
[client.collection], doc,
[], [], txn_commit, sync,
update_count)
elif op_type == "update":
self.log.info("updating all the keys through threads")
exception = Transaction().RunTransaction(client.cluster,
transaction,
[client.collection], [],
doc, [], txn_commit, sync,
update_count)
elif op_type == "delete":
exception = Transaction().RunTransaction(client.cluster,
transaction,
[client.collection], [],
[], doc, txn_commit, sync,
update_count)
if set_exception and exception:
self.set_exception("Failed")
def doc_gen(self,
num_items,
start=0,
value={'value': 'value1'},
op_type="create"):
self.docs = []
self.keys = []
self.content = self.client.translate_to_json_object(value)
for i in range(start, self.num_items):
key = "test_docs-" + str(i)
if op_type == "create":
doc = Tuples.of(key, self.content)
self.keys.append(key)
self.docs.append(doc)
else:
self.docs.append(key)
def verify_doc(self, num_items, client):
for i in range(num_items):
key = "test_docs-" + str(i)
result = client.read(key)
actual_val = self.client.translate_to_json_object(result['value'])
self.assertEquals(self.content, actual_val)
def test_MultiThreadTxnLoad(self):
"""
Load data through txn, update half the items through different threads
and delete half the items through different threads. if update_retry
then update and delete the same key in two different transaction
and make sure update fails
"""
self.num_txn = self.input.param("num_txn", 9)
self.update_retry = self.input.param("update_retry", False)
self.doc_gen(self.num_items)
threads = []
# create the docs
exception = Transaction().RunTransaction(self.client.cluster,
self.transaction,
[self.client.collection],
self.docs, [], [],
self.transaction_commit, True,
self.update_count)
if exception:
self.set_exception("Failed")
if self.update_retry:
threads.append(
threading.Thread(target=self.__thread_to_transaction,
args=(self.transaction, "delete", self.keys,
self.transaction_commit,
self.update_count)))
threads.append(
threading.Thread(target=self.__thread_to_transaction,
args=(self.transaction, "update", self.keys,
10, self.update_count)))
else:
update_docs = self.__chunks(self.keys[:self.num_items / 2],
self.num_txn)
delete_docs = self.__chunks(self.keys[self.num_items / 2:],
self.num_txn)
for keys in update_docs:
threads.append(
threading.Thread(target=self.__thread_to_transaction,
args=(self.transaction, "update", keys,
self.transaction_commit,
self.update_count)))
for keys in delete_docs:
threads.append(
threading.Thread(target=self.__thread_to_transaction,
args=(self.transaction, "delete", keys,
self.transaction_commit,
self.update_count)))
for thread in threads:
thread.start()
for thread in threads:
thread.join()
self.sleep(60, "Wait for transactions to complete")
if self.update_retry:
for key in self.keys:
result = self.client.read(key)
self.assertEquals(result['status'], False)
else:
self.value = {'mutated': 1, 'value': 'value1'}
self.content = self.client.translate_to_json_object(self.value)
self.verify_doc(self.num_items / 2, self.client)
for key in self.keys[self.num_items / 2:]:
result = self.client.read(key)
self.assertEquals(result['status'], False)
def test_basic_retry(self):
"""
Load set of data to the cluster, update through 2 different threads,
make sure transaction maintains the order of update
:return:
"""
self.write_conflict = self.input.param("write_conflict", 2)
self.log.info("going to create and execute the task")
self.gen_create = self.get_doc_generator(0, self.num_items)
task = self.task.async_load_gen_docs_atomicity(
self.cluster,
self.def_bucket,
self.gen_create,
"create",
exp=0,
batch_size=10,
process_concurrency=8,
replicate_to=self.replicate_to,
persist_to=self.persist_to,
timeout_secs=self.sdk_timeout,
retries=self.sdk_retries,
update_count=self.update_count,
transaction_timeout=self.transaction_timeout,
commit=True,
durability=self.durability_level,
sync=self.sync)
self.task.jython_task_manager.get_task_result(task)
self.log.info("Get all the keys in the cluster")
self.doc_gen(self.num_items)
threads = []
for update_count in [2, 4, 6]:
threads.append(
threading.Thread(target=self.__thread_to_transaction,
args=(self.transaction, "update", self.keys,
self.transaction_commit, update_count)))
# Add verification task
if self.transaction_commit:
self.update_count = 6
else:
self.update_count = 0
for thread in threads:
thread.start()
self.sleep(2, "Wait for transaction thread to start")
for thread in threads:
thread.join()
def test_basic_retry_async(self):
self.log.info("going to create and execute the task")
self.gen_create = self.get_doc_generator(0, self.num_items)
task = self.task.async_load_gen_docs_atomicity(
self.cluster,
self.def_bucket,
self.gen_create,
"create",
exp=0,
batch_size=10,
process_concurrency=1,
replicate_to=self.replicate_to,
persist_to=self.persist_to,
timeout_secs=self.sdk_timeout,
retries=self.sdk_retries,
update_count=self.update_count,
transaction_timeout=self.transaction_timeout,
commit=True,
durability=self.durability_level,
sync=True,
num_threads=1)
self.task.jython_task_manager.get_task_result(task)
self.log.info("get all the keys in the cluster")
keys = ["test_docs-0"] * 2
exception = Transaction().RunTransaction(
self.client.cluster, self.transaction, [self.client.collection],
[], keys, [], self.transaction_commit, False, 0)
if exception:
self.set_exception(Exception(exception))
def basic_concurrency(self):
self.crash = self.input.param("crash", False)
self.doc_gen(self.num_items)
# run transaction
thread = threading.Thread(target=self.__thread_to_transaction,
args=(self.transaction, "create", self.docs,
self.transaction_commit,
self.update_count, True, False))
thread.start()
self.sleep(1, "Wait for transaction thread to start")
if self.crash:
self.client.cluster.disconnect()
self.transaction.close()
self.client1 = SDKClient([self.cluster.master], self.def_bucket[0])
self.create_Transaction(self.client1)
self.sleep(self.transaction_timeout + 60,
"Wait for transaction cleanup to complete")
exception = Transaction().RunTransaction(
self.client.cluster, self.transaction,
[self.client1.collection], self.docs, [], [],
self.transaction_commit, self.sync, self.update_count)
if exception:
self.sleep(60, "Wait for transaction cleanup to happen")
self.verify_doc(self.num_items, self.client1)
self.client1.close()
else:
key = "test_docs-0"
# insert will succeed due to doc_isoloation feature
result = self.client.insert(key, "value")
self.assertEqual(result["status"], True)
# Update should pass
result = self.client.upsert(key, "value")
self.assertEqual(result["status"], True)
# delete should pass
result = self.client.delete(key)
self.assertEqual(result["status"], True)
thread.join()
def test_stop_loading(self):
"""
Load through transactions and close the transaction abruptly,
create a new transaction sleep for 60 seconds and
perform create on the same set of docs
"""
self.num_txn = self.input.param("num_txn", 9)
self.doc_gen(self.num_items)
threads = []
docs = list(self.__chunks(self.docs, len(self.docs) / self.num_txn))
for doc in docs:
threads.append(
threading.Thread(target=self.__thread_to_transaction,
args=(self.transaction, "create", doc,
self.transaction_commit,
self.update_count, True, False)))
for thread in threads:
thread.start()
self.client.cluster.disconnect()
self.transaction.close()
self.client1 = SDKClient([self.cluster.master], self.def_bucket[0])
self.create_Transaction(self.client1)
self.sleep(self.transaction_timeout + 60,
"Wait for transaction cleanup to happen")
self.log.info("going to start the load")
for doc in docs:
exception = Transaction().RunTransaction(
self.client1.cluster, self.transaction,
[self.client1.collection], doc, [], [],
self.transaction_commit, self.sync, self.update_count)
if exception:
self.sleep(60, "Wait for transaction cleanup to happen")
self.verify_doc(self.num_items, self.client1)
self.client1.close()
def __insert_sub_doc_and_validate(self, doc_id, op_type, key, value):
_, failed_items = self.client.crud(op_type,
doc_id, [key, value],
durability=self.durability_level,
timeout=self.sdk_timeout,
time_unit="seconds",
create_path=True,
xattr=True)
self.assertFalse(failed_items, "Subdoc Xattr insert failed")
def __read_doc_and_validate(self, doc_id, expected_val, subdoc_key=None):
if subdoc_key:
success, failed_items = self.client.crud("subdoc_read",
doc_id,
subdoc_key,
xattr=True)
self.assertFalse(failed_items, "Xattr read failed")
self.assertEqual(
expected_val, str(success[doc_id]["value"][0]),
"Sub_doc value mismatch: %s != %s" %
(success[doc_id]["value"][0], expected_val))
def test_TxnWithXattr(self):
self.system_xattr = self.input.param("system_xattr", False)
if self.system_xattr:
xattr_key = "my._attr"
else:
xattr_key = "my.attr"
val = "v" * self.doc_size
self.doc_gen(self.num_items)
thread = threading.Thread(target=self.__thread_to_transaction,
args=(self.transaction, "create", self.docs,
self.transaction_commit,
self.update_count))
thread.start()
thread.join()
self.doc_gen(self.num_items,
op_type="update",
value={
"mutated": 1,
"value": "value1"
})
thread = threading.Thread(target=self.__thread_to_transaction,
args=(self.transaction, "update", self.docs,
self.transaction_commit,
self.update_count))
thread.start()
self.sleep(1)
self.__insert_sub_doc_and_validate("test_docs-0", "subdoc_insert",
xattr_key, val)
thread.join()
if self.transaction_commit:
self.__read_doc_and_validate("test_docs-0", val, xattr_key)
self.sleep(60, "Wait for transaction to complete")
self.verify_doc(self.num_items, self.client)
def test_TxnWithMultipleXattr(self):
xattrs_to_insert = [["my.attr", "value"], ["new_my.attr", "new_value"]]
self.doc_gen(self.num_items)
thread = threading.Thread(target=self.__thread_to_transaction,
args=(self.transaction, "create", self.docs,
self.transaction_commit,
self.update_count))
thread.start()
thread.join()
self.doc_gen(self.num_items,
op_type="update",
value={
"mutated": 1,
"value": "value1"
})
thread = threading.Thread(target=self.__thread_to_transaction,
args=(self.transaction, "update", self.docs,
self.transaction_commit,
self.update_count))
thread.start()
self.sleep(1, "Wait for transx-thread to start")
for key, val in xattrs_to_insert:
self.__insert_sub_doc_and_validate("test_docs-0", "subdoc_insert",
key, val)
thread.join()
if self.transaction_commit:
for key, val in xattrs_to_insert:
self.__read_doc_and_validate("test_docs-0", val, key)
self.sleep(60, "Wait for transaction to complete")
self.verify_doc(self.num_items, self.client)
