class MagmaCrashTests(MagmaBaseTest):
def setUp(self):
self.input = TestInputSingleton.input
self.input.test_params.update({"random_key": True})
super(MagmaCrashTests, self).setUp()
self.sdk_timeout = self.input.param("sdk_timeout", 10)
self.time_unit = "seconds"
self.graceful = self.input.param("graceful", False)
self.assertTrue(self.rest.update_autofailover_settings(False, 600),
"AutoFailover disabling failed")
self.crash_th = None
self.sdk_retry_strategy = self.input.param(
"sdk_retry_strategy", SDKConstants.RetryStrategy.FAIL_FAST)
def tearDown(self):
self.stop_crash = True
if self.crash_th and self.crash_th.is_alive():
self.crash_th.join()
super(MagmaCrashTests, self).tearDown()
def kill_magma_check_wal_file_size(self):
nIter = 200
while nIter > 0:
shell = RemoteMachineShellConnection(self.cluster.master)
shell.kill_memcached()
# self.bucket_util._wait_warmup_completed()
self.sleep(10, "sleep of 5s so that memcached can restart")
def test_crash_during_ops(self):
self.graceful = self.input.param("graceful", False)
wait_warmup = self.input.param("wait_warmup", True)
self.log.info("====test_crash_during_ops starts====")
self.compute_docs_ranges()
tasks_info = dict()
for collection in self.collections:
self.generate_docs(doc_ops=self.doc_ops, target_vbucket=None)
tem_tasks_info = self.loadgen_docs(
self.retry_exceptions,
self.ignore_exceptions,
scope=CbServer.default_scope,
collection=collection,
suppress_error_table=True,
skip_read_on_error=True,
_sync=False,
doc_ops=self.doc_ops,
track_failures=False,
sdk_retry_strategy=self.sdk_retry_strategy)
tasks_info.update(tem_tasks_info.items())
self.crash_th = threading.Thread(target=self.crash,
kwargs=dict(graceful=self.graceful,
wait=wait_warmup))
self.crash_th.start()
for task in tasks_info:
self.task_manager.get_task_result(task)
self.stop_crash = True
self.crash_th.join()
self.assertFalse(self.crash_failure,
"CRASH | CRITICAL | WARN messages found in cb_logs")
self.validate_seq_itr()
def test_crash_during_recovery(self):
self.compute_docs_ranges()
tasks_info = dict()
for collection in self.collections:
self.generate_docs(doc_ops=self.doc_ops, target_vbucket=None)
tem_tasks_info = self.loadgen_docs(
self.retry_exceptions,
self.ignore_exceptions,
scope=CbServer.default_scope,
collection=collection,
suppress_error_table=True,
skip_read_on_error=True,
_sync=False,
doc_ops=self.doc_ops,
track_failures=False,
sdk_retry_strategy=self.sdk_retry_strategy)
tasks_info.update(tem_tasks_info.items())
self.crash_th = threading.Thread(target=self.crash,
kwargs={"kill_itr": 5})
self.crash_th.start()
for task in tasks_info:
self.task_manager.get_task_result(task)
self.stop_crash = True
self.crash_th.join()
self.assertFalse(self.crash_failure,
"CRASH | CRITICAL | WARN messages found in cb_logs")
self.validate_seq_itr()
def test_crash_before_upserts(self):
self.log.info("test_update_multi starts")
self.change_swap_space(self.cluster.nodes_in_cluster)
upsert_doc_list = self.get_fragmentation_upsert_docs_list()
count = 0
self.mutate = 0
while count < self.test_itr:
self.log.info("Iteration == {}".format(count + 1))
self.sigkill_memcached(graceful=self.graceful)
for itr in upsert_doc_list:
self.doc_ops = "update"
self.update_start = 0
self.update_end = itr
if self.rev_update:
self.update_start = -int(itr - 1)
self.update_end = 1
self.generate_docs(doc_ops="update")
_ = self.loadgen_docs(self.retry_exceptions,
self.ignore_exceptions,
suppress_error_table=True,
_sync=True,
doc_ops="update")
self.bucket_util._wait_for_stats_all_buckets(
self.cluster, self.cluster.buckets)
count += 1
self.validate_data("update", self.gen_update)
self.change_swap_space(self.cluster.nodes_in_cluster, disable=False)
self.validate_seq_itr()
self.log.info("====test_update_multi ends====")
def test_crash_before_multi_update_deletes(self):
self.log.info("===test_crash_before_multi_update_deletes starts===")
self.change_swap_space(self.cluster.nodes_in_cluster)
count = 0
self.mutate = 0
for i in range(self.test_itr):
self.log.info("Step 1, Iteration= {}".format(i + 1))
while count < self.update_itr:
self.sigkill_memcached(graceful=self.graceful)
self.doc_ops = "update"
self.update_start = 0
self.update_end = self.num_items
self.generate_docs(doc_ops="update")
_ = self.loadgen_docs(self.retry_exceptions,
self.ignore_exceptions,
suppress_error_table=True,
_sync=True,
doc_ops="update")
self.bucket_util._wait_for_stats_all_buckets(
self.cluster, self.cluster.buckets)
count += 1
self.update_itr += self.update_itr
# data validation is done only for the last iteration
if i + 1 == self.test_itr:
self.validate_data("update", self.gen_update)
self.log.debug("Step 2, Iteration {}".format(i + 1))
self.sigkill_memcached()
self.doc_ops = "delete"
self.delete_start = 0
self.delete_end = self.num_items // 2
self.generate_docs(doc_ops="delete")
_ = self.loadgen_docs(self.retry_exceptions,
self.ignore_exceptions,
suppress_error_table=True,
_sync=True,
doc_ops="delete")
self.bucket_util._wait_for_stats_all_buckets(
self.cluster, self.cluster.buckets)
self.bucket_util.verify_stats_all_buckets(self.cluster,
self.num_items)
self.log.debug("Step 3, Iteration= {}".format(i + 1))
self.sigkill_memcached()
self.gen_create = copy.deepcopy(self.gen_delete)
self.doc_ops = "create"
_ = self.loadgen_docs(self.retry_exceptions,
self.ignore_exceptions,
suppress_error_table=True,
_sync=True,
doc_ops="create")
self.bucket_util._wait_for_stats_all_buckets(
self.cluster, self.cluster.buckets)
self.bucket_util.verify_stats_all_buckets(self.cluster,
self.num_items)
self.validate_data("create", self.gen_create)
self.change_swap_space(self.cluster.nodes_in_cluster, disable=False)
self.validate_seq_itr()
self.log.info("===test_crash_before_multi_update_deletes ends===")
def test_crash_during_get_ops(self):
self.log.info("test_crash_during_get_ops starts")
self.change_swap_space(self.cluster.nodes_in_cluster)
tasks_info = dict()
upsert_doc_list = self.get_fragmentation_upsert_docs_list()
for itr in upsert_doc_list:
self.doc_ops = "update"
self.update_start = 0
self.update_end = itr
self.mutate = -1
self.generate_docs(doc_ops="update")
update_task_info = self.loadgen_docs(self.retry_exceptions,
self.ignore_exceptions,
suppress_error_table=True,
_sync=False)
#update_task_info = self.loadgen_docs(
# self.retry_exceptions,
# self.ignore_exceptions,
# _sync=False)
tasks_info.update(update_task_info.items())
self.doc_ops = "read"
self.generate_docs(doc_ops="read")
start = -int(self.num_items - 1)
end = 1
reverse_read_gen = self.genrate_docs_basic(start, end)
count = 0
while count < self.read_thread_count:
read_task_info = self.loadgen_docs(self.retry_exceptions,
self.ignore_exceptions,
suppress_error_table=True,
_sync=False)
tasks_info.update(read_task_info.items())
count += 1
if count < self.read_thread_count:
read_task_info = self.bucket_util._async_validate_docs(
self.cluster,
reverse_read_gen,
"read",
0,
batch_size=self.batch_size,
process_concurrency=self.process_concurrency,
timeout_secs=self.sdk_timeout,
retry_exceptions=self.retry_exceptions,
ignore_exceptions=self.ignore_exceptions,
suppress_error_table=False)
tasks_info.update(read_task_info.items())
count += 1
self.crash_th = threading.Thread(target=self.crash,
kwargs={"graceful": self.graceful})
self.crash_th.start()
for task in tasks_info:
self.task_manager.get_task_result(task)
self.stop_crash = True
self.crash_th.join()
self.assertFalse(self.crash_failure,
"CRASH | CRITICAL | WARN messages found in cb_logs")
self.bucket_util._wait_for_stats_all_buckets(self.cluster,
self.cluster.buckets)
self.change_swap_space(self.cluster.nodes_in_cluster, disable=False)
self.validate_seq_itr()
self.log.info("test_crash_during_get_ops ends")
def test_crash_during_upserts_using_multithreads(self):
self.log.info("test_crash_during_upserts_using_multithreads starts")
self.change_swap_space(self.cluster.nodes_in_cluster)
tasks_info = dict()
self.doc_ops = "update"
self.update_start = 0
self.update_end = self.num_items
count = 0
while count < self.read_thread_count:
self.generate_docs(doc_ops="update")
update_task_info = self.loadgen_docs(
self.retry_exceptions,
self.ignore_exceptions,
suppress_error_table=True,
skip_read_on_error=True,
_sync=False,
track_failures=False,
sdk_retry_strategy=self.sdk_retry_strategy)
tasks_info.update(update_task_info.items())
count += 1
self.sleep(5)
self.crash_th = threading.Thread(target=self.crash,
kwargs={"graceful": self.graceful})
self.crash_th.start()
for task in tasks_info:
self.task_manager.get_task_result(task)
self.stop_crash = True
self.crash_th.join()
self.assertFalse(self.crash_failure,
"CRASH | CRITICAL | WARN messages found in cb_logs")
self.bucket_util._wait_for_stats_all_buckets(self.cluster,
self.cluster.buckets)
self.change_swap_space(self.cluster.nodes_in_cluster, disable=False)
self.validate_seq_itr()
self.log.info("test_crash_during_upserts_using_multithreads ends")
def test_crash_during_multi_updates_of_single_doc(self):
self.log.info(
"==test_crash_during_multi_updates_of_single_doc starts==")
self.change_swap_space(self.cluster.nodes_in_cluster)
self.client = SDKClient([self.cluster.master],
self.cluster.buckets[0],
scope=CbServer.default_scope,
collection=CbServer.default_collection)
self.doc_ops = "update"
self.gen_update = self.genrate_docs_basic(start=0, end=1)
key, val = self.gen_update.next()
def upsert_doc(start_num, end_num, key_obj, val_obj):
for i in range(start_num, end_num):
val_obj.put("mutated", i)
self.client.upsert(key_obj, val_obj)
self.crash_th = threading.Thread(target=self.crash,
kwargs={"graceful": self.graceful})
self.crash_th.start()
threads = []
start = 0
end = 0
for _ in range(10):
start = end
end += 10
th = threading.Thread(target=upsert_doc,
args=[start, end, key, val])
th.start()
threads.append(th)
for th in threads:
th.join()
self.stop_crash = True
self.crash_th.join()
self.assertFalse(self.crash_failure,
"CRASH | CRITICAL | WARN messages found in cb_logs")
self.bucket_util._wait_for_stats_all_buckets(self.cluster,
self.cluster.buckets)
success, _ = self.client.get_multi([key], self.wait_timeout)
self.assertIs(key in success,
True,
msg="key {} doesn't exist\
".format(key))
expected_val = Json.loads(val.toString())
actual_val = Json.loads(success[key]['value'].toString())
self.assertIs(expected_val == actual_val,
True,
msg="expected_val-{} != Actual_val-{}\
".format(expected_val, actual_val))
self.change_swap_space(self.cluster.nodes_in_cluster, disable=False)
self.validate_seq_itr()
self.log.info("==test_crash_during_multi_updates_of_single_doc ends==")
def test_crash_during_val_movement_across_trees(self):
self.log.info("==test_crash_during_val_movement_across_trees starts==")
self.change_swap_space(self.cluster.nodes_in_cluster)
upsert_size = 0
if self.doc_size < 32:
upsert_size = 2048
self.update_start = 0
self.update_end = self.num_items
if self.rev_update:
self.update_start = -int(self.num_items - 1)
self.update_end = 1
self.doc_ops = "update"
self.crash_th = threading.Thread(target=self.crash,
kwargs={"graceful": self.graceful})
self.crash_th.start()
count = 0
while count < self.test_itr:
self.log.info("Iteration == {}".format(count))
self.mutate += 1
self.gen_update = doc_generator(
self.key,
self.update_start,
self.update_end,
doc_size=upsert_size,
doc_type=self.doc_type,
target_vbucket=self.target_vbucket,
vbuckets=self.cluster.vbuckets,
key_size=self.key_size,
mutate=self.mutate,
randomize_doc_size=self.randomize_doc_size,
randomize_value=self.randomize_value,
mix_key_size=self.mix_key_size,
deep_copy=self.deep_copy)
_ = self.loadgen_docs(self.retry_exceptions,
self.ignore_exceptions,
suppress_error_table=True,
skip_read_on_error=True,
_sync=True,
track_failures=False,
sdk_retry_strategy=self.sdk_retry_strategy)
self.bucket_util._wait_for_stats_all_buckets(
self.cluster, self.cluster.buckets)
self.generate_docs(doc_ops="update")
_ = self.loadgen_docs(self.retry_exceptions,
self.ignore_exceptions,
suppress_error_table=True,
skip_read_on_error=True,
_sync=True,
track_failures=False,
sdk_retry_strategy=self.sdk_retry_strategy)
self.bucket_util._wait_for_stats_all_buckets(
self.cluster, self.cluster.buckets)
count += 1
self.stop_crash = True
self.crash_th.join()
self.assertFalse(self.crash_failure,
"CRASH | CRITICAL | WARN messages found in cb_logs")
self.change_swap_space(self.cluster.nodes_in_cluster, disable=False)
self.validate_seq_itr()
self.log.info("==test_crash_during_val_movement_across_trees ends==")
class basic_ops(BaseTestCase):
def setUp(self):
super(basic_ops, self).setUp()
self.test_log = logging.getLogger("test")
self.key = 'test_docs'.rjust(self.key_size, '0')
nodes_init = self.cluster.servers[1:self.nodes_init] if self.nodes_init != 1 else []
self.task.rebalance([self.cluster.master], nodes_init, [])
self.cluster.nodes_in_cluster.extend([self.cluster.master] + nodes_init)
self.bucket_util.add_rbac_user()
if self.default_bucket:
self.bucket_util.create_default_bucket(replica=self.num_replicas,
compression_mode=self.compression_mode, ram_quota=100)
time.sleep(10)
self.def_bucket= self.bucket_util.get_all_buckets()
self.client = VBucketAwareMemcached(RestConnection(self.cluster.master), self.def_bucket[0])
self.__durability_level()
self.create_Transaction()
self._stop = threading.Event()
self.log.info("==========Finished Basic_ops base setup========")
def tearDown(self):
self.client.close()
super(basic_ops, self).tearDown()
def __durability_level(self):
if self.durability_level == "MAJORITY":
self.durability = 1
elif self.durability_level == "MAJORITY_AND_PERSIST_ON_MASTER":
self.durability = 2
elif self.durability_level == "PERSIST_TO_MAJORITY":
self.durability = 3
elif self.durability_level == "ONLY_NONE":
self.durability = 4
else:
self.durability = 0
def get_doc_generator(self, start, end):
age = range(5)
first = ['james', 'sharon']
body = [''.rjust(self.doc_size - 10, 'a')]
template = '{{ "age": {0}, "first_name": "{1}", "body": "{2}"}}'
generator = DocumentGenerator(self.key, template, age, first, body, start=start,
end=end)
return generator
def set_exception(self, exception):
self.exception = exception
raise BaseException("Got an exception {}".format(self.exception))
def __chunks(self, l, n):
"""Yield successive n-sized chunks from l."""
for i in range(0, len(l), n):
yield l[i:i + n]
def create_Transaction(self, client=None):
if not client:
client = self.client
transaction_config = Transaction().createTransactionConfig(self.transaction_timeout, self.durability)
try:
self.transaction = Transaction().createTansaction(client.cluster, transaction_config)
except Exception as e:
self.set_exception(e)
def __thread_to_transaction(self, transaction, op_type, doc, txn_commit, update_count=1, sync=True, set_exception=True, client=None):
if not client:
client = self.client
if op_type == "create":
exception = Transaction().RunTransaction(transaction, [client.collection], doc, [], [], txn_commit, sync, update_count)
elif op_type == "update":
self.test_log.info("updating all the keys through threads")
exception = Transaction().RunTransaction(transaction, [client.collection], [], doc, [], txn_commit, sync, update_count)
elif op_type == "delete":
exception = Transaction().RunTransaction(transaction, [client.collection], [], [], doc, txn_commit, sync, update_count)
if set_exception:
if exception:
self.set_exception("Failed")
def doc_gen(self, num_items, start=0, value={'value':'value1'}):
self.docs = []
self.keys = []
self.content = self.client.translate_to_json_object(value)
for i in range(start, self.num_items):
key = "test_docs-" + str(i)
doc = Tuples.of(key, self.content)
self.keys.append(key)
self.docs.append(doc)
def verify_doc(self, num_items, client):
for i in range(num_items):
key = "test_docs-" + str(i)
result = client.read(key)
actual_val = self.client.translate_to_json_object(result['value'])
self.assertEquals(self.content, actual_val)
def test_MultiThreadTxnLoad(self):
# Atomicity.basic_retry.basic_ops.test_MultiThreadTxnLoad,num_items=1000
''' Load data through txn, update half the items through different threads
and delete half the items through different threads. if update_retry then update and delete
the same key in two different transaction and make sure update fails '''
self.num_txn = self.input.param("num_txn", 9)
self.update_retry = self.input.param("update_retry", False)
self.doc_gen(self.num_items)
threads = []
# create the docs
exception = Transaction().RunTransaction(self.transaction, [self.client.collection], self.docs, [], [], self.transaction_commit, True, self.update_count)
if exception:
self.set_exception("Failed")
if self.update_retry:
threads.append(threading.Thread(target=self.__thread_to_transaction, args=(self.transaction, "delete", self.keys, self.transaction_commit, self.update_count)))
threads.append(threading.Thread(target=self.__thread_to_transaction, args=(self.transaction, "update", self.keys, self.transaction_commit, self.update_count)))
else:
update_docs = self.__chunks(self.keys[:self.num_items/2], self.num_txn)
delete_docs = self.__chunks(self.keys[self.num_items/2:], self.num_txn)
for keys in update_docs:
threads.append(threading.Thread(target=self.__thread_to_transaction, args=(self.transaction, "update", keys, self.transaction_commit, self.update_count)))
for keys in delete_docs:
threads.append(threading.Thread(target=self.__thread_to_transaction, args=(self.transaction, "delete", keys, self.transaction_commit, self.update_count)))
for thread in threads:
thread.start()
for thread in threads:
thread.join()
self.sleep(60)
if self.update_retry:
for key in self.keys:
result = self.client.read(key)
self.assertEquals(result['status'], False)
else:
self.value = {'mutated':1, 'value':'value1'}
self.content = self.client.translate_to_json_object(self.value)
self.verify_doc(self.num_items/2, self.client)
for key in self.keys[self.num_items/2:]:
result = self.client.read(key)
self.assertEquals(result['status'], False)
def test_basic_retry(self):
''' Load set of data to the cluster, update through 2 different threads, make sure transaction maintains the order of update'''
self.write_conflict = self.input.param("write_conflict", 2)
self.test_log.info("going to create and execute the task")
self.gen_create = self.get_doc_generator(0, self.num_items)
task = self.task.async_load_gen_docs_atomicity(self.cluster, self.def_bucket,
self.gen_create, "create" , exp=0,
batch_size=10,
process_concurrency=8,
replicate_to=self.replicate_to,
persist_to=self.persist_to, timeout_secs=self.sdk_timeout,
retries=self.sdk_retries,update_count=self.update_count, transaction_timeout=self.transaction_timeout,
commit=True,durability=self.durability_level,sync=self.sync)
self.task.jython_task_manager.get_task_result(task)
self.test_log.info("get all the keys in the cluster")
self.doc_gen(self.num_items)
threads = []
for update_count in [2, 4, 6]:
threads.append(threading.Thread(target=self.__thread_to_transaction, args=(self.transaction, "update", self.keys, self.transaction_commit, update_count)))
# Add verification task
if self.transaction_commit:
self.update_count = 6
else:
self.update_count = 0
for thread in threads:
thread.start()
self.sleep(2)
for thread in threads:
thread.join()
self.sleep(10)
task = self.task.async_load_gen_docs_atomicity(self.cluster, self.def_bucket,
self.gen_create, "verify" , exp=0,
batch_size=10,
process_concurrency=8,
replicate_to=self.replicate_to,
persist_to=self.persist_to, timeout_secs=self.sdk_timeout,
retries=self.sdk_retries,update_count=self.update_count, transaction_timeout=self.transaction_timeout,
commit=True,durability=self.durability_level)
self.task.jython_task_manager.get_task_result(task)
def test_basic_retry_async(self):
self.test_log.info("going to create and execute the task")
self.gen_create = self.get_doc_generator(0, self.num_items)
task = self.task.async_load_gen_docs_atomicity(self.cluster, self.def_bucket,
self.gen_create, "create" , exp=0,
batch_size=10,
process_concurrency=1,
replicate_to=self.replicate_to,
persist_to=self.persist_to, timeout_secs=self.sdk_timeout,
retries=self.sdk_retries,update_count=self.update_count, transaction_timeout=self.transaction_timeout,
commit=True,durability=self.durability_level,sync=True,num_threads=1)
self.task.jython_task_manager.get_task_result(task)
self.test_log.info("get all the keys in the cluster")
keys = ["test_docs-0"]*20
exception = Transaction().RunTransaction(self.transaction, [self.client.collection], [], keys, [], self.transaction_commit, False, 0)
if exception:
self.set_exception(Exception(exception))
def basic_concurrency(self):
self.crash = self.input.param("crash", False)
self.doc_gen(self.num_items)
# run transaction
thread = threading.Thread(target=self.__thread_to_transaction, args=(self.transaction, "create", self.docs, self.transaction_commit, self.update_count, True, False))
thread.start()
self.sleep(1)
if self.crash:
self.client.cluster.shutdown()
self.transaction.close()
print "going to create a new transaction"
self.client1 = VBucketAwareMemcached(RestConnection(self.cluster.master), self.def_bucket[0])
self.create_Transaction(self.client1)
self.sleep(self.transaction_timeout+60)
exception = Transaction().RunTransaction(self.transaction, [self.client1.collection], self.docs, [], [], self.transaction_commit, self.sync, self.update_count)
if exception:
time.sleep(60)
self.verify_doc(self.num_items, self.client1)
self.client1.close()
else:
key = "test_docs-0"
# insert will fail
result = self.client.insert(key, "value")
self.assertEqual(result["status"], False)
# Update should pass
result = self.client.upsert(key,"value")
self.assertEqual(result["status"], True)
# delete should pass
result = self.client.delete(key)
self.assertEqual(result["status"], True)
thread.join()
def test_stop_loading(self):
''' Load through transactions and close the transaction abruptly, create a new transaction sleep for 60 seconds and
perform create on the same set of docs '''
self.num_txn = self.input.param("num_txn", 9)
self.doc_gen(self.num_items)
threads = []
docs = list(self.__chunks(self.docs, len(self.docs)/self.num_txn))
for doc in docs:
threads.append(threading.Thread(target=self.__thread_to_transaction, args=(self.transaction, "create", doc, self.transaction_commit, self.update_count, True, False)))
for thread in threads:
thread.start()
self.client.cluster.shutdown()
self.transaction.close()
self.client1 = VBucketAwareMemcached(RestConnection(self.cluster.master), self.def_bucket[0])
self.create_Transaction(self.client1)
self.sleep(self.transaction_timeout+60) # sleep for 60 seconds so that transaction cleanup can happen
self.test_log.info("going to start the load")
for doc in docs:
exception = Transaction().RunTransaction(self.transaction, [self.client1.collection], doc, [], [], self.transaction_commit, self.sync, self.update_count)
if exception:
time.sleep(60)
self.verify_doc(self.num_items, self.client1)
self.client1.close()
def __insert_sub_doc_and_validate(self, doc_id, op_type, key, value):
_, failed_items = self.client.crud(
op_type,
doc_id,
[key, value],
durability=self.durability_level,
timeout=self.sdk_timeout,
time_unit="seconds",
create_path=True,
xattr=True)
self.assertFalse(failed_items, "Subdoc Xattr insert failed")
def __read_doc_and_validate(self, doc_id, expected_val, subdoc_key=None):
if subdoc_key:
success, failed_items = self.client.crud("subdoc_read",
doc_id,
subdoc_key,
xattr=True)
self.assertFalse(failed_items, "Xattr read failed")
self.assertEqual(expected_val,
str(success[doc_id]["value"][0]),
"Sub_doc value mismatch: %s != %s"
% (success[doc_id]["value"][0],
expected_val))
def test_TxnWithXattr(self):
self.system_xattr = self.input.param("system_xattr", False)
if self.system_xattr:
xattr_key = "my._attr"
else:
xattr_key="my.attr"
val = "v" * self.doc_size
self.doc_gen(self.num_items)
threads = threading.Thread(target=self.__thread_to_transaction, args=(self.transaction, "create", self.docs, self.transaction_commit, self.update_count))
threads.start()
self.sleep(1)
self.__insert_sub_doc_and_validate("test_docs-0", "subdoc_insert",
xattr_key, val)
threads.join()
if self.transaction_commit:
self.__read_doc_and_validate("test_docs-0", val, xattr_key)
self.sleep(60)
self.verify_doc(self.num_items, self.client)
def test_TxnWithMultipleXattr(self):
xattrs_to_insert = [["my.attr", "value"],
["new_my.attr", "new_value"]]
self.doc_gen(self.num_items)
threads = threading.Thread(target=self.__thread_to_transaction, args=(self.transaction, "create", self.docs, self.transaction_commit, self.update_count))
threads.start()
self.sleep(1)
for key, val in xattrs_to_insert:
self.__insert_sub_doc_and_validate("test_docs-0", "subdoc_insert",
key, val)
threads.join()
if self.transaction_commit:
for key, val in xattrs_to_insert:
self.__read_doc_and_validate("test_docs-0", val, key)
self.sleep(60)
self.verify_doc(self.num_items, self.client)
class BasicCrudTests(MagmaBaseTest):
def setUp(self):
super(BasicCrudTests, self).setUp()
self.enable_disable_swap_space(self.servers)
start = 0
end = self.num_items
start_read = 0
end_read = self.num_items
if self.rev_write:
start = -int(self.num_items - 1)
end = 1
if self.rev_read:
start_read = -int(self.num_items - 1)
end_read = 1
self.gen_create = doc_generator(
self.key,
start,
end,
doc_size=self.doc_size,
doc_type=self.doc_type,
target_vbucket=self.target_vbucket,
vbuckets=self.cluster_util.vbuckets,
key_size=self.key_size,
randomize_doc_size=self.randomize_doc_size,
randomize_value=self.randomize_value,
mix_key_size=self.mix_key_size,
deep_copy=self.deep_copy)
self.result_task = self._load_all_buckets(self.cluster,
self.gen_create,
"create",
0,
batch_size=self.batch_size,
dgm_batch=self.dgm_batch)
if self.active_resident_threshold != 100:
for task in self.result_task.keys():
self.num_items = task.doc_index
self.log.info("Verifying num_items counts after doc_ops")
self.bucket_util._wait_for_stats_all_buckets()
self.bucket_util.verify_stats_all_buckets(self.num_items)
self.disk_usage = dict()
if self.standard_buckets == 1 or self.standard_buckets == self.magma_buckets:
for bucket in self.bucket_util.get_all_buckets():
disk_usage = self.get_disk_usage(bucket, self.servers)
self.disk_usage[bucket.name] = disk_usage[0]
self.log.info(
"For bucket {} disk usage after initial creation is {}MB\
".format(bucket.name, self.disk_usage[bucket.name]))
self.gen_read = doc_generator(
self.key,
start_read,
end_read,
doc_size=self.doc_size,
doc_type=self.doc_type,
target_vbucket=self.target_vbucket,
vbuckets=self.cluster_util.vbuckets,
key_size=self.key_size,
randomize_doc_size=self.randomize_doc_size,
randomize_value=self.randomize_value,
mix_key_size=self.mix_key_size,
deep_copy=self.deep_copy)
self.cluster_util.print_cluster_stats()
self.bucket_util.print_bucket_stats()
def tearDown(self):
super(BasicCrudTests, self).tearDown()
def test_expiry(self):
result = True
self.gen_create = doc_generator(self.key,
0,
10,
doc_size=20,
doc_type=self.doc_type,
key_size=self.key_size)
tasks_info = self.bucket_util._async_load_all_buckets(
self.cluster,
self.gen_create,
"create",
10,
batch_size=10,
process_concurrency=1,
persist_to=self.persist_to,
replicate_to=self.replicate_to,
durability=self.durability_level,
pause_secs=5,
timeout_secs=self.sdk_timeout,
retries=self.sdk_retries,
)
self.task.jython_task_manager.get_task_result(tasks_info.keys()[0])
self.sleep(20)
self.client = SDKClient([self.cluster.master],
self.bucket_util.buckets[0],
scope=CbServer.default_scope,
collection=CbServer.default_collection)
for i in range(10):
key = (self.key + "-" +
str(i).zfill(self.key_size - len(self.key)))
try:
getReplicaResult = self.client.collection.getAnyReplica(
key, GetAnyReplicaOptions.getAnyReplicaOptions())
if getReplicaResult:
result = False
try:
self.log.info(
"Able to retreive: %s" % {
"key": key,
"value": getReplicaResult.contentAsObject(),
"cas": getReplicaResult.cas()
})
except Exception as e:
print str(e)
except DocumentUnretrievableException as e:
pass
if len(self.client.getFromAllReplica(key)) > 0:
result = False
self.client.close()
self.assertTrue(result, "SDK is able to retrieve expired documents")
def test_basic_create_read(self):
"""
Write and Read docs parallely , While reading we are using
old doc generator (self.gen_create)
using which we already created docs in magam_base
for writing we are creating a new doc generator.
Befor we start read, killing memcached to make sure,
all reads happen from magma/storage
"""
self.log.info("Loading and Reading docs parallel")
count = 0
init_items = self.num_items
while count < self.test_itr:
self.log.info("Create Iteration count == {}".format(count))
for node in self.cluster.nodes_in_cluster:
shell = RemoteMachineShellConnection(node)
shell.kill_memcached()
shell.disconnect()
self.doc_ops = "create:read"
start = self.num_items
end = self.num_items + init_items
start_read = self.num_items
end_read = self.num_items + init_items
if self.rev_write:
start = -int(self.num_items + init_items - 1)
end = -int(self.num_items - 1)
if self.rev_read:
start_read = -int(self.num_items + init_items - 1)
end_read = -int(self.num_items - 1)
self.gen_create = doc_generator(
self.key,
start,
end,
doc_size=self.doc_size,
doc_type=self.doc_type,
target_vbucket=self.target_vbucket,
vbuckets=self.cluster_util.vbuckets,
key_size=self.key_size,
randomize_doc_size=self.randomize_doc_size,
randomize_value=self.randomize_value,
mix_key_size=self.mix_key_size,
deep_copy=self.deep_copy)
_ = self.loadgen_docs(self.retry_exceptions,
self.ignore_exceptions,
_sync=True)
self.log.info("Verifying doc counts after create doc_ops")
self.bucket_util._wait_for_stats_all_buckets()
self.bucket_util.verify_stats_all_buckets(self.num_items)
self.gen_read = doc_generator(
self.key,
start_read,
end_read,
doc_size=self.doc_size,
doc_type=self.doc_type,
target_vbucket=self.target_vbucket,
vbuckets=self.cluster_util.vbuckets,
key_size=self.key_size,
randomize_doc_size=self.randomize_doc_size,
randomize_value=self.randomize_value,
mix_key_size=self.mix_key_size,
deep_copy=self.deep_copy)
if self.doc_size <= 32:
for bucket in self.bucket_util.get_all_buckets():
disk_usage = self.get_disk_usage(bucket, self.servers)
self.assertIs(
disk_usage[2] > disk_usage[3], True,
"For Bucket {} , Disk Usage for seqIndex'\n' \
After new Creates count {}'\n' \
exceeds keyIndex disk'\n' \
usage".format(bucket.name, count + 1))
if self.standard_buckets > 1 and self.standard_buckets == self.magma_buckets:
disk_usage = dict()
for bucket in self.bucket_util.get_all_buckets():
usage = self.get_disk_usage(bucket, self.servers)
disk_usage[bucket.name] = usage[0]
self.assertTrue(
all([
disk_usage[disk_usage.keys()[0]] == disk_usage[key]
for key in disk_usage.keys()
]), '''Disk Usage for magma buckets
is not equal for same number of docs ''')
count += 1
self.log.info("====test_basic_create_read ends====")
def test_update_multi(self):
"""
Update all the docs n times, and after each iteration
check for space amplificationa and data validation
"""
count = 0
mutated = 1
update_doc_count = int(
math.ceil(
float(self.fragmentation * self.num_items) /
(100 - self.fragmentation)))
self.log.info("Count of docs to be updated is {}\
".format(update_doc_count))
num_update = list()
while update_doc_count > self.num_items:
num_update.append(self.num_items)
update_doc_count -= self.num_items
if update_doc_count > 0:
num_update.append(update_doc_count)
while count < self.test_itr:
self.log.info("Update Iteration count == {}".format(count))
for node in self.cluster.nodes_in_cluster:
shell = RemoteMachineShellConnection(node)
shell.kill_memcached()
shell.disconnect()
self.assertTrue(
self.bucket_util._wait_warmup_completed(
[self.cluster_util.cluster.master],
self.bucket_util.buckets[0],
wait_time=self.wait_timeout * 10))
self.log.debug("List of docs to be updated {}\
".format(num_update))
for itr in num_update:
self.doc_ops = "update"
start = 0
end = itr
if self.rev_update:
start = -int(itr - 1)
end = 1
self.gen_update = doc_generator(
self.key,
start,
end,
doc_size=self.doc_size,
doc_type=self.doc_type,
target_vbucket=self.target_vbucket,
vbuckets=self.cluster_util.vbuckets,
key_size=self.key_size,
mutate=mutated,
randomize_doc_size=self.randomize_doc_size,
randomize_value=self.randomize_value,
mix_key_size=self.mix_key_size,
deep_copy=self.deep_copy)
mutated += 1
_ = self.loadgen_docs(self.retry_exceptions,
self.ignore_exceptions,
_sync=True)
self.log.info("Waiting for ep-queues to get drained")
self.bucket_util._wait_for_stats_all_buckets()
disk_usage = self.get_disk_usage(
self.bucket_util.get_all_buckets()[0], self.servers)
_res = disk_usage[0]
self.log.info("After update count {} disk usage is {}\
".format(count + 1, _res))
usage_factor = (
(float(self.num_items + sum(num_update)) / self.num_items) +
0.5)
self.log.debug("Disk usage factor is {}".format(usage_factor))
self.assertIs(
_res >
usage_factor * self.disk_usage[self.disk_usage.keys()[0]],
False, "Disk Usage {}MB After Update'\n' \
Count {} exceeds Actual'\n' \
disk usage {}MB by {}'\n' \
times".format(_res, count,
self.disk_usage[self.disk_usage.keys()[0]],
usage_factor))
count += 1
data_validation = self.task.async_validate_docs(
self.cluster,
self.bucket_util.buckets[0],
self.gen_update,
"update",
0,
batch_size=self.batch_size,
process_concurrency=self.process_concurrency,
pause_secs=5,
timeout_secs=self.sdk_timeout)
self.task.jython_task_manager.get_task_result(data_validation)
self.enable_disable_swap_space(self.servers, disable=False)
self.log.info("====test_update_multi ends====")
def test_multi_update_delete(self):
"""
Step 1: Kill memcached and Update all the docs update_itr times
After each iteration check for space amplification
and for last iteration
of test_itr validate docs
Step 2: Delete half the docs, check sapce amplification
Step 3 Recreate check for space amplification.
Repeat all above steps test_itr times
Step 4 : Do data validation for newly create docs
"""
count = 0
mutated = 1
for i in range(self.test_itr):
while count < self.update_itr:
self.log.debug(
"Iteration {}: Step 1 of test_multi_update_delete \
".format(self.test_itr + 1))
for node in self.cluster.nodes_in_cluster:
shell = RemoteMachineShellConnection(node)
shell.kill_memcached()
shell.disconnect()
self.assertTrue(
self.bucket_util._wait_warmup_completed(
[self.cluster_util.cluster.master],
self.bucket_util.buckets[0],
wait_time=self.wait_timeout * 10))
self.doc_ops = "update"
start = 0
end = self.num_items
if self.rev_update:
start = -int(self.num_items - 1)
end = 1
self.gen_update = doc_generator(
self.key,
start,
end,
doc_size=self.doc_size,
doc_type=self.doc_type,
target_vbucket=self.target_vbucket,
vbuckets=self.cluster_util.vbuckets,
key_size=self.key_size,
mutate=mutated,
randomize_doc_size=self.randomize_doc_size,
randomize_value=self.randomize_value,
mix_key_size=self.mix_key_size,
deep_copy=self.deep_copy)
mutated += 1
_ = self.loadgen_docs(self.retry_exceptions,
self.ignore_exceptions,
_sync=True)
self.log.info("Waiting for ep-queues to get drained")
self.bucket_util._wait_for_stats_all_buckets()
disk_usage = self.get_disk_usage(
self.bucket_util.get_all_buckets()[0], self.servers)
_res = disk_usage[0]
self.log.info("After update count {} disk usage is {}MB\
".format(count + 1, _res))
self.assertIs(
_res > 2.5 * self.disk_usage[self.disk_usage.keys()[0]],
False, "Disk Usage {}MB After \
Update Count {} exceeds Actual \
disk usage {}MB by 2.5\
times".format(_res, count,
self.disk_usage[self.disk_usage.keys()[0]]))
count += 1
# Will check data validatio only in the last
# iteration of test_tr to avoid multiple
# data validation, that is why below if check
if i + 1 == self.test_itr:
data_validation = self.task.async_validate_docs(
self.cluster,
self.bucket_util.buckets[0],
self.gen_update,
"update",
0,
batch_size=self.batch_size,
process_concurrency=self.process_concurrency,
pause_secs=5,
timeout_secs=self.sdk_timeout)
self.task.jython_task_manager.get_task_result(data_validation)
self.update_itr += self.update_itr
self.log.debug("Iteration {}: Step 2 of test_multi_update_delete \
".format(self.test_itr + 1))
start_del = 0
end_del = self.num_items // 2
if self.rev_del:
start_del = -int(self.num_items // 2 - 1)
end_del = 1
self.gen_delete = doc_generator(
self.key,
start_del,
end_del,
doc_size=self.doc_size,
doc_type=self.doc_type,
target_vbucket=self.target_vbucket,
vbuckets=self.cluster_util.vbuckets,
key_size=self.key_size,
randomize_doc_size=self.randomize_doc_size,
randomize_value=self.randomize_value,
mix_key_size=self.mix_key_size,
deep_copy=self.deep_copy)
self.log.info("Deleting num_items//2 docs")
self.doc_ops = "delete"
_ = self.loadgen_docs(self.retry_exceptions,
self.ignore_exceptions,
_sync=True)
self.bucket_util._wait_for_stats_all_buckets()
self.bucket_util.verify_stats_all_buckets(self.num_items)
disk_usage = self.get_disk_usage(
self.bucket_util.get_all_buckets()[0], self.servers)
_res = disk_usage[0]
self.log.info("After delete count {} disk usage is {}MB\
".format(i + 1, _res))
self.assertIs(
_res > 2.5 * self.disk_usage[self.disk_usage.keys()[0]], False,
"Disk Usage {}MB After \
Delete count {} exceeds Actual \
disk usage {}MB by 2.5 \
times".format(_res, i + 1,
self.disk_usage[self.disk_usage.keys()[0]]))
self.log.debug("Iteration{}: Step 3 of test_multi_update_delete \
".format(self.test_itr + 1))
self.gen_create = copy.deepcopy(self.gen_delete)
self.log.info("Recreating num_items//2 docs")
self.doc_ops = "create"
_ = self.loadgen_docs(self.retry_exceptions,
self.ignore_exceptions,
_sync=True)
self.bucket_util._wait_for_stats_all_buckets()
self.bucket_util.verify_stats_all_buckets(self.num_items)
disk_usage = self.get_disk_usage(
self.bucket_util.get_all_buckets()[0], self.servers)
_res = disk_usage[0]
self.log.info("disk usage after new create \
is {}".format(_res))
self.assertIs(
_res > 2.5 * self.disk_usage[self.disk_usage.keys()[0]], False,
"Disk Usage {}MB After \
new Creates count {} exceeds \
Actual disk usage {}MB by \
2.5 times".format(_res, i + 1,
self.disk_usage[self.disk_usage.keys()[0]]))
self.log.debug("Iteration{}: Step 4 of test_multi_update_delete \
".format(self.test_itr + 1))
data_validation = self.task.async_validate_docs(
self.cluster,
self.bucket_util.buckets[0],
self.gen_create,
"create",
0,
batch_size=self.batch_size,
process_concurrency=self.process_concurrency,
pause_secs=5,
timeout_secs=self.sdk_timeout)
self.task.jython_task_manager.get_task_result(data_validation)
self.log.info("====test_multiUpdate_delete ends====")
def test_update_rev_update(self):
count = 0
mutated = 1
for i in range(self.test_itr):
while count < self.update_itr:
for node in self.cluster.nodes_in_cluster:
shell = RemoteMachineShellConnection(node)
shell.kill_memcached()
shell.disconnect()
self.assertTrue(
self.bucket_util._wait_warmup_completed(
[self.cluster_util.cluster.master],
self.bucket_util.buckets[0],
wait_time=self.wait_timeout * 10))
tasks_info = dict()
data_validation = []
g_update = doc_generator(
self.key,
0,
self.num_items // 2,
doc_size=self.doc_size,
doc_type=self.doc_type,
target_vbucket=self.target_vbucket,
vbuckets=self.cluster_util.vbuckets,
key_size=self.key_size,
mutate=mutated,
randomize_doc_size=self.randomize_doc_size,
randomize_value=self.randomize_value,
mix_key_size=self.mix_key_size,
deep_copy=self.deep_copy)
mutated += 1
tem_tasks_info = self.bucket_util._async_load_all_buckets(
self.cluster,
g_update,
"update",
0,
batch_size=self.batch_size,
process_concurrency=self.process_concurrency,
persist_to=self.persist_to,
replicate_to=self.replicate_to,
durability=self.durability_level,
pause_secs=5,
timeout_secs=self.sdk_timeout,
retries=self.sdk_retries,
retry_exceptions=self.retry_exceptions,
ignore_exceptions=self.ignore_exceptions)
tasks_info.update(tem_tasks_info.items())
start = -(self.num_items // 2 - 1)
end = 1
r_update = doc_generator(
self.key,
start,
end,
doc_size=self.doc_size,
doc_type=self.doc_type,
target_vbucket=self.target_vbucket,
vbuckets=self.cluster_util.vbuckets,
key_size=self.key_size,
mutate=mutated,
randomize_doc_size=self.randomize_doc_size,
randomize_value=self.randomize_value,
mix_key_size=self.mix_key_size,
deep_copy=self.deep_copy)
mutated += 1
if self.next_half:
mutated -= 2
start = -(self.num_items - 1)
end = -(self.num_items // 2 - 1)
r_update = doc_generator(
self.key,
start,
end,
doc_size=self.doc_size,
doc_type=self.doc_type,
target_vbucket=self.target_vbucket,
vbuckets=self.cluster_util.vbuckets,
key_size=self.key_size,
mutate=mutated,
randomize_doc_size=self.randomize_doc_size,
randomize_value=self.randomize_value,
mix_key_size=self.mix_key_size,
deep_copy=self.deep_copy)
mutated += 1
tem_tasks_info = self.bucket_util._async_load_all_buckets(
self.cluster,
r_update,
"update",
0,
batch_size=self.batch_size,
process_concurrency=self.process_concurrency,
persist_to=self.persist_to,
replicate_to=self.replicate_to,
durability=self.durability_level,
pause_secs=5,
timeout_secs=self.sdk_timeout,
retries=self.sdk_retries,
retry_exceptions=self.retry_exceptions,
ignore_exceptions=self.ignore_exceptions)
tasks_info.update(tem_tasks_info.items())
for task in tasks_info:
self.task_manager.get_task_result(task)
self.bucket_util.verify_doc_op_task_exceptions(
tasks_info, self.cluster)
self.bucket_util.log_doc_ops_task_failures(tasks_info)
if not self.next_half:
tem_tasks_info = self.bucket_util._async_load_all_buckets(
self.cluster,
r_update,
"update",
0,
batch_size=self.batch_size,
process_concurrency=self.process_concurrency,
persist_to=self.persist_to,
replicate_to=self.replicate_to,
durability=self.durability_level,
pause_secs=5,
timeout_secs=self.sdk_timeout,
retries=self.sdk_retries,
retry_exceptions=self.retry_exceptions,
ignore_exceptions=self.ignore_exceptions)
for task in tem_tasks_info:
self.task_manager.get_task_result(task)
self.bucket_util.verify_doc_op_task_exceptions(
tem_tasks_info, self.cluster)
self.bucket_util.log_doc_ops_task_failures(tem_tasks_info)
self.log.info("Waiting for ep-queues to get drained")
self.bucket_util._wait_for_stats_all_buckets()
if self.next_half:
data_validation.extend([
self.task.async_validate_docs(
self.cluster,
self.bucket_util.buckets[0],
g_update,
"update",
0,
batch_size=self.batch_size,
process_concurrency=self.process_concurrency,
pause_secs=5,
timeout_secs=self.sdk_timeout),
self.task.async_validate_docs(
self.cluster,
self.bucket_util.buckets[0],
r_update,
"update",
0,
batch_size=self.batch_size,
process_concurrency=self.process_concurrency,
pause_secs=5,
timeout_secs=self.sdk_timeout)
])
else:
data_validation.append(
self.task.async_validate_docs(
self.cluster,
self.bucket_util.buckets[0],
r_update,
"update",
0,
batch_size=self.batch_size,
process_concurrency=self.process_concurrency,
pause_secs=5,
timeout_secs=self.sdk_timeout))
for task in data_validation:
self.task.jython_task_manager.get_task_result(task)
disk_usage = self.get_disk_usage(
self.bucket_util.get_all_buckets()[0], self.servers)
_res = disk_usage[0] - disk_usage[1]
self.log.info("disk usage after update count {}\
is {}".format(count + 1, _res))
self.assertIs(
_res > 4 * self.disk_usage[self.disk_usage.keys()[0]],
False, "Disk Usage {} After \
Update Count {} exceeds \
Actual disk usage {} by four \
times".format(_res, count,
self.disk_usage[self.disk_usage.keys()[0]]))
count += 1
self.update_itr += self.update_itr
start_del = 0
end_del = self.num_items // 2
if self.rev_del:
start_del = -int(self.num_items // 2 - 1)
end_del = 1
self.gen_delete = doc_generator(
self.key,
start_del,
end_del,
doc_size=self.doc_size,
doc_type=self.doc_type,
target_vbucket=self.target_vbucket,
vbuckets=self.cluster_util.vbuckets,
key_size=self.key_size,
randomize_doc_size=self.randomize_doc_size,
randomize_value=self.randomize_value,
mix_key_size=self.mix_key_size,
deep_copy=self.deep_copy)
self.log.info("Deleting num_items//2 docs")
self.doc_ops = "delete"
_ = self.loadgen_docs(self.retry_exceptions,
self.ignore_exceptions,
_sync=True)
self.bucket_util._wait_for_stats_all_buckets()
self.bucket_util.verify_stats_all_buckets(self.num_items)
disk_usage = self.get_disk_usage(
self.bucket_util.get_all_buckets()[0], self.servers)
_res = disk_usage[0] - disk_usage[1]
self.log.info("disk usage after delete is {}".format(_res))
self.assertIs(
_res > 4 * self.disk_usage[self.disk_usage.keys()[0]], False,
"Disk Usage {} After \
Delete count {} exceeds Actual \
disk usage {} by four \
times".format(_res, i + 1,
self.disk_usage[self.disk_usage.keys()[0]]))
self.gen_create = copy.deepcopy(self.gen_delete)
self.log.info("Recreating num_items//2 docs")
self.doc_ops = "create"
_ = self.loadgen_docs(self.retry_exceptions,
self.ignore_exceptions,
_sync=True)
self.bucket_util._wait_for_stats_all_buckets()
self.bucket_util.verify_stats_all_buckets(self.num_items)
d_validation = self.task.async_validate_docs(
self.cluster,
self.bucket_util.buckets[0],
self.gen_create,
"create",
0,
batch_size=self.batch_size,
process_concurrency=self.process_concurrency,
pause_secs=5,
timeout_secs=self.sdk_timeout)
self.task.jython_task_manager.get_task_result(d_validation)
disk_usage = self.get_disk_usage(
self.bucket_util.get_all_buckets()[0], self.servers)
_res = disk_usage[0] - disk_usage[1]
self.log.info("disk usage after new create \
is {}".format(_res))
self.assertIs(
_res > 4 * self.disk_usage[self.disk_usage.keys()[0]], False,
"Disk Usage {} After \
new Creates count {} exceeds \
Actual disk usage {} by four \
times".format(_res, i + 1,
self.disk_usage[self.disk_usage.keys()[0]]))
self.log.info("====test_update_rev_update ends====")
def test_update_single_doc_n_times(self):
"""
Update same document n times, where n is number which
gets derived from given fragmentation value and after
updates check for space amplification and data
validation
"""
count = 0
self.assertIs((self.fragmentation <= 0 or self.fragmentation >= 100),
False,
msg="Fragmentation value can't be <=0 or >=100")
update_count = int(
math.ceil(
float(self.fragmentation * self.num_items) /
(100 - self.fragmentation)))
self.log.info("{} is the count with which doc will be updated \
".format(update_count))
self.doc_ops = "update"
self.client = SDKClient([self.cluster.master],
self.bucket_util.buckets[0],
scope=CbServer.default_scope,
collection=CbServer.default_collection)
self.gen_update = doc_generator(
self.key,
0,
1,
doc_size=self.doc_size,
doc_type=self.doc_type,
target_vbucket=self.target_vbucket,
vbuckets=self.cluster_util.vbuckets,
key_size=self.key_size,
mutate=count,
randomize_doc_size=self.randomize_doc_size,
randomize_value=self.randomize_value,
mix_key_size=self.mix_key_size,
deep_copy=self.deep_copy)
key, val = self.gen_update.next()
for node in self.cluster.nodes_in_cluster:
shell = RemoteMachineShellConnection(node)
shell.kill_memcached()
shell.disconnect()
self.assertTrue(
self.bucket_util._wait_warmup_completed(
[self.cluster_util.cluster.master],
self.bucket_util.buckets[0],
wait_time=self.wait_timeout * 10))
while count < (update_count + 1):
self.log.debug("Update Iteration count == {}".format(count))
val.put("mutated", count + 1)
self.client.upsert(key, val)
count += 1
self.bucket_util._wait_for_stats_all_buckets()
disk_usage = self.get_disk_usage(self.bucket_util.get_all_buckets()[0],
self.servers)
_res = disk_usage[0]
self.log.info("After all updates disk usage is {}MB\
".format(_res))
usage_factor = (
(float(self.num_items + update_count) / self.num_items) + 0.5)
self.log.debug("Disk usage factor is {}".format(usage_factor))
self.assertIs(
_res > usage_factor * self.disk_usage[self.disk_usage.keys()[0]],
False, "Disk Usage {}MB After all Updates'\n' \
exceeds Actual'\n' \
disk usage {}MB by {}'\n' \
times".format(_res, self.disk_usage[self.disk_usage.keys()[0]],
usage_factor))
data_validation = self.task.async_validate_docs(
self.cluster,
self.bucket_util.buckets[0],
self.gen_update,
"update",
0,
batch_size=self.batch_size,
process_concurrency=self.process_concurrency,
pause_secs=5,
timeout_secs=self.sdk_timeout)
self.task.jython_task_manager.get_task_result(data_validation)
self.enable_disable_swap_space(self.servers, disable=False)
self.log.info("====test_update_single_doc_n_times====")
def test_read_docs_using_multithreads(self):
"""
Read same docs together using multithreads.
"""
self.log.info("Reading docs parallelly using multi threading")
tasks_info = dict()
update_doc_count = int(
math.ceil(
float(self.fragmentation * self.num_items) /
(100 - self.fragmentation)))
self.log.info("Count of docs to be updated is {}\
".format(update_doc_count))
num_update = list()
while update_doc_count > self.num_items:
num_update.append(self.num_items)
update_doc_count -= self.num_items
if update_doc_count > 0:
num_update.append(update_doc_count)
for itr in num_update:
self.doc_ops = "update"
start = 0
end = itr
self.gen_update = doc_generator(
self.key,
start,
end,
doc_size=self.doc_size,
doc_type=self.doc_type,
target_vbucket=self.target_vbucket,
vbuckets=self.cluster_util.vbuckets,
key_size=self.key_size,
mutate=0,
randomize_doc_size=self.randomize_doc_size,
randomize_value=self.randomize_value,
mix_key_size=self.mix_key_size,
deep_copy=self.deep_copy)
update_task_info = self.loadgen_docs(self.retry_exceptions,
self.ignore_exceptions,
_sync=False)
tasks_info.update(update_task_info.items())
count = 0
self.doc_ops = "read"
# if self.next_half is true then one thread will read
# in ascending order and other in descending order
if self.next_half:
start = -int(self.num_items - 1)
end = 1
g_read = doc_generator(self.key,
start,
end,
doc_size=self.doc_size,
doc_type=self.doc_type,
target_vbucket=self.target_vbucket,
vbuckets=self.cluster_util.vbuckets,
key_size=self.key_size,
randomize_doc_size=self.randomize_doc_size,
randomize_value=self.randomize_value,
mix_key_size=self.mix_key_size,
deep_copy=self.deep_copy)
for node in self.cluster.nodes_in_cluster:
shell = RemoteMachineShellConnection(node)
shell.kill_memcached()
shell.disconnect()
while count < self.read_thread_count:
read_task_info = self.loadgen_docs(self.retry_exceptions,
self.ignore_exceptions,
_sync=False)
tasks_info.update(read_task_info.items())
count += 1
if self.next_half and count < self.read_thread_count:
read_tasks_info = self.bucket_util._async_validate_docs(
self.cluster,
g_read,
"read",
0,
batch_size=self.batch_size,
process_concurrency=self.process_concurrency,
pause_secs=5,
timeout_secs=self.sdk_timeout,
retry_exceptions=self.retry_exceptions,
ignore_exceptions=self.ignore_exceptions)
tasks_info.update(read_task_info.items())
count += 1
for task in tasks_info:
self.task_manager.get_task_result(task)
self.log.info("Waiting for ep-queues to get drained")
self.bucket_util._wait_for_stats_all_buckets()
self.log.info("test_read_docs_using_multithreads ends")
def test_basic_create_delete(self):
"""
CREATE(n)-> DELETE(n)->DISK_USAGE_CHECK
REPEAT ABove test_itr_times
"""
self.log.info("Cretaing and Deletes docs n times ")
count = 0
start = 0
end = self.num_items
self.gen_delete = doc_generator(
self.key,
start,
end,
doc_size=self.doc_size,
doc_type=self.doc_type,
target_vbucket=self.target_vbucket,
vbuckets=self.cluster_util.vbuckets,
key_size=self.key_size,
randomize_doc_size=self.randomize_doc_size,
randomize_value=self.randomize_value,
mix_key_size=self.mix_key_size,
deep_copy=self.deep_copy)
while count < self.test_itr:
self.doc_ops = "delete"
_ = self.loadgen_docs(self.retry_exceptions,
self.ignore_exceptions,
_sync=True)
self.log.info("Verifying doc counts after delete doc_ops")
self.bucket_util._wait_for_stats_all_buckets()
self.bucket_util.verify_stats_all_buckets(self.num_items)
disk_usage = self.get_disk_usage(
self.bucket_util.get_all_buckets()[0], self.servers)
_res = disk_usage[0]
self.log.info("disk usage after delete count {} \
is {}MB".format(count + 1, _res))
self.assertIs(
_res > 2.5 * self.disk_usage[self.disk_usage.keys()[0]], False,
"Disk Usage {}MB After \
delete count {} exceeds \
Actual disk usage {}MB by \
2.5 times".format(_res, count + 1,
self.disk_usage[self.disk_usage.keys()[0]]))
self.doc_ops = "create"
_ = self.loadgen_docs(self.retry_exceptions,
self.ignore_exceptions,
_sync=True)
self.bucket_util._wait_for_stats_all_buckets()
self.bucket_util.verify_stats_all_buckets(self.num_items)
count += 1
self.log.info("====test_basic_create_delete ends====")
class BasicUpsertTests(BasicCrudTests):
def test_update_n_times(self):
"""
Test Focus: Update items n times and
test space amplification
STEPS:
-- Update items n times (where n gets calculated
from fragmentation value
-- Check space amplification
-- Repeat the above steps n times
-- After all iterations validate the data
"""
self.log.info("test_update_n_times starts")
upsert_doc_list = self.get_fragmentation_upsert_docs_list()
self.mutate = 0
count = 0
while count < self.test_itr:
self.log.info("Iteration == {}".format(count+1))
#######################################################################
'''
STEP - 1, Update Items
'''
for itr in upsert_doc_list:
self.doc_ops = "update"
self.update_start = 0
self.update_end = itr
if self.rev_update:
self.update_start = -int(itr - 1)
self.update_end = 1
self.generate_docs(doc_ops="update")
_ = self.loadgen_docs(self.retry_exceptions,
self.ignore_exceptions,
_sync=True)
self.log.info("Waiting for ep-queues to get drained")
self.bucket_util._wait_for_stats_all_buckets(timeout=3600)
#######################################################################
'''
STEP - 2, Space Amplification Check
'''
msg = "Fragmentation value for {} stats exceeds\
the configured value"
_result = self.check_fragmentation_using_magma_stats(
self.buckets[0],
self.cluster.nodes_in_cluster)
self.assertIs(_result, True,
msg.format("magma"))
_r = self.check_fragmentation_using_bucket_stats(
self.buckets[0], self.cluster.nodes_in_cluster)
self.assertIs(_r, True,
msg.format("KV"))
usage_factor = ((float(
self.num_items + sum(upsert_doc_list)
) / self.num_items) + 0.5)
self.log.debug("Disk usage factor = {}".format(usage_factor))
time_end = time.time() + 60 * 2
while time.time() < time_end:
disk_usage = self.get_disk_usage(self.buckets[0],
self.cluster.nodes_in_cluster)
_res = disk_usage[0]
self.log.debug("usage at time {} is {}".format((time_end - time.time()), _res))
if _res < usage_factor * self.disk_usage[self.disk_usage.keys()[0]]:
break
msg = "Iteration= {}, Disk Usage = {}MB\
exceeds {} times from Actual disk usage = {}MB"
self.assertIs(_res > usage_factor * self.disk_usage[
self.disk_usage.keys()[0]],
False, msg.format(count+1, _res, usage_factor,
self.disk_usage[self.disk_usage.keys()[0]]))
count += 1
#######################################################################
'''
STEP - 3, Data Validation
'''
self.validate_data("update", self.gen_update)
self.change_swap_space(self.cluster.nodes_in_cluster,
disable=False)
self.log.info("====test_update_n_times ends====")
def test_multi_update_delete(self):
"""
STEPS:
-- Update items x times
-- Check space amplification
-- Delete half of the items
-- Check space Amplification
-- Recreate deleted items
-- Check Space Amplification
-- Repeat above steps for n times
-- After all iterations validate the data
"""
self.log.info("==== test_multi_update_delete starts =====")
count = 0
msg_stats = "Fragmentation value for {} stats exceeds\
the configured value"
msg = "{} Iteration= {}, Disk Usage = {}MB\
exceeds 2.5 times from Actual disk usage = {}MB"
self.mutate = 0
for i in range(self.test_itr):
self.log.info("Step 1, Iteration= {}".format(i+1))
#######################################################################
'''
STEP - 1, Update Items, update_itr times
'''
while count < self.update_itr:
self.doc_ops = "update"
self.update_start = 0
self.update_end = self.num_items
if self.rev_update:
self.update_start = -int(self.num_items - 1)
self.update_end = 1
self.generate_docs(doc_ops="update")
_ = self.loadgen_docs(self.retry_exceptions,
self.ignore_exceptions,
_sync=True)
self.log.info("Waiting for ep-queues to get drained")
self.bucket_util._wait_for_stats_all_buckets(timeout=3600)
###################################################################
'''
STEP - 2
-- Space Amplification check after each update iteration.
-- Data validation only for last update iteration
'''
self.log.info("Step 2, Iteration= {}".format(i+1))
_result = self.check_fragmentation_using_magma_stats(
self.buckets[0],
self.cluster.nodes_in_cluster)
self.assertIs(_result, True,
msg_stats.format("magma"))
_r = self.check_fragmentation_using_bucket_stats(
self.buckets[0], self.cluster.nodes_in_cluster)
self.assertIs(_r, True,
msg_stats.format("KV"))
time_end = time.time() + 60 * 2
while time.time() < time_end:
disk_usage = self.get_disk_usage(self.buckets[0],
self.cluster.nodes_in_cluster)
_res = disk_usage[0]
self.log.info("Update Iteration-{}, Disk Usage at time {} is {}MB \
".format(count+1, time_end - time.time(), _res))
if _res < 2.5 * self.disk_usage[self.disk_usage.keys()[0]]:
break
self.assertIs(
_res > 2.5 * self.disk_usage[self.disk_usage.keys()[0]],
False, msg.format("update", count+1, _res,
self.disk_usage[self.disk_usage.keys()[0]]))
count += 1
self.update_itr += self.update_itr
if i+1 == self.test_itr:
self.validate_data("update", self.gen_update)
###################################################################
'''
STEP - 3
-- Delete half of the docs.
'''
self.log.debug("Step 3, Iteration {}".format(i+1))
self.doc_ops = "delete"
self.delete_start = 0
self.delete_end = self.num_items//2
if self.rev_del:
self.delete_start = -int(self.num_items//2 - 1)
self.delete_end = 1
self.generate_docs(doc_ops="delete")
_ = self.loadgen_docs(self.retry_exceptions,
self.ignore_exceptions,
_sync=True)
self.bucket_util._wait_for_stats_all_buckets(timeout=3600)
self.bucket_util.verify_stats_all_buckets(self.num_items)
###################################################################
'''
STEP - 4
-- Space Amplification Check after deletion.
'''
self.log.debug("Step 4, Iteration {}".format(i+1))
_result = self.check_fragmentation_using_magma_stats(
self.buckets[0],
self.cluster.nodes_in_cluster)
self.assertIs(_result, True,
msg_stats.format("magma"))
_r = self.check_fragmentation_using_bucket_stats(
self.buckets[0], self.cluster.nodes_in_cluster)
self.assertIs(_r, True,
msg_stats.format("KV"))
disk_usage = self.get_disk_usage(self.buckets[0],
self.cluster.nodes_in_cluster)
_res = disk_usage[0]
self.log.info("Delete Iteration {}, Disk Usage- {}MB\
".format(i+1, _res))
self.assertIs(
_res > 2.5 * self.disk_usage[
self.disk_usage.keys()[0]],
False, msg.format(
"delete", i+1, _res,
self.disk_usage[self.disk_usage.keys()[0]]))
###################################################################
'''
STEP - 5
-- ReCreation of docs.
'''
self.log.debug("Step 5, Iteration= {}".format(i+1))
self.gen_create = copy.deepcopy(self.gen_delete)
self.doc_ops = "create"
_ = self.loadgen_docs(self.retry_exceptions,
self.ignore_exceptions,
_sync=True)
self.bucket_util._wait_for_stats_all_buckets(timeout=3600)
self.bucket_util.verify_stats_all_buckets(self.num_items)
###################################################################
'''
STEP - 6
-- Space Amplification Check after Recreation.
'''
self.log.debug("Step 6, Iteration= {}".format(i+1))
_result = self.check_fragmentation_using_magma_stats(
self.buckets[0],
self.cluster.nodes_in_cluster)
self.assertIs(_result, True,
msg_stats.format("magma"))
_r = self.check_fragmentation_using_bucket_stats(
self.buckets[0], self.cluster.nodes_in_cluster)
self.assertIs(_r, True,
msg_stats.format("KV"))
disk_usage = self.get_disk_usage(self.buckets[0],
self.cluster.nodes_in_cluster)
_res = disk_usage[0]
self.log.info("Create Iteration{}, Disk Usage= {}MB \
".format(i+1, _res))
self.assertIs(_res > 2.5 * self.disk_usage[
self.disk_usage.keys()[0]],
False, msg.format("Create", _res, i+1,
self.disk_usage[self.disk_usage.keys()[0]]))
###################################################################
'''
STEP - 7
-- Validate data
-- Data validation is only for the creates in last iterations.
'''
self.log.debug("Step 7, Iteration= {}".format(i+1))
self.validate_data("create", self.gen_create)
self.log.info("====test_multiUpdate_delete ends====")
def test_update_rev_update(self):
"""
STEPS:
-- Update num_items // 2 items.
-- Reverse update remaining num_items // 2 items.
-- If next.half is false skip above step
-- And reverse update items in first point
-- Check space amplification
-- Repeat above steps x times
-- Delete all the items
-- Check space Amplification
-- Recreate deleted items
-- Check Space Amplification
-- Repeat above steps for n times
-- After all iterations validate the data
"""
self.log.info("==== test_update_rev_update starts =====")
msg_stats = "Fragmentation value for {} stats exceeds\
the configured value"
msg = "{} Iteration= {}, Disk Usage = {}MB\
exceeds {} times from Actual disk usage = {}MB"
count = 0
mutated = 1
for i in range(self.test_itr):
self.log.debug("Step 1, Iteration= {}".format(i+1))
#######################################################################
'''
STEP - 1, Update Items, update_itr times
-- Update n // 2 items
-- If self.next_half is true
-- Update remaining n//2 items
-- Else, again update items in
reverse order in first point
'''
while count < self.update_itr:
tasks_info = dict()
self.doc_ops = "update"
self.gen_update = self.genrate_docs_basic(0, self.num_items //2,
mutate=mutated)
tem_tasks_info = self.loadgen_docs(self.retry_exceptions,
self.ignore_exceptions,
_sync=False)
tasks_info.update(tem_tasks_info.items())
if self.next_half:
start = - (self.num_items - 1)
end = - (self.num_items // 2 - 1)
self.gen_update = self.genrate_docs_basic(start, end,
mutate=mutated)
tem_tasks_info = self.loadgen_docs(self.retry_exceptions,
self.ignore_exceptions,
_sync=False)
tasks_info.update(tem_tasks_info.items())
for task in tasks_info:
self.task_manager.get_task_result(task)
self.bucket_util.verify_doc_op_task_exceptions(
tasks_info, self.cluster)
self.bucket_util.log_doc_ops_task_failures(tasks_info)
mutated += 1
if not self.next_half:
start = - (self.num_items - 1)
end = - (self.num_items // 2 - 1)
self.gen_update = self.genrate_docs_basic(start, end,
mutate=mutated)
_ = self.loadgen_docs(self.retry_exceptions,
self.ignore_exceptions,
_sync=True)
mutated += 1
self.log.info("Waiting for ep-queues to get drained")
self.bucket_util._wait_for_stats_all_buckets(timeout=3600)
###################################################################
'''
STEP - 2
-- Space Amplification check after each update iteration.
'''
self.log.debug("Step 2, Iteration= {}".format(i+1))
_result = self.check_fragmentation_using_magma_stats(
self.buckets[0],
self.cluster.nodes_in_cluster)
self.assertIs(_result, True,
msg_stats.format("magma"))
_r = self.check_fragmentation_using_bucket_stats(
self.buckets[0], self.cluster.nodes_in_cluster)
self.assertIs(_r, True,
msg_stats.format("KV"))
disk_usage = self.get_disk_usage(
self.buckets[0], self.cluster.nodes_in_cluster)
_res = disk_usage[0]
self.log.info("Update Iteration- {}, Disk Usage- {}MB\
".format(count+1, _res))
self.assertIs(
_res > 2.5 * self.disk_usage[self.disk_usage.keys()[0]],
False, msg.format("update", count+1, _res, 2.5,
self.disk_usage[self.disk_usage.keys()[0]]))
count += 1
self.update_itr += self.update_itr
###################################################################
'''
STEP - 3
-- Delete all the items.
'''
self.log.debug("Step 3, Iteration {}".format(i+1))
self.doc_ops = "delete"
self.delete_start = 0
self.delete_end = self.num_items
if self.rev_del:
self.delete_start = -int(self.num_items - 1)
self.delete_end = 1
self.generate_docs(doc_ops="delete")
_ = self.loadgen_docs(self.retry_exceptions,
self.ignore_exceptions,
_sync=True)
self.bucket_util._wait_for_stats_all_buckets(timeout=3600)
self.bucket_util.verify_stats_all_buckets(self.num_items)
###################################################################
'''
STEP - 4
-- Space Amplification Check after deletion.
'''
self.log.debug("Step 4, Iteration {}".format(i+1))
_result = self.check_fragmentation_using_magma_stats(
self.buckets[0],
self.cluster.nodes_in_cluster)
self.assertIs(_result, True,
msg_stats.format("magma"))
_r = self.check_fragmentation_using_bucket_stats(
self.buckets[0], self.cluster.nodes_in_cluster)
self.assertIs(_r, True,
msg_stats.format("KV"))
disk_usage = self.get_disk_usage(self.buckets[0],
self.cluster.nodes_in_cluster)
_res = disk_usage[0]
self.log.info("Delete Iteration {}, Disk Usage- {}MB\
".format(i+1, _res))
self.assertIs(
_res > 0.5 * self.disk_usage[
self.disk_usage.keys()[0]],
False, msg.format(
"delete", i+1, _res, 0.5,
self.disk_usage[self.disk_usage.keys()[0]]))
###################################################################
'''
STEP - 5
-- ReCreation of docs.
'''
self.log.debug("Step 5, Iteration= {}".format(i+1))
self.gen_create = copy.deepcopy(self.gen_delete)
self.doc_ops = "create"
_ = self.loadgen_docs(self.retry_exceptions,
self.ignore_exceptions,
_sync=True)
self.bucket_util._wait_for_stats_all_buckets(timeout=3600)
self.bucket_util.verify_stats_all_buckets(self.num_items)
###################################################################
'''
STEP - 6
-- Space Amplification Check after Recreation.
'''
self.log.debug("Step 6, Iteration= {}".format(i+1))
_result = self.check_fragmentation_using_magma_stats(
self.buckets[0],
self.cluster.nodes_in_cluster)
self.assertIs(_result, True,
msg_stats.format("magma"))
_r = self.check_fragmentation_using_bucket_stats(
self.buckets[0], self.cluster.nodes_in_cluster)
self.assertIs(_r, True,
msg_stats.format("KV"))
disk_usage = self.get_disk_usage(self.buckets[0],
self.cluster.nodes_in_cluster)
_res = disk_usage[0]
self.log.info("Create Iteration{}, Disk Usage= {}MB \
".format(i+1, _res))
self.assertIs(_res > 1.5 * self.disk_usage[
self.disk_usage.keys()[0]],
False, msg.format("Create", _res, i+1, 1.5,
self.disk_usage[self.disk_usage.keys()[0]]))
###################################################################
'''
STEP - 7
-- Validate data
-- Data validation is only for the creates in last iterations.
'''
self.log.debug("Step 7, Iteration= {}".format(i+1))
self.validate_data("create", self.gen_create)
self.log.info("====test_update_rev_update ends====")
def test_update_single_doc_n_times(self):
"""
Test Focus: Update single/same doc n times
Note: MultiThreading is used to update
single doc, since we are not worried
about what should be the final mutate
value of the document semaphores have
been avoided. MultiThreading also speed up
the execution of test
"""
self.log.info("test_update_single_doc_n_times starts")
self.doc_ops = "update"
self.client = SDKClient([self.cluster.master],
self.bucket_util.buckets[0],
scope=CbServer.default_scope,
collection=CbServer.default_collection)
self.gen_update = self.genrate_docs_basic(start=0, end=1)
key, val = self.gen_update.next()
for node in self.cluster.nodes_in_cluster:
shell = RemoteMachineShellConnection(node)
shell.restart_couchbase()
shell.disconnect()
self.assertTrue(
self.bucket_util._wait_warmup_completed(
[self.cluster_util.cluster.master],
self.bucket_util.buckets[0],
wait_time=self.wait_timeout * 10))
def upsert_doc(start_num, end_num, key_obj, val_obj):
for i in range(start_num, end_num):
val_obj.put("mutated", i)
self.client.upsert(key_obj, val_obj)
threads = []
start = 0
end = 0
for _ in range(10):
start = end
end += 100000
th = threading.Thread(
target=upsert_doc, args=[start, end, key, val])
th.start()
threads.append(th)
for th in threads:
th.join()
self.bucket_util._wait_for_stats_all_buckets(timeout=3600)
# Space amplification check
msg_stats = "Fragmentation value for {} stats exceeds\
the configured value"
_result = self.check_fragmentation_using_magma_stats(
self.buckets[0],
self.cluster.nodes_in_cluster)
self.assertIs(_result, True,
msg_stats.format("magma"))
_r = self.check_fragmentation_using_bucket_stats(
self.buckets[0], self.cluster.nodes_in_cluster)
self.assertIs(_r, True,
msg_stats.format("KV"))
disk_usage = self.get_disk_usage(
self.buckets[0],
self.cluster.nodes_in_cluster)
self.log.debug("Disk usage after updates {}".format(
disk_usage))
_res = disk_usage[0]
msg = "Disk Usage = {}MB exceeds 2.2 times \
from Actual disk usage = {}MB"
self.assertIs(
_res > 2.2 * self.disk_usage[
self.disk_usage.keys()[0]],
False,
msg.format(_res, self.disk_usage[self.disk_usage.keys()[0]]))
# Space amplification check ends
success, fail = self.client.get_multi([key],
self.wait_timeout)
self.assertIs(key in success, True,
msg="key {} doesn't exist\
".format(key))
actual_val = dict()
expected_val = Json.loads(val.toString())
actual_val = Json.loads(success[key][
'value'].toString())
self.log.debug("Expected_val= {} and actual_val = {}\
".format(expected_val, actual_val))
self.assertIs(expected_val == actual_val, True,
msg="expected_val-{} != Actual_val-{}\
".format(expected_val, actual_val))
self.change_swap_space(self.cluster.nodes_in_cluster,
disable=False)
self.log.info("====test_update_single_doc_n_times ends====")
def test_move_val_btwn_key_and_seq_trees(self):
"""
Test Focus: Update items such that values moves
Sequence Tree and Key Trees.
STEPS:
-- Update items with new size , so that
items move from sequence tree to key
tree or vice versa
-- Do data validation
-- Again update items with initial size
-- Check space amplification
-- Again validate documents
"""
self.log.info("test_move_val_btwn_key_and_seq_trees starts")
msg_stats = "Fragmentation value for {} stats exceeds\
the configured value"
count = 0
keyTree, seqTree = (self.get_disk_usage(
self.buckets[0],
self.cluster.nodes_in_cluster)[2:4])
self.log.debug("Disk usage after pure creates {}".format((
self.disk_usage, keyTree, seqTree)))
initial_doc_size = self.doc_size
upsert_size = 0
if self.doc_size < 32:
upsert_size = 2048
while count < self.test_itr:
self.log.info("Update Iteration count == {}".format(count))
for node in self.cluster.nodes_in_cluster:
shell = RemoteMachineShellConnection(node)
shell.kill_memcached()
shell.disconnect()
self.assertTrue(self.bucket_util._wait_warmup_completed(
[self.cluster_util.cluster.master],
self.bucket_util.buckets[0],
wait_time=self.wait_timeout * 10))
#######################################################################
'''
STEP - 1, Update items with changed/new size
'''
self.log.info("Step 1, Iteration= {}".format(count+1))
self.doc_ops = "update"
self.update_start = 0
self.update_end = self.num_items
if self.rev_update:
self.update_start = -int(self.num_items - 1)
self.update_end = 1
self.doc_size = upsert_size
self.generate_docs()
_ = self.loadgen_docs(self.retry_exceptions,
self.ignore_exceptions,
_sync=True)
self.bucket_util._wait_for_stats_all_buckets(timeout=3600)
if upsert_size > 32:
seqTree_update = (self.get_disk_usage(
self.buckets[0],
self.cluster.nodes_in_cluster)[-1])
self.log.info("For upsert_size > 32 seqIndex usage-{}\
".format(seqTree_update))
#######################################################################
'''
STEP - 2, Validate data after initial upsert
'''
self.log.info("Step 2, Iteration= {}".format(count+1))
self.validate_data("update", self.gen_update)
#######################################################################
'''
STEP - 3, Updating items with changed doc size
to move between tress
'''
self.log.info("Step 3, Iteration= {}".format(count+1))
self.doc_size = initial_doc_size
self.generate_docs()
_ = self.loadgen_docs(self.retry_exceptions,
self.ignore_exceptions,
_sync=True)
self.bucket_util._wait_for_stats_all_buckets(timeout=3600)
#######################################################################
'''
STEP - 4, Space Amplification Checks
'''
_result = self.check_fragmentation_using_magma_stats(
self.buckets[0],
self.cluster.nodes_in_cluster)
self.assertIs(_result, True,
msg_stats.format("magma"))
_r = self.check_fragmentation_using_bucket_stats(
self.buckets[0], self.cluster.nodes_in_cluster)
self.assertIs(_r, True,
msg_stats.format("KV"))
disk_usage = self.get_disk_usage(
self.buckets[0], self.cluster.nodes_in_cluster)
_res = disk_usage[0]
self.log.info("disk usage after upsert count {} is {}MB \
".format(count+1, _res))
if self.doc_size > 32:
self.assertIs(
_res > 1.5 * self.disk_usage[self.disk_usage.keys()[0]],
False, "Disk Usage {} After \
update count {} exceeds \
Actual disk usage {} by 1.5 \
times".format(_res, count+1,
self.disk_usage[self.disk_usage.keys()[0]]))
else:
self.assertIs(disk_usage[3] > 0.5 * seqTree_update,
False, " Current seqTree usage-{} exceeds by'\n'\
0.5 times from the earlier '\n' \
seqTree usage (after update) -{} \
".format(disk_usage[3], seqTree_update))
count += 1
#######################################################################
'''
STEP - 5, Data validation
'''
self.log.info("Step 5, Iteration= {}".format(count+1))
self.validate_data("update", self.gen_update)
#######################################################################
self.change_swap_space(self.cluster.nodes_in_cluster, disable=False)
self.log.info("====test_move_docs_btwn_key_and_seq_trees ends====")
def test_parallel_create_update(self):
"""
STEPS:
-- Create new items and update already
existing items
-- Check disk_usage after each Iteration
-- Data validation for last iteration
"""
self.log.info("test_parallel_create_update starts")
count = 0
init_items = self.num_items
self.doc_ops = "create:update"
self.update_start = 0
self.update_end = self.num_items
while count < self.test_itr:
self.log.info("Iteration {}".format(count+1))
self.create_start = self.num_items
self.create_end = self.num_items+init_items
if self.rev_write:
self.create_start = -int(self.num_items+init_items - 1)
self.create_end = -int(self.num_items - 1)
self.generate_docs()
_ = self.loadgen_docs(self.retry_exceptions,
self.ignore_exceptions,
_sync=True)
self.bucket_util._wait_for_stats_all_buckets(timeout=3600)
self.bucket_util.verify_stats_all_buckets(self.num_items)
if count == self.test_itr - 1:
self.validate_data("update", self.gen_update)
self.update_start = self.num_items
self.update_end = self.num_items+init_items
if self.rev_update:
self.update_start = -int(self.num_items+init_items - 1)
self.update_end = -int(self.num_items - 1)
disk_usage = self.get_disk_usage(
self.buckets[0],
self.cluster.nodes_in_cluster)
if self.doc_size <= 32:
self.assertIs(
disk_usage[2] >= disk_usage[3], True,
"seqIndex usage = {}MB'\n' \
after Iteration {}'\n' \
exceeds keyIndex usage={}MB'\n' \
".format(disk_usage[3],
count+1,
disk_usage[2]))
self.assertIs(
disk_usage[0] > 2.2 * (2 * self.disk_usage[
self.disk_usage.keys()[0]]),
False, "Disk Usage {}MB After '\n\'\
Updates exceeds '\n\'\
Actual disk usage {}MB by '\n'\
2.2 times".format(disk_usage[0],
(2 * self.disk_usage[
self.disk_usage.keys()[0]])))
count += 1
self.change_swap_space(self.cluster.nodes_in_cluster, disable=False)
self.log.info("====test_parallel_create_update ends====")
class MagmaFlushBucketTests(MagmaBaseTest):
def setUp(self):
super(MagmaFlushBucketTests, self).setUp()
self.sdk_timeout = self.input.param("sdk_timeout", 100)
self.assertTrue(self.rest.update_autofailover_settings(False, 600),
"AutoFailover disabling failed")
self.sigkill = self.input.param("sigkill", False)
self.multiplier = self.input.param("multiplier", 2)
self.flush_th = None
self.shell_conn = list()
for node in self.cluster.nodes_in_cluster:
shell = RemoteMachineShellConnection(node)
self.shell_conn.append(shell)
def tearDown(self):
self.stop_flush = True
if self.flush_th and self.flush_th.is_alive():
self.flush_th.join()
super(MagmaFlushBucketTests, self).tearDown()
def bucket_flush(self, sigkill=False, wait=False):
self.stop_flush = False
flush_iteration = 1
while not self.stop_flush:
for bucket in self.buckets:
result = self.bucket_util.flush_bucket(self.cluster, bucket)
if sigkill and result:
for shell in self.shell_conn:
shell.kill_memcached()
if wait:
for node in self.cluster.nodes_in_cluster:
if "kv" in node.services:
result = self.bucket_util._wait_warmup_completed(
[node],
self.cluster.buckets[0],
wait_time=self.wait_timeout * 5)
if not result:
msg = "warm-up couldn't complete in %s seconds" %\
(self.wait_timeout * 5)
self.log.critical(msg)
self.task.jython_task_manager.abort_all_tasks()
self.stop_flush = True
sleep = random.randint(30, 60)
self.sleep(sleep, "Iteration:{} done, waiting for {} sec after bucket flush".
format(flush_iteration, sleep))
flush_iteration += 1
def loadgen_docs_per_bucket(self, bucket,
retry_exceptions=[],
ignore_exceptions=[],
skip_read_on_error=False,
suppress_error_table=False,
scope=CbServer.default_scope,
collection=CbServer.default_collection,
_sync=True,
track_failures=True,
doc_ops=None):
doc_ops = doc_ops or self.doc_ops
tasks_info = dict()
tem_tasks_info = dict()
read_tasks_info = dict()
read_task = False
if self.check_temporary_failure_exception:
retry_exceptions.append(SDKException.TemporaryFailureException)
if "update" in doc_ops and self.gen_update is not None:
task = self.bucket_util.async_load_bucket(
self.cluster, bucket, self.gen_update, "update", 0,
batch_size=self.batch_size,
process_concurrency=self.process_concurrency,
persist_to=self.persist_to, replicate_to=self.replicate_to,
durability=self.durability_level,
sdk_timeout=self.sdk_timeout, retries=self.sdk_retries,
skip_read_on_error=skip_read_on_error,
suppress_error_table=suppress_error_table,
scope=scope,
collection=collection,
monitor_stats=self.monitor_stats,
track_failures=track_failures)
tem_tasks_info[task] = self.bucket_util.get_doc_op_info_dict(
bucket, "update", 0,
scope=scope,
collection=collection,
replicate_to=self.replicate_to,
persist_to=self.persist_to,
durability=self.durability_level,
timeout=self.sdk_timeout, time_unit="seconds",
ignore_exceptions=ignore_exceptions,
retry_exceptions=retry_exceptions)
tasks_info.update(tem_tasks_info.items())
if "create" in doc_ops and self.gen_create is not None:
task = self.bucket_util.async_load_bucket(
self.cluster, bucket, self.gen_create, "create", 0,
batch_size=self.batch_size,
process_concurrency=self.process_concurrency,
persist_to=self.persist_to, replicate_to=self.replicate_to,
durability=self.durability_level,
sdk_timeout=self.sdk_timeout, retries=self.sdk_retries,
skip_read_on_error=skip_read_on_error,
suppress_error_table=suppress_error_table,
scope=scope,
collection=collection,
monitor_stats=self.monitor_stats,
track_failures=track_failures)
tem_tasks_info[task] = self.bucket_util.get_doc_op_info_dict(
bucket, "create", 0,
scope=scope,
collection=collection,
replicate_to=self.replicate_to,
persist_to=self.persist_to,
durability=self.durability_level,
timeout=self.sdk_timeout, time_unit="seconds",
ignore_exceptions=ignore_exceptions,
retry_exceptions=retry_exceptions)
tasks_info.update(tem_tasks_info.items())
self.num_items += (self.gen_create.end - self.gen_create.start)
if "expiry" in doc_ops and self.gen_expiry is not None and self.maxttl:
task = self.bucket_util.async_load_bucket(
self.cluster, bucket, self.gen_expiry, "update", self.maxttl,
batch_size=self.batch_size,
process_concurrency=self.process_concurrency,
persist_to=self.persist_to, replicate_to=self.replicate_to,
durability=self.durability_level,
sdk_timeout=self.sdk_timeout, retries=self.sdk_retries,
skip_read_on_error=skip_read_on_error,
suppress_error_table=suppress_error_table,
scope=scope,
collection=collection,
monitor_stats=self.monitor_stats,
track_failures=track_failures)
tem_tasks_info[task] = self.bucket_util.get_doc_op_info_dict(
bucket, "update", 0,
scope=scope,
collection=collection,
replicate_to=self.replicate_to,
persist_to=self.persist_to,
durability=self.durability_level,
timeout=self.sdk_timeout, time_unit="seconds",
ignore_exceptions=ignore_exceptions,
retry_exceptions=retry_exceptions)
tasks_info.update(tem_tasks_info.items())
self.num_items -= (self.gen_expiry.end - self.gen_expiry.start)
if "read" in doc_ops and self.gen_read is not None:
read_tasks_info = self.bucket_util.async_validate_docs(
self.cluster, bucket, self.gen_read, "read", 0,
batch_size=self.batch_size,
process_concurrency=self.process_concurrency,
timeout_secs=self.sdk_timeout,
retry_exceptions=retry_exceptions,
ignore_exceptions=ignore_exceptions,
scope=scope,
collection=collection)
read_task = True
if "delete" in doc_ops and self.gen_delete is not None:
task = self.bucket_util.async_load_bucket(
self.cluster, bucket, self.gen_delete, "delete", 0,
batch_size=self.batch_size,
process_concurrency=self.process_concurrency,
persist_to=self.persist_to, replicate_to=self.replicate_to,
durability=self.durability_level,
sdk_timeout=self.sdk_timeout, retries=self.sdk_retries,
skip_read_on_error=skip_read_on_error,
suppress_error_table=suppress_error_table,
scope=scope,
collection=collection,
monitor_stats=self.monitor_stats,
track_failures=track_failures)
tem_tasks_info[task] = self.bucket_util.get_doc_op_info_dict(
bucket, "delete", 0,
scope=scope,
collection=collection,
replicate_to=self.replicate_to,
persist_to=self.persist_to,
durability=self.durability_level,
timeout=self.sdk_timeout, time_unit="seconds",
ignore_exceptions=ignore_exceptions,
retry_exceptions=retry_exceptions)
tasks_info.update(tem_tasks_info.items())
self.num_items -= (self.gen_delete.end - self.gen_delete.start)
if _sync:
for task in tasks_info:
self.task_manager.get_task_result(task)
self.bucket_util.verify_doc_op_task_exceptions(tasks_info,
self.cluster)
self.bucket_util.log_doc_ops_task_failures(tasks_info)
if read_task:
# TODO: Need to converge read_tasks_info into tasks_info before
# itself to avoid confusions during _sync=False case
tasks_info.update(read_tasks_info.items())
if _sync:
for task in read_tasks_info:
self.task_manager.get_task_result(task)
return tasks_info
def compute_docs(self, start, mem_only_items, doc_ops=None):
doc_ops = doc_ops or self.doc_ops
ops_len = len(self.doc_ops.split(":"))
if "create" in self.doc_ops:
self.create_start = start
self.create_end = mem_only_items
if ops_len == 1:
if "update" in self.doc_ops:
self.update_start = 0
self.update_end = mem_only_items
if "delete" in self.doc_ops:
self.delete_start = 0
self.delete_end = mem_only_items
if "expiry" in self.doc_ops:
self.expiry_start = 0
self.expiry_end = mem_only_items
elif ops_len == 2:
self.expiry_start = 0
self.expiry_end = mem_only_items
self.delete_start = start // 2
self.delete_end = mem_only_items
if "update" in self.doc_ops:
self.delete_start = 0
self.delete_end = mem_only_items
self.update_start = start // 2
self.update_end = mem_only_items
else:
self.expiry_start = 0
self.expiry_end = mem_only_items
self.delete_start = start // 3
self.delete_end = mem_only_items
self.update_start = (2 * start) // 3
self.update_end = mem_only_items
def test_flush_bucket_during_creates(self):
self.log.info("====test_flush_bucket_during_creates starts====")
self.create_start = self.init_items_per_collection
self.create_end = self.init_items_per_collection * self.multiplier
self.generate_docs(doc_ops=self.doc_ops, target_vbucket=None)
tasks_info = dict()
for scope in self.scopes:
for collection in self.collections:
task_info = self.loadgen_docs(
self.retry_exceptions,
self.ignore_exceptions,
scope=scope,
collection=collection,
suppress_error_table=True,
skip_read_on_error=True,
_sync=False,
doc_ops=self.doc_ops,
track_failures=False,
sdk_retry_strategy=SDKConstants.RetryStrategy.FAIL_FAST)
tasks_info.update(task_info.items())
self.flush_th = threading.Thread(target=self.bucket_flush,
kwargs=dict(sigkill=self.sigkill))
self.flush_th.start()
for task in tasks_info:
self.task_manager.get_task_result(task)
self.stop_flush = True
self.flush_th.join()
def test_create_fragmentation_before_flush_bucket(self):
self.log.info("====test_create_fragmentation_before_flush_bucket starts====")
self.doc_ops = "update"
self.num_threads = self.input.param("num_threads", 100)
count = 0
self.client = SDKClient([self.cluster.master],
self.cluster.buckets[0],
scope=CbServer.default_scope,
collection=CbServer.default_collection)
self.gen_update = self.genrate_docs_basic(start=0, end=1)
key, val = self.gen_update.next()
def upsert_doc(start_num, end_num, key_obj, val_obj):
print threading.currentThread().getName(), 'Starting'
for i in range(start_num, end_num):
val_obj.put("mutated", i)
self.client.upsert(key_obj, val_obj)
if threading.currentThread().getName() == 't'+str(self.num_threads):
self.bucket_util.flush_bucket(self.cluster, self.cluster.buckets[0])
print threading.currentThread().getName(), 'Exiting'
while count < self.test_itr:
self.log.info("Iteration : {}".format(count))
self.generate_docs(create_start =0, create_end =self.init_items_per_collection,
doc_ops="create")
_ = self.loadgen_docs(self.retry_exceptions,
self.ignore_exceptions,
doc_ops="create",
_sync=True)
threads = []
start = 0
end = 0
for i in range(self.num_threads+1):
start = end
end += self.init_items_per_collection
th = threading.Thread(name='t'+str(i),
target=upsert_doc, args=[start, end, key, val])
th.start()
threads.append(th)
for th in threads:
th.join()
count +=1
def test_flush_bucket_during_rollback(self):
'''
Test focus: Stopping persistence one by one on all nodes,
and trigger roll back on other nodes,
During rollback flush the data
Above step will be done num_rollback
(variable defined in test) times
STEPS:
-- Ensure creation of at least a single state file
-- Below steps will be repeated on all nodes, with stopping peristence on one at a time
-- Stop persistence on node x
-- Start load on node x for a given duration(self.duration * 60 seconds)
-- Above step ensures creation of new state files (# equal to self.duration)
-- Kill MemCached on Node x
-- Trigger roll back on other/replica nodes
-- ReStart persistence on Node -x
-- Repeat all the above steps for num_rollback times
'''
self.assertTrue(self.rest.update_autofailover_settings(False, 600),
"AutoFailover disabling failed")
items = copy.deepcopy(self.init_items_per_collection)
mem_only_items = self.input.param("rollback_items", 10000)
ops_len = len(self.doc_ops.split(":"))
if self.nodes_init < 2 or self.num_replicas < 1:
self.fail("Not enough nodes/replicas in the cluster/bucket \
to test rollback")
self.duration = self.input.param("duration", 2)
self.num_rollbacks = self.input.param("num_rollbacks", 3)
#######################################################################
'''
STEP - 1, Ensures creation of at least one snapshot
To ensure at least one snapshot should get created before rollback
starts, we need to sleep for 60 seconds as per magma design which
create state file every 60s
'''
self.sleep(60, "Ensures creation of at least one snapshot")
#######################################################################
'''
STEP - 2, Stop persistence on node - x
'''
for i in range(1, self.num_rollbacks+1):
self.log.info("Roll back Iteration == {}".format(i))
start = items
for x, node in enumerate(self.cluster.nodes_in_cluster):
shell = RemoteMachineShellConnection(node)
cbstats = Cbstats(shell)
self.target_vbucket = cbstats.vbucket_list(self.cluster.buckets[0].
name)
mem_item_count = 0
# Stopping persistence on Node-x
self.log.debug("Iteration == {}, Stopping persistence on Node-{}, ip ={}"
.format(i, x+1, node))
Cbepctl(shell).persistence(self.cluster.buckets[0].name, "stop")
###############################################################
'''
STEP - 3
-- Load documents on node x for self.duration * 60 seconds
-- This step ensures new state files (number equal to self.duration)
'''
self.compute_docs(start, mem_only_items)
self.gen_create = None
self.gen_update = None
self.gen_delete = None
self.gen_expiry = None
time_end = time.time() + 60 * self.duration
itr = 0
while time.time() < time_end:
itr += 1
time_start = time.time()
mem_item_count += mem_only_items * ops_len
self.generate_docs(doc_ops=self.doc_ops,
target_vbucket=self.target_vbucket)
self.loadgen_docs(_sync=True,
retry_exceptions=self.retry_exceptions)
if self.gen_create is not None:
self.create_start = self.gen_create.key_counter
if self.gen_update is not None:
self.update_start = self.gen_update.key_counter
if self.gen_delete is not None:
self.delete_start = self.gen_delete.key_counter
if self.gen_expiry is not None:
self.expiry_start = self.gen_expiry.key_counter
if time.time() < time_start + 60:
self.log.info("Rollback Iteration== {}, itr== {}, Active-Node=={}, Node=={}".format(i, itr, x+1, node))
self.sleep(time_start + 60 - time.time(),
"Sleep to ensure creation of state files for roll back")
self.log.info("state files == {}".format(
self.get_state_files(self.buckets[0])))
ep_queue_size_map = {node:
mem_item_count}
if self.durability_level:
self.log.info("updating the num_items on disk check to double due to durability")
ep_queue_size_map = {node:
mem_item_count * 2}
vb_replica_queue_size_map = {node: 0}
for nod in self.cluster.nodes_in_cluster:
if nod != node:
ep_queue_size_map.update({nod: 0})
vb_replica_queue_size_map.update({nod: 0})
for bucket in self.cluster.buckets:
self.bucket_util._wait_for_stat(bucket, ep_queue_size_map,
timeout=1200)
self.bucket_util._wait_for_stat(bucket, vb_replica_queue_size_map,
cbstat_cmd="all",
stat_name="vb_replica_queue_size",
timeout=1200)
# replica vBuckets
for bucket in self.cluster.buckets:
self.log.debug(cbstats.failover_stats(bucket.name))
###############################################################
'''
STEP - 4
-- Kill Memcached on Node - x and trigger rollback on other nodes
-- After 20 seconds , flush bucket
'''
shell.kill_memcached()
self.sleep(20, "sleep after killing memcached")
self.bucket_util.flush_bucket(self.cluster, self.cluster.buckets[0])
###############################################################
'''
STEP -5
-- Restarting persistence on Node -- x
'''
self.assertTrue(self.bucket_util._wait_warmup_completed(
[self.cluster.master],
self.cluster.buckets[0],
wait_time=self.wait_timeout * 10))
self.log.debug("Iteration=={}, Re-Starting persistence on Node -- {}".format(i, node))
Cbepctl(shell).persistence(self.cluster.buckets[0].name, "start")
self.sleep(5, "Sleep after re-starting persistence, Iteration{}".format(i))
shell.disconnect()
###################################################################
'''
STEP - 6
-- Load Docs on all the nodes
-- Loading of doc for 60 seconds
-- Ensures creation of new state file
'''
self.create_start = 0
self.create_end = self.init_items_per_collection
self.generate_docs(doc_ops="create", target_vbucket=None)
self.loadgen_docs(self.retry_exceptions,
self.ignore_exceptions, _sync=True,
doc_ops="create")
self.bucket_util._wait_for_stats_all_buckets(self.cluster,
self.cluster.buckets,
timeout=1200)
def test_flush_bucket_during_create_delete_kvstores(self):
"""
Test Focus: Create and Delete bucket multiple times.
Since buckets are already created in magma base
we'll start by deleting the buckets, then will recreate
STEPS:
-- Doc ops on bucket which we'll not be deleting(optional)
-- Delete already exisiting buckets
-- Recreate new buckets
-- Doc ops on buckets
-- Repaeat all the above steps
"""
self.log.info("=====test_create_delete_bucket_n_times starts=====")
count = 0
self.num_delete_buckets = self.input.param("num_delete_buckets", 1)
'''
Sorting bucket list
'''
bucket_lst = []
for bucket in self.cluster.buckets:
bucket_lst.append((bucket, bucket.name))
bucket_lst = sorted(bucket_lst, key = lambda x : x[-1])
self.log.debug ("bucket list is {} ".format(bucket_lst))
bucket_ram_quota = bucket_lst[0][0].ramQuotaMB
self.log.debug("ram_quota is {}".format(bucket_ram_quota))
scope_name = CbServer.default_scope
while count < self.test_itr:
self.log.info("Iteration=={}".format(count+1))
start = self.init_items_per_collection
'''
Step 1
-- Doc loading to buckets, which will not be getting deleted
'''
self.compute_docs(start, start//2)
self.generate_docs(doc_ops=self.doc_ops, target_vbucket=None)
tasks_info = dict()
for bucket, _ in bucket_lst[self.num_delete_buckets:]:
self.log.debug("Iteration=={}, Bucket=={}".format(count+1, bucket.name))
for collection in self.collections:
tem_tasks_info = self.loadgen_docs_per_bucket(bucket, self.retry_exceptions,
self.ignore_exceptions,
scope=scope_name,
collection=collection,
_sync=False,
doc_ops=self.doc_ops)
tasks_info.update(tem_tasks_info.items())
'''
Step 2
-- Start flushing the buckets
-- Deletion of buckets
'''
self.flush_th = threading.Thread(target=self.bucket_flush,
kwargs=dict(sigkill=self.sigkill))
self.flush_th.start()
for bucket, _ in bucket_lst[:self.num_delete_buckets]:
self.log.info("Iteration=={}, Deleting bucket=={}".format(count+1, bucket.name))
self.bucket_util.delete_bucket(self.cluster, bucket)
self.sleep(30, "waiting for 30 seconds after deletion of bucket")
for task in tasks_info:
self.task_manager.get_task_result(task)
self.stop_flush = True
self.flush_th.join()
'''
Step 4
-- Loading to buckets which are not deleted
-- Bucket recreation steps
'''
self.log.debug("Doc loading after flush")
start = 0
self.compute_docs(start, self.init_items_per_collection, doc_ops="create")
self.generate_docs(doc_ops="create", target_vbucket=None)
tasks_info = dict()
for bucket, _ in bucket_lst[self.num_delete_buckets:]:
self.log.debug("Loading after flush, Iteration=={}, Bucket=={}".
format(count+1, bucket.name))
for collection in self.collections:
tem_tasks_info = self.loadgen_docs_per_bucket(bucket, self.retry_exceptions,
self.ignore_exceptions,
scope=scope_name,
collection=collection,
_sync=False,
doc_ops="create")
tasks_info.update(tem_tasks_info.items())
self.sleep(30, "Ensuring doc loading for 30 seconds")
self.flush_th = threading.Thread(target=self.bucket_flush,
kwargs=dict(sigkill=self.sigkill))
self.flush_th.start()
buckets_created = self.bucket_util.create_multiple_buckets(
self.cluster, self.num_replicas,
bucket_count=self.num_delete_buckets,
bucket_type=self.bucket_type,
storage={"couchstore": 0,
"magma": self.num_delete_buckets},
eviction_policy=self.bucket_eviction_policy,
ram_quota=bucket_ram_quota,
bucket_name=self.bucket_name)
self.stop_flush = True
self.flush_th.join()
if not buckets_created:
buckets_created = self.bucket_util.create_multiple_buckets(
self.cluster, self.num_replicas,
bucket_count=self.num_delete_buckets,
bucket_type=self.bucket_type,
storage={"couchstore": 0,
"magma": self.num_delete_buckets},
eviction_policy=self.bucket_eviction_policy,
ram_quota=bucket_ram_quota,
bucket_name=self.bucket_name,
flush_enabled=1)
self.assertTrue(buckets_created, "Unable to create multiple buckets after bucket deletion")
for bucket in self.cluster.buckets:
ready = self.bucket_util.wait_for_memcached(
self.cluster.master,
bucket)
self.assertTrue(ready, msg="Wait_for_memcached failed")
task_info = dict()
bucket_lst = []
for bucket in self.cluster.buckets:
bucket_lst.append((bucket, bucket.name))
bucket_lst = sorted(bucket_lst, key = lambda x : x[-1])
self.log.debug("Iteration=={}, Bucket list after recreation of bucket =={} ".
format(count+1, bucket_lst))
'''
Step 5
-- Doc loading in all buckets
'''
self.generate_docs(create_end=self.init_items_per_collection,
create_start=0,
doc_ops="create",
target_vbucket=None)
for bucket, _ in bucket_lst:
self.log.info("Iteration=={}, doc loading to bucket=={}".format(count+1, bucket.name))
for collection in self.collections:
tem_task_info = self.loadgen_docs_per_bucket(bucket, self.retry_exceptions,
self.ignore_exceptions,
scope=scope_name,
collection=collection,
_sync=False,
doc_ops="create")
task_info.update(tem_task_info.items())
for task in task_info:
self.task_manager.get_task_result(task)
count += 1
def test_flush_bucket_during_data_persistence(self):
self.assertTrue(self.rest.update_autofailover_settings(False, 600),
"AutoFailover disabling failed")
count = 0
start = copy.deepcopy(self.init_items_per_collection)
while count < self.test_itr:
self.log.info("Iteration {}".format(count+1))
self.compute_docs(start, start)
for shell in self.shell_conn:
for bucket in self.cluster.buckets:
Cbepctl(shell).persistence(bucket.name, "stop")
self.generate_docs()
tasks_info = dict()
for scope in self.scopes:
for collection in self.collections:
task_info = self.loadgen_docs(
self.retry_exceptions,
self.ignore_exceptions,
scope=scope,
collection=collection,
suppress_error_table=True,
skip_read_on_error=True,
_sync=False,
doc_ops=self.doc_ops,
track_failures=False,
sdk_retry_strategy=SDKConstants.RetryStrategy.FAIL_FAST)
tasks_info.update(task_info.items())
for task in tasks_info:
self.task_manager.get_task_result(task)
for shell in self.shell_conn:
for bucket in self.cluster.buckets:
Cbepctl(shell).persistence(bucket.name, "start")
self.sleep(10, "sleep before flush thread")
for bucket in self.buckets:
self.bucket_util.flush_bucket(self.cluster, bucket)
count += 1
class basic_ops(ClusterSetup):
def setUp(self):
super(basic_ops, self).setUp()
if self.default_bucket:
# Over-ride bucket ram quota=100MB
self.bucket_size = 100
self.create_bucket(self.cluster)
self.sleep(10, "Wait for bucket to become ready for ops")
self.def_bucket = self.bucket_util.get_all_buckets(self.cluster)
self.client = SDKClient([self.cluster.master], self.def_bucket[0])
self.__durability_level()
self.create_Transaction()
self._stop = threading.Event()
self.log.info("==========Finished Basic_ops base setup========")
def tearDown(self):
self.client.close()
super(basic_ops, self).tearDown()
def __durability_level(self):
if self.durability_level == Bucket.DurabilityLevel.MAJORITY:
self.durability = 1
elif self.durability_level \
== Bucket.DurabilityLevel.MAJORITY_AND_PERSIST_TO_ACTIVE:
self.durability = 2
elif self.durability_level \
== Bucket.DurabilityLevel.PERSIST_TO_MAJORITY:
self.durability = 3
elif self.durability_level == "ONLY_NONE":
self.durability = 4
else:
self.durability = 0
def get_doc_generator(self, start, end):
age = range(5)
name = ['james', 'sharon']
body = [''.rjust(self.doc_size - 10, 'a')]
template = JsonObject.create()
template.put("age", age)
template.put("first_name", name)
template.put("body", body)
generator = DocumentGenerator(self.key,
template,
start=start,
end=end,
key_size=self.key_size,
doc_size=self.doc_size,
doc_type=self.doc_type,
randomize=True,
age=age,
first_name=name)
return generator
def set_exception(self, exception):
self.exception = exception
raise BaseException("Got an exception %s" % self.exception)
def __chunks(self, l, n):
"""Yield successive n-sized chunks from l."""
for i in range(0, len(l), n):
yield l[i:i + n]
def create_Transaction(self, client=None):
if not client:
client = self.client
transaction_config = Transaction().createTransactionConfig(
self.transaction_timeout, self.durability)
try:
self.transaction = Transaction().createTansaction(
client.cluster, transaction_config)
except Exception as e:
self.set_exception(e)
def __thread_to_transaction(self,
transaction,
op_type,
doc,
txn_commit,
update_count=1,
sync=True,
set_exception=True,
client=None):
exception = None
if client is None:
client = self.client
if op_type == "create":
exception = Transaction().RunTransaction(client.cluster,
transaction,
[client.collection], doc,
[], [], txn_commit, sync,
update_count)
elif op_type == "update":
self.log.info("updating all the keys through threads")
exception = Transaction().RunTransaction(client.cluster,
transaction,
[client.collection], [],
doc, [], txn_commit, sync,
update_count)
elif op_type == "delete":
exception = Transaction().RunTransaction(client.cluster,
transaction,
[client.collection], [],
[], doc, txn_commit, sync,
update_count)
if set_exception and exception:
self.set_exception("Failed")
def doc_gen(self,
num_items,
start=0,
value={'value': 'value1'},
op_type="create"):
self.docs = []
self.keys = []
self.content = self.client.translate_to_json_object(value)
for i in range(start, self.num_items):
key = "test_docs-" + str(i)
if op_type == "create":
doc = Tuples.of(key, self.content)
self.keys.append(key)
self.docs.append(doc)
else:
self.docs.append(key)
def verify_doc(self, num_items, client):
for i in range(num_items):
key = "test_docs-" + str(i)
result = client.read(key)
actual_val = self.client.translate_to_json_object(result['value'])
self.assertEquals(self.content, actual_val)
def test_MultiThreadTxnLoad(self):
"""
Load data through txn, update half the items through different threads
and delete half the items through different threads. if update_retry
then update and delete the same key in two different transaction
and make sure update fails
"""
self.num_txn = self.input.param("num_txn", 9)
self.update_retry = self.input.param("update_retry", False)
self.doc_gen(self.num_items)
threads = []
# create the docs
exception = Transaction().RunTransaction(self.client.cluster,
self.transaction,
[self.client.collection],
self.docs, [], [],
self.transaction_commit, True,
self.update_count)
if exception:
self.set_exception("Failed")
if self.update_retry:
threads.append(
threading.Thread(target=self.__thread_to_transaction,
args=(self.transaction, "delete", self.keys,
self.transaction_commit,
self.update_count)))
threads.append(
threading.Thread(target=self.__thread_to_transaction,
args=(self.transaction, "update", self.keys,
10, self.update_count)))
else:
update_docs = self.__chunks(self.keys[:self.num_items / 2],
self.num_txn)
delete_docs = self.__chunks(self.keys[self.num_items / 2:],
self.num_txn)
for keys in update_docs:
threads.append(
threading.Thread(target=self.__thread_to_transaction,
args=(self.transaction, "update", keys,
self.transaction_commit,
self.update_count)))
for keys in delete_docs:
threads.append(
threading.Thread(target=self.__thread_to_transaction,
args=(self.transaction, "delete", keys,
self.transaction_commit,
self.update_count)))
for thread in threads:
thread.start()
for thread in threads:
thread.join()
self.sleep(60, "Wait for transactions to complete")
if self.update_retry:
for key in self.keys:
result = self.client.read(key)
self.assertEquals(result['status'], False)
else:
self.value = {'mutated': 1, 'value': 'value1'}
self.content = self.client.translate_to_json_object(self.value)
self.verify_doc(self.num_items / 2, self.client)
for key in self.keys[self.num_items / 2:]:
result = self.client.read(key)
self.assertEquals(result['status'], False)
def test_basic_retry(self):
"""
Load set of data to the cluster, update through 2 different threads,
make sure transaction maintains the order of update
:return:
"""
self.write_conflict = self.input.param("write_conflict", 2)
self.log.info("going to create and execute the task")
self.gen_create = self.get_doc_generator(0, self.num_items)
task = self.task.async_load_gen_docs_atomicity(
self.cluster,
self.def_bucket,
self.gen_create,
"create",
exp=0,
batch_size=10,
process_concurrency=8,
replicate_to=self.replicate_to,
persist_to=self.persist_to,
timeout_secs=self.sdk_timeout,
retries=self.sdk_retries,
update_count=self.update_count,
transaction_timeout=self.transaction_timeout,
commit=True,
durability=self.durability_level,
sync=self.sync)
self.task.jython_task_manager.get_task_result(task)
self.log.info("Get all the keys in the cluster")
self.doc_gen(self.num_items)
threads = []
for update_count in [2, 4, 6]:
threads.append(
threading.Thread(target=self.__thread_to_transaction,
args=(self.transaction, "update", self.keys,
self.transaction_commit, update_count)))
# Add verification task
if self.transaction_commit:
self.update_count = 6
else:
self.update_count = 0
for thread in threads:
thread.start()
self.sleep(2, "Wait for transaction thread to start")
for thread in threads:
thread.join()
def test_basic_retry_async(self):
self.log.info("going to create and execute the task")
self.gen_create = self.get_doc_generator(0, self.num_items)
task = self.task.async_load_gen_docs_atomicity(
self.cluster,
self.def_bucket,
self.gen_create,
"create",
exp=0,
batch_size=10,
process_concurrency=1,
replicate_to=self.replicate_to,
persist_to=self.persist_to,
timeout_secs=self.sdk_timeout,
retries=self.sdk_retries,
update_count=self.update_count,
transaction_timeout=self.transaction_timeout,
commit=True,
durability=self.durability_level,
sync=True,
num_threads=1)
self.task.jython_task_manager.get_task_result(task)
self.log.info("get all the keys in the cluster")
keys = ["test_docs-0"] * 2
exception = Transaction().RunTransaction(
self.client.cluster, self.transaction, [self.client.collection],
[], keys, [], self.transaction_commit, False, 0)
if exception:
self.set_exception(Exception(exception))
def basic_concurrency(self):
self.crash = self.input.param("crash", False)
self.doc_gen(self.num_items)
# run transaction
thread = threading.Thread(target=self.__thread_to_transaction,
args=(self.transaction, "create", self.docs,
self.transaction_commit,
self.update_count, True, False))
thread.start()
self.sleep(1, "Wait for transaction thread to start")
if self.crash:
self.client.cluster.disconnect()
self.transaction.close()
self.client1 = SDKClient([self.cluster.master], self.def_bucket[0])
self.create_Transaction(self.client1)
self.sleep(self.transaction_timeout + 60,
"Wait for transaction cleanup to complete")
exception = Transaction().RunTransaction(
self.client.cluster, self.transaction,
[self.client1.collection], self.docs, [], [],
self.transaction_commit, self.sync, self.update_count)
if exception:
self.sleep(60, "Wait for transaction cleanup to happen")
self.verify_doc(self.num_items, self.client1)
self.client1.close()
else:
key = "test_docs-0"
# insert will succeed due to doc_isoloation feature
result = self.client.insert(key, "value")
self.assertEqual(result["status"], True)
# Update should pass
result = self.client.upsert(key, "value")
self.assertEqual(result["status"], True)
# delete should pass
result = self.client.delete(key)
self.assertEqual(result["status"], True)
thread.join()
def test_stop_loading(self):
"""
Load through transactions and close the transaction abruptly,
create a new transaction sleep for 60 seconds and
perform create on the same set of docs
"""
self.num_txn = self.input.param("num_txn", 9)
self.doc_gen(self.num_items)
threads = []
docs = list(self.__chunks(self.docs, len(self.docs) / self.num_txn))
for doc in docs:
threads.append(
threading.Thread(target=self.__thread_to_transaction,
args=(self.transaction, "create", doc,
self.transaction_commit,
self.update_count, True, False)))
for thread in threads:
thread.start()
self.client.cluster.disconnect()
self.transaction.close()
self.client1 = SDKClient([self.cluster.master], self.def_bucket[0])
self.create_Transaction(self.client1)
self.sleep(self.transaction_timeout + 60,
"Wait for transaction cleanup to happen")
self.log.info("going to start the load")
for doc in docs:
exception = Transaction().RunTransaction(
self.client1.cluster, self.transaction,
[self.client1.collection], doc, [], [],
self.transaction_commit, self.sync, self.update_count)
if exception:
self.sleep(60, "Wait for transaction cleanup to happen")
self.verify_doc(self.num_items, self.client1)
self.client1.close()
def __insert_sub_doc_and_validate(self, doc_id, op_type, key, value):
_, failed_items = self.client.crud(op_type,
doc_id, [key, value],
durability=self.durability_level,
timeout=self.sdk_timeout,
time_unit="seconds",
create_path=True,
xattr=True)
self.assertFalse(failed_items, "Subdoc Xattr insert failed")
def __read_doc_and_validate(self, doc_id, expected_val, subdoc_key=None):
if subdoc_key:
success, failed_items = self.client.crud("subdoc_read",
doc_id,
subdoc_key,
xattr=True)
self.assertFalse(failed_items, "Xattr read failed")
self.assertEqual(
expected_val, str(success[doc_id]["value"][0]),
"Sub_doc value mismatch: %s != %s" %
(success[doc_id]["value"][0], expected_val))
def test_TxnWithXattr(self):
self.system_xattr = self.input.param("system_xattr", False)
if self.system_xattr:
xattr_key = "my._attr"
else:
xattr_key = "my.attr"
val = "v" * self.doc_size
self.doc_gen(self.num_items)
thread = threading.Thread(target=self.__thread_to_transaction,
args=(self.transaction, "create", self.docs,
self.transaction_commit,
self.update_count))
thread.start()
thread.join()
self.doc_gen(self.num_items,
op_type="update",
value={
"mutated": 1,
"value": "value1"
})
thread = threading.Thread(target=self.__thread_to_transaction,
args=(self.transaction, "update", self.docs,
self.transaction_commit,
self.update_count))
thread.start()
self.sleep(1)
self.__insert_sub_doc_and_validate("test_docs-0", "subdoc_insert",
xattr_key, val)
thread.join()
if self.transaction_commit:
self.__read_doc_and_validate("test_docs-0", val, xattr_key)
self.sleep(60, "Wait for transaction to complete")
self.verify_doc(self.num_items, self.client)
def test_TxnWithMultipleXattr(self):
xattrs_to_insert = [["my.attr", "value"], ["new_my.attr", "new_value"]]
self.doc_gen(self.num_items)
thread = threading.Thread(target=self.__thread_to_transaction,
args=(self.transaction, "create", self.docs,
self.transaction_commit,
self.update_count))
thread.start()
thread.join()
self.doc_gen(self.num_items,
op_type="update",
value={
"mutated": 1,
"value": "value1"
})
thread = threading.Thread(target=self.__thread_to_transaction,
args=(self.transaction, "update", self.docs,
self.transaction_commit,
self.update_count))
thread.start()
self.sleep(1, "Wait for transx-thread to start")
for key, val in xattrs_to_insert:
self.__insert_sub_doc_and_validate("test_docs-0", "subdoc_insert",
key, val)
thread.join()
if self.transaction_commit:
for key, val in xattrs_to_insert:
self.__read_doc_and_validate("test_docs-0", val, key)
self.sleep(60, "Wait for transaction to complete")
self.verify_doc(self.num_items, self.client)
