class MagmaCrashTests(MagmaBaseTest):
def setUp(self):
self.input = TestInputSingleton.input
self.input.test_params.update({"random_key": True})
super(MagmaCrashTests, self).setUp()
self.sdk_timeout = self.input.param("sdk_timeout", 10)
self.time_unit = "seconds"
self.graceful = self.input.param("graceful", False)
self.assertTrue(self.rest.update_autofailover_settings(False, 600),
"AutoFailover disabling failed")
self.crash_th = None
self.sdk_retry_strategy = self.input.param(
"sdk_retry_strategy", SDKConstants.RetryStrategy.FAIL_FAST)
def tearDown(self):
self.stop_crash = True
if self.crash_th and self.crash_th.is_alive():
self.crash_th.join()
super(MagmaCrashTests, self).tearDown()
def kill_magma_check_wal_file_size(self):
nIter = 200
while nIter > 0:
shell = RemoteMachineShellConnection(self.cluster.master)
shell.kill_memcached()
# self.bucket_util._wait_warmup_completed()
self.sleep(10, "sleep of 5s so that memcached can restart")
def test_crash_during_ops(self):
self.graceful = self.input.param("graceful", False)
wait_warmup = self.input.param("wait_warmup", True)
self.log.info("====test_crash_during_ops starts====")
self.compute_docs_ranges()
tasks_info = dict()
for collection in self.collections:
self.generate_docs(doc_ops=self.doc_ops, target_vbucket=None)
tem_tasks_info = self.loadgen_docs(
self.retry_exceptions,
self.ignore_exceptions,
scope=CbServer.default_scope,
collection=collection,
suppress_error_table=True,
skip_read_on_error=True,
_sync=False,
doc_ops=self.doc_ops,
track_failures=False,
sdk_retry_strategy=self.sdk_retry_strategy)
tasks_info.update(tem_tasks_info.items())
self.crash_th = threading.Thread(target=self.crash,
kwargs=dict(graceful=self.graceful,
wait=wait_warmup))
self.crash_th.start()
for task in tasks_info:
self.task_manager.get_task_result(task)
self.stop_crash = True
self.crash_th.join()
self.assertFalse(self.crash_failure,
"CRASH | CRITICAL | WARN messages found in cb_logs")
self.validate_seq_itr()
def test_crash_during_recovery(self):
self.compute_docs_ranges()
tasks_info = dict()
for collection in self.collections:
self.generate_docs(doc_ops=self.doc_ops, target_vbucket=None)
tem_tasks_info = self.loadgen_docs(
self.retry_exceptions,
self.ignore_exceptions,
scope=CbServer.default_scope,
collection=collection,
suppress_error_table=True,
skip_read_on_error=True,
_sync=False,
doc_ops=self.doc_ops,
track_failures=False,
sdk_retry_strategy=self.sdk_retry_strategy)
tasks_info.update(tem_tasks_info.items())
self.crash_th = threading.Thread(target=self.crash,
kwargs={"kill_itr": 5})
self.crash_th.start()
for task in tasks_info:
self.task_manager.get_task_result(task)
self.stop_crash = True
self.crash_th.join()
self.assertFalse(self.crash_failure,
"CRASH | CRITICAL | WARN messages found in cb_logs")
self.validate_seq_itr()
def test_crash_before_upserts(self):
self.log.info("test_update_multi starts")
self.change_swap_space(self.cluster.nodes_in_cluster)
upsert_doc_list = self.get_fragmentation_upsert_docs_list()
count = 0
self.mutate = 0
while count < self.test_itr:
self.log.info("Iteration == {}".format(count + 1))
self.sigkill_memcached(graceful=self.graceful)
for itr in upsert_doc_list:
self.doc_ops = "update"
self.update_start = 0
self.update_end = itr
if self.rev_update:
self.update_start = -int(itr - 1)
self.update_end = 1
self.generate_docs(doc_ops="update")
_ = self.loadgen_docs(self.retry_exceptions,
self.ignore_exceptions,
suppress_error_table=True,
_sync=True,
doc_ops="update")
self.bucket_util._wait_for_stats_all_buckets(
self.cluster, self.cluster.buckets)
count += 1
self.validate_data("update", self.gen_update)
self.change_swap_space(self.cluster.nodes_in_cluster, disable=False)
self.validate_seq_itr()
self.log.info("====test_update_multi ends====")
def test_crash_before_multi_update_deletes(self):
self.log.info("===test_crash_before_multi_update_deletes starts===")
self.change_swap_space(self.cluster.nodes_in_cluster)
count = 0
self.mutate = 0
for i in range(self.test_itr):
self.log.info("Step 1, Iteration= {}".format(i + 1))
while count < self.update_itr:
self.sigkill_memcached(graceful=self.graceful)
self.doc_ops = "update"
self.update_start = 0
self.update_end = self.num_items
self.generate_docs(doc_ops="update")
_ = self.loadgen_docs(self.retry_exceptions,
self.ignore_exceptions,
suppress_error_table=True,
_sync=True,
doc_ops="update")
self.bucket_util._wait_for_stats_all_buckets(
self.cluster, self.cluster.buckets)
count += 1
self.update_itr += self.update_itr
# data validation is done only for the last iteration
if i + 1 == self.test_itr:
self.validate_data("update", self.gen_update)
self.log.debug("Step 2, Iteration {}".format(i + 1))
self.sigkill_memcached()
self.doc_ops = "delete"
self.delete_start = 0
self.delete_end = self.num_items // 2
self.generate_docs(doc_ops="delete")
_ = self.loadgen_docs(self.retry_exceptions,
self.ignore_exceptions,
suppress_error_table=True,
_sync=True,
doc_ops="delete")
self.bucket_util._wait_for_stats_all_buckets(
self.cluster, self.cluster.buckets)
self.bucket_util.verify_stats_all_buckets(self.cluster,
self.num_items)
self.log.debug("Step 3, Iteration= {}".format(i + 1))
self.sigkill_memcached()
self.gen_create = copy.deepcopy(self.gen_delete)
self.doc_ops = "create"
_ = self.loadgen_docs(self.retry_exceptions,
self.ignore_exceptions,
suppress_error_table=True,
_sync=True,
doc_ops="create")
self.bucket_util._wait_for_stats_all_buckets(
self.cluster, self.cluster.buckets)
self.bucket_util.verify_stats_all_buckets(self.cluster,
self.num_items)
self.validate_data("create", self.gen_create)
self.change_swap_space(self.cluster.nodes_in_cluster, disable=False)
self.validate_seq_itr()
self.log.info("===test_crash_before_multi_update_deletes ends===")
def test_crash_during_get_ops(self):
self.log.info("test_crash_during_get_ops starts")
self.change_swap_space(self.cluster.nodes_in_cluster)
tasks_info = dict()
upsert_doc_list = self.get_fragmentation_upsert_docs_list()
for itr in upsert_doc_list:
self.doc_ops = "update"
self.update_start = 0
self.update_end = itr
self.mutate = -1
self.generate_docs(doc_ops="update")
update_task_info = self.loadgen_docs(self.retry_exceptions,
self.ignore_exceptions,
suppress_error_table=True,
_sync=False)
#update_task_info = self.loadgen_docs(
# self.retry_exceptions,
# self.ignore_exceptions,
# _sync=False)
tasks_info.update(update_task_info.items())
self.doc_ops = "read"
self.generate_docs(doc_ops="read")
start = -int(self.num_items - 1)
end = 1
reverse_read_gen = self.genrate_docs_basic(start, end)
count = 0
while count < self.read_thread_count:
read_task_info = self.loadgen_docs(self.retry_exceptions,
self.ignore_exceptions,
suppress_error_table=True,
_sync=False)
tasks_info.update(read_task_info.items())
count += 1
if count < self.read_thread_count:
read_task_info = self.bucket_util._async_validate_docs(
self.cluster,
reverse_read_gen,
"read",
0,
batch_size=self.batch_size,
process_concurrency=self.process_concurrency,
timeout_secs=self.sdk_timeout,
retry_exceptions=self.retry_exceptions,
ignore_exceptions=self.ignore_exceptions,
suppress_error_table=False)
tasks_info.update(read_task_info.items())
count += 1
self.crash_th = threading.Thread(target=self.crash,
kwargs={"graceful": self.graceful})
self.crash_th.start()
for task in tasks_info:
self.task_manager.get_task_result(task)
self.stop_crash = True
self.crash_th.join()
self.assertFalse(self.crash_failure,
"CRASH | CRITICAL | WARN messages found in cb_logs")
self.bucket_util._wait_for_stats_all_buckets(self.cluster,
self.cluster.buckets)
self.change_swap_space(self.cluster.nodes_in_cluster, disable=False)
self.validate_seq_itr()
self.log.info("test_crash_during_get_ops ends")
def test_crash_during_upserts_using_multithreads(self):
self.log.info("test_crash_during_upserts_using_multithreads starts")
self.change_swap_space(self.cluster.nodes_in_cluster)
tasks_info = dict()
self.doc_ops = "update"
self.update_start = 0
self.update_end = self.num_items
count = 0
while count < self.read_thread_count:
self.generate_docs(doc_ops="update")
update_task_info = self.loadgen_docs(
self.retry_exceptions,
self.ignore_exceptions,
suppress_error_table=True,
skip_read_on_error=True,
_sync=False,
track_failures=False,
sdk_retry_strategy=self.sdk_retry_strategy)
tasks_info.update(update_task_info.items())
count += 1
self.sleep(5)
self.crash_th = threading.Thread(target=self.crash,
kwargs={"graceful": self.graceful})
self.crash_th.start()
for task in tasks_info:
self.task_manager.get_task_result(task)
self.stop_crash = True
self.crash_th.join()
self.assertFalse(self.crash_failure,
"CRASH | CRITICAL | WARN messages found in cb_logs")
self.bucket_util._wait_for_stats_all_buckets(self.cluster,
self.cluster.buckets)
self.change_swap_space(self.cluster.nodes_in_cluster, disable=False)
self.validate_seq_itr()
self.log.info("test_crash_during_upserts_using_multithreads ends")
def test_crash_during_multi_updates_of_single_doc(self):
self.log.info(
"==test_crash_during_multi_updates_of_single_doc starts==")
self.change_swap_space(self.cluster.nodes_in_cluster)
self.client = SDKClient([self.cluster.master],
self.cluster.buckets[0],
scope=CbServer.default_scope,
collection=CbServer.default_collection)
self.doc_ops = "update"
self.gen_update = self.genrate_docs_basic(start=0, end=1)
key, val = self.gen_update.next()
def upsert_doc(start_num, end_num, key_obj, val_obj):
for i in range(start_num, end_num):
val_obj.put("mutated", i)
self.client.upsert(key_obj, val_obj)
self.crash_th = threading.Thread(target=self.crash,
kwargs={"graceful": self.graceful})
self.crash_th.start()
threads = []
start = 0
end = 0
for _ in range(10):
start = end
end += 10
th = threading.Thread(target=upsert_doc,
args=[start, end, key, val])
th.start()
threads.append(th)
for th in threads:
th.join()
self.stop_crash = True
self.crash_th.join()
self.assertFalse(self.crash_failure,
"CRASH | CRITICAL | WARN messages found in cb_logs")
self.bucket_util._wait_for_stats_all_buckets(self.cluster,
self.cluster.buckets)
success, _ = self.client.get_multi([key], self.wait_timeout)
self.assertIs(key in success,
True,
msg="key {} doesn't exist\
".format(key))
expected_val = Json.loads(val.toString())
actual_val = Json.loads(success[key]['value'].toString())
self.assertIs(expected_val == actual_val,
True,
msg="expected_val-{} != Actual_val-{}\
".format(expected_val, actual_val))
self.change_swap_space(self.cluster.nodes_in_cluster, disable=False)
self.validate_seq_itr()
self.log.info("==test_crash_during_multi_updates_of_single_doc ends==")
def test_crash_during_val_movement_across_trees(self):
self.log.info("==test_crash_during_val_movement_across_trees starts==")
self.change_swap_space(self.cluster.nodes_in_cluster)
upsert_size = 0
if self.doc_size < 32:
upsert_size = 2048
self.update_start = 0
self.update_end = self.num_items
if self.rev_update:
self.update_start = -int(self.num_items - 1)
self.update_end = 1
self.doc_ops = "update"
self.crash_th = threading.Thread(target=self.crash,
kwargs={"graceful": self.graceful})
self.crash_th.start()
count = 0
while count < self.test_itr:
self.log.info("Iteration == {}".format(count))
self.mutate += 1
self.gen_update = doc_generator(
self.key,
self.update_start,
self.update_end,
doc_size=upsert_size,
doc_type=self.doc_type,
target_vbucket=self.target_vbucket,
vbuckets=self.cluster.vbuckets,
key_size=self.key_size,
mutate=self.mutate,
randomize_doc_size=self.randomize_doc_size,
randomize_value=self.randomize_value,
mix_key_size=self.mix_key_size,
deep_copy=self.deep_copy)
_ = self.loadgen_docs(self.retry_exceptions,
self.ignore_exceptions,
suppress_error_table=True,
skip_read_on_error=True,
_sync=True,
track_failures=False,
sdk_retry_strategy=self.sdk_retry_strategy)
self.bucket_util._wait_for_stats_all_buckets(
self.cluster, self.cluster.buckets)
self.generate_docs(doc_ops="update")
_ = self.loadgen_docs(self.retry_exceptions,
self.ignore_exceptions,
suppress_error_table=True,
skip_read_on_error=True,
_sync=True,
track_failures=False,
sdk_retry_strategy=self.sdk_retry_strategy)
self.bucket_util._wait_for_stats_all_buckets(
self.cluster, self.cluster.buckets)
count += 1
self.stop_crash = True
self.crash_th.join()
self.assertFalse(self.crash_failure,
"CRASH | CRITICAL | WARN messages found in cb_logs")
self.change_swap_space(self.cluster.nodes_in_cluster, disable=False)
self.validate_seq_itr()
self.log.info("==test_crash_during_val_movement_across_trees ends==")
class BasicUpsertTests(BasicCrudTests):
def test_update_n_times(self):
"""
Test Focus: Update items n times and
test space amplification
STEPS:
-- Update items n times (where n gets calculated
from fragmentation value
-- Check space amplification
-- Repeat the above steps n times
-- After all iterations validate the data
"""
self.log.info("test_update_n_times starts")
upsert_doc_list = self.get_fragmentation_upsert_docs_list()
self.mutate = 0
count = 0
while count < self.test_itr:
self.log.info("Iteration == {}".format(count+1))
#######################################################################
'''
STEP - 1, Update Items
'''
for itr in upsert_doc_list:
self.doc_ops = "update"
self.update_start = 0
self.update_end = itr
if self.rev_update:
self.update_start = -int(itr - 1)
self.update_end = 1
self.generate_docs(doc_ops="update")
_ = self.loadgen_docs(self.retry_exceptions,
self.ignore_exceptions,
_sync=True)
self.log.info("Waiting for ep-queues to get drained")
self.bucket_util._wait_for_stats_all_buckets(timeout=3600)
#######################################################################
'''
STEP - 2, Space Amplification Check
'''
msg = "Fragmentation value for {} stats exceeds\
the configured value"
_result = self.check_fragmentation_using_magma_stats(
self.buckets[0],
self.cluster.nodes_in_cluster)
self.assertIs(_result, True,
msg.format("magma"))
_r = self.check_fragmentation_using_bucket_stats(
self.buckets[0], self.cluster.nodes_in_cluster)
self.assertIs(_r, True,
msg.format("KV"))
usage_factor = ((float(
self.num_items + sum(upsert_doc_list)
) / self.num_items) + 0.5)
self.log.debug("Disk usage factor = {}".format(usage_factor))
time_end = time.time() + 60 * 2
while time.time() < time_end:
disk_usage = self.get_disk_usage(self.buckets[0],
self.cluster.nodes_in_cluster)
_res = disk_usage[0]
self.log.debug("usage at time {} is {}".format((time_end - time.time()), _res))
if _res < usage_factor * self.disk_usage[self.disk_usage.keys()[0]]:
break
msg = "Iteration= {}, Disk Usage = {}MB\
exceeds {} times from Actual disk usage = {}MB"
self.assertIs(_res > usage_factor * self.disk_usage[
self.disk_usage.keys()[0]],
False, msg.format(count+1, _res, usage_factor,
self.disk_usage[self.disk_usage.keys()[0]]))
count += 1
#######################################################################
'''
STEP - 3, Data Validation
'''
self.validate_data("update", self.gen_update)
self.change_swap_space(self.cluster.nodes_in_cluster,
disable=False)
self.log.info("====test_update_n_times ends====")
def test_multi_update_delete(self):
"""
STEPS:
-- Update items x times
-- Check space amplification
-- Delete half of the items
-- Check space Amplification
-- Recreate deleted items
-- Check Space Amplification
-- Repeat above steps for n times
-- After all iterations validate the data
"""
self.log.info("==== test_multi_update_delete starts =====")
count = 0
msg_stats = "Fragmentation value for {} stats exceeds\
the configured value"
msg = "{} Iteration= {}, Disk Usage = {}MB\
exceeds 2.5 times from Actual disk usage = {}MB"
self.mutate = 0
for i in range(self.test_itr):
self.log.info("Step 1, Iteration= {}".format(i+1))
#######################################################################
'''
STEP - 1, Update Items, update_itr times
'''
while count < self.update_itr:
self.doc_ops = "update"
self.update_start = 0
self.update_end = self.num_items
if self.rev_update:
self.update_start = -int(self.num_items - 1)
self.update_end = 1
self.generate_docs(doc_ops="update")
_ = self.loadgen_docs(self.retry_exceptions,
self.ignore_exceptions,
_sync=True)
self.log.info("Waiting for ep-queues to get drained")
self.bucket_util._wait_for_stats_all_buckets(timeout=3600)
###################################################################
'''
STEP - 2
-- Space Amplification check after each update iteration.
-- Data validation only for last update iteration
'''
self.log.info("Step 2, Iteration= {}".format(i+1))
_result = self.check_fragmentation_using_magma_stats(
self.buckets[0],
self.cluster.nodes_in_cluster)
self.assertIs(_result, True,
msg_stats.format("magma"))
_r = self.check_fragmentation_using_bucket_stats(
self.buckets[0], self.cluster.nodes_in_cluster)
self.assertIs(_r, True,
msg_stats.format("KV"))
time_end = time.time() + 60 * 2
while time.time() < time_end:
disk_usage = self.get_disk_usage(self.buckets[0],
self.cluster.nodes_in_cluster)
_res = disk_usage[0]
self.log.info("Update Iteration-{}, Disk Usage at time {} is {}MB \
".format(count+1, time_end - time.time(), _res))
if _res < 2.5 * self.disk_usage[self.disk_usage.keys()[0]]:
break
self.assertIs(
_res > 2.5 * self.disk_usage[self.disk_usage.keys()[0]],
False, msg.format("update", count+1, _res,
self.disk_usage[self.disk_usage.keys()[0]]))
count += 1
self.update_itr += self.update_itr
if i+1 == self.test_itr:
self.validate_data("update", self.gen_update)
###################################################################
'''
STEP - 3
-- Delete half of the docs.
'''
self.log.debug("Step 3, Iteration {}".format(i+1))
self.doc_ops = "delete"
self.delete_start = 0
self.delete_end = self.num_items//2
if self.rev_del:
self.delete_start = -int(self.num_items//2 - 1)
self.delete_end = 1
self.generate_docs(doc_ops="delete")
_ = self.loadgen_docs(self.retry_exceptions,
self.ignore_exceptions,
_sync=True)
self.bucket_util._wait_for_stats_all_buckets(timeout=3600)
self.bucket_util.verify_stats_all_buckets(self.num_items)
###################################################################
'''
STEP - 4
-- Space Amplification Check after deletion.
'''
self.log.debug("Step 4, Iteration {}".format(i+1))
_result = self.check_fragmentation_using_magma_stats(
self.buckets[0],
self.cluster.nodes_in_cluster)
self.assertIs(_result, True,
msg_stats.format("magma"))
_r = self.check_fragmentation_using_bucket_stats(
self.buckets[0], self.cluster.nodes_in_cluster)
self.assertIs(_r, True,
msg_stats.format("KV"))
disk_usage = self.get_disk_usage(self.buckets[0],
self.cluster.nodes_in_cluster)
_res = disk_usage[0]
self.log.info("Delete Iteration {}, Disk Usage- {}MB\
".format(i+1, _res))
self.assertIs(
_res > 2.5 * self.disk_usage[
self.disk_usage.keys()[0]],
False, msg.format(
"delete", i+1, _res,
self.disk_usage[self.disk_usage.keys()[0]]))
###################################################################
'''
STEP - 5
-- ReCreation of docs.
'''
self.log.debug("Step 5, Iteration= {}".format(i+1))
self.gen_create = copy.deepcopy(self.gen_delete)
self.doc_ops = "create"
_ = self.loadgen_docs(self.retry_exceptions,
self.ignore_exceptions,
_sync=True)
self.bucket_util._wait_for_stats_all_buckets(timeout=3600)
self.bucket_util.verify_stats_all_buckets(self.num_items)
###################################################################
'''
STEP - 6
-- Space Amplification Check after Recreation.
'''
self.log.debug("Step 6, Iteration= {}".format(i+1))
_result = self.check_fragmentation_using_magma_stats(
self.buckets[0],
self.cluster.nodes_in_cluster)
self.assertIs(_result, True,
msg_stats.format("magma"))
_r = self.check_fragmentation_using_bucket_stats(
self.buckets[0], self.cluster.nodes_in_cluster)
self.assertIs(_r, True,
msg_stats.format("KV"))
disk_usage = self.get_disk_usage(self.buckets[0],
self.cluster.nodes_in_cluster)
_res = disk_usage[0]
self.log.info("Create Iteration{}, Disk Usage= {}MB \
".format(i+1, _res))
self.assertIs(_res > 2.5 * self.disk_usage[
self.disk_usage.keys()[0]],
False, msg.format("Create", _res, i+1,
self.disk_usage[self.disk_usage.keys()[0]]))
###################################################################
'''
STEP - 7
-- Validate data
-- Data validation is only for the creates in last iterations.
'''
self.log.debug("Step 7, Iteration= {}".format(i+1))
self.validate_data("create", self.gen_create)
self.log.info("====test_multiUpdate_delete ends====")
def test_update_rev_update(self):
"""
STEPS:
-- Update num_items // 2 items.
-- Reverse update remaining num_items // 2 items.
-- If next.half is false skip above step
-- And reverse update items in first point
-- Check space amplification
-- Repeat above steps x times
-- Delete all the items
-- Check space Amplification
-- Recreate deleted items
-- Check Space Amplification
-- Repeat above steps for n times
-- After all iterations validate the data
"""
self.log.info("==== test_update_rev_update starts =====")
msg_stats = "Fragmentation value for {} stats exceeds\
the configured value"
msg = "{} Iteration= {}, Disk Usage = {}MB\
exceeds {} times from Actual disk usage = {}MB"
count = 0
mutated = 1
for i in range(self.test_itr):
self.log.debug("Step 1, Iteration= {}".format(i+1))
#######################################################################
'''
STEP - 1, Update Items, update_itr times
-- Update n // 2 items
-- If self.next_half is true
-- Update remaining n//2 items
-- Else, again update items in
reverse order in first point
'''
while count < self.update_itr:
tasks_info = dict()
self.doc_ops = "update"
self.gen_update = self.genrate_docs_basic(0, self.num_items //2,
mutate=mutated)
tem_tasks_info = self.loadgen_docs(self.retry_exceptions,
self.ignore_exceptions,
_sync=False)
tasks_info.update(tem_tasks_info.items())
if self.next_half:
start = - (self.num_items - 1)
end = - (self.num_items // 2 - 1)
self.gen_update = self.genrate_docs_basic(start, end,
mutate=mutated)
tem_tasks_info = self.loadgen_docs(self.retry_exceptions,
self.ignore_exceptions,
_sync=False)
tasks_info.update(tem_tasks_info.items())
for task in tasks_info:
self.task_manager.get_task_result(task)
self.bucket_util.verify_doc_op_task_exceptions(
tasks_info, self.cluster)
self.bucket_util.log_doc_ops_task_failures(tasks_info)
mutated += 1
if not self.next_half:
start = - (self.num_items - 1)
end = - (self.num_items // 2 - 1)
self.gen_update = self.genrate_docs_basic(start, end,
mutate=mutated)
_ = self.loadgen_docs(self.retry_exceptions,
self.ignore_exceptions,
_sync=True)
mutated += 1
self.log.info("Waiting for ep-queues to get drained")
self.bucket_util._wait_for_stats_all_buckets(timeout=3600)
###################################################################
'''
STEP - 2
-- Space Amplification check after each update iteration.
'''
self.log.debug("Step 2, Iteration= {}".format(i+1))
_result = self.check_fragmentation_using_magma_stats(
self.buckets[0],
self.cluster.nodes_in_cluster)
self.assertIs(_result, True,
msg_stats.format("magma"))
_r = self.check_fragmentation_using_bucket_stats(
self.buckets[0], self.cluster.nodes_in_cluster)
self.assertIs(_r, True,
msg_stats.format("KV"))
disk_usage = self.get_disk_usage(
self.buckets[0], self.cluster.nodes_in_cluster)
_res = disk_usage[0]
self.log.info("Update Iteration- {}, Disk Usage- {}MB\
".format(count+1, _res))
self.assertIs(
_res > 2.5 * self.disk_usage[self.disk_usage.keys()[0]],
False, msg.format("update", count+1, _res, 2.5,
self.disk_usage[self.disk_usage.keys()[0]]))
count += 1
self.update_itr += self.update_itr
###################################################################
'''
STEP - 3
-- Delete all the items.
'''
self.log.debug("Step 3, Iteration {}".format(i+1))
self.doc_ops = "delete"
self.delete_start = 0
self.delete_end = self.num_items
if self.rev_del:
self.delete_start = -int(self.num_items - 1)
self.delete_end = 1
self.generate_docs(doc_ops="delete")
_ = self.loadgen_docs(self.retry_exceptions,
self.ignore_exceptions,
_sync=True)
self.bucket_util._wait_for_stats_all_buckets(timeout=3600)
self.bucket_util.verify_stats_all_buckets(self.num_items)
###################################################################
'''
STEP - 4
-- Space Amplification Check after deletion.
'''
self.log.debug("Step 4, Iteration {}".format(i+1))
_result = self.check_fragmentation_using_magma_stats(
self.buckets[0],
self.cluster.nodes_in_cluster)
self.assertIs(_result, True,
msg_stats.format("magma"))
_r = self.check_fragmentation_using_bucket_stats(
self.buckets[0], self.cluster.nodes_in_cluster)
self.assertIs(_r, True,
msg_stats.format("KV"))
disk_usage = self.get_disk_usage(self.buckets[0],
self.cluster.nodes_in_cluster)
_res = disk_usage[0]
self.log.info("Delete Iteration {}, Disk Usage- {}MB\
".format(i+1, _res))
self.assertIs(
_res > 0.5 * self.disk_usage[
self.disk_usage.keys()[0]],
False, msg.format(
"delete", i+1, _res, 0.5,
self.disk_usage[self.disk_usage.keys()[0]]))
###################################################################
'''
STEP - 5
-- ReCreation of docs.
'''
self.log.debug("Step 5, Iteration= {}".format(i+1))
self.gen_create = copy.deepcopy(self.gen_delete)
self.doc_ops = "create"
_ = self.loadgen_docs(self.retry_exceptions,
self.ignore_exceptions,
_sync=True)
self.bucket_util._wait_for_stats_all_buckets(timeout=3600)
self.bucket_util.verify_stats_all_buckets(self.num_items)
###################################################################
'''
STEP - 6
-- Space Amplification Check after Recreation.
'''
self.log.debug("Step 6, Iteration= {}".format(i+1))
_result = self.check_fragmentation_using_magma_stats(
self.buckets[0],
self.cluster.nodes_in_cluster)
self.assertIs(_result, True,
msg_stats.format("magma"))
_r = self.check_fragmentation_using_bucket_stats(
self.buckets[0], self.cluster.nodes_in_cluster)
self.assertIs(_r, True,
msg_stats.format("KV"))
disk_usage = self.get_disk_usage(self.buckets[0],
self.cluster.nodes_in_cluster)
_res = disk_usage[0]
self.log.info("Create Iteration{}, Disk Usage= {}MB \
".format(i+1, _res))
self.assertIs(_res > 1.5 * self.disk_usage[
self.disk_usage.keys()[0]],
False, msg.format("Create", _res, i+1, 1.5,
self.disk_usage[self.disk_usage.keys()[0]]))
###################################################################
'''
STEP - 7
-- Validate data
-- Data validation is only for the creates in last iterations.
'''
self.log.debug("Step 7, Iteration= {}".format(i+1))
self.validate_data("create", self.gen_create)
self.log.info("====test_update_rev_update ends====")
def test_update_single_doc_n_times(self):
"""
Test Focus: Update single/same doc n times
Note: MultiThreading is used to update
single doc, since we are not worried
about what should be the final mutate
value of the document semaphores have
been avoided. MultiThreading also speed up
the execution of test
"""
self.log.info("test_update_single_doc_n_times starts")
self.doc_ops = "update"
self.client = SDKClient([self.cluster.master],
self.bucket_util.buckets[0],
scope=CbServer.default_scope,
collection=CbServer.default_collection)
self.gen_update = self.genrate_docs_basic(start=0, end=1)
key, val = self.gen_update.next()
for node in self.cluster.nodes_in_cluster:
shell = RemoteMachineShellConnection(node)
shell.restart_couchbase()
shell.disconnect()
self.assertTrue(
self.bucket_util._wait_warmup_completed(
[self.cluster_util.cluster.master],
self.bucket_util.buckets[0],
wait_time=self.wait_timeout * 10))
def upsert_doc(start_num, end_num, key_obj, val_obj):
for i in range(start_num, end_num):
val_obj.put("mutated", i)
self.client.upsert(key_obj, val_obj)
threads = []
start = 0
end = 0
for _ in range(10):
start = end
end += 100000
th = threading.Thread(
target=upsert_doc, args=[start, end, key, val])
th.start()
threads.append(th)
for th in threads:
th.join()
self.bucket_util._wait_for_stats_all_buckets(timeout=3600)
# Space amplification check
msg_stats = "Fragmentation value for {} stats exceeds\
the configured value"
_result = self.check_fragmentation_using_magma_stats(
self.buckets[0],
self.cluster.nodes_in_cluster)
self.assertIs(_result, True,
msg_stats.format("magma"))
_r = self.check_fragmentation_using_bucket_stats(
self.buckets[0], self.cluster.nodes_in_cluster)
self.assertIs(_r, True,
msg_stats.format("KV"))
disk_usage = self.get_disk_usage(
self.buckets[0],
self.cluster.nodes_in_cluster)
self.log.debug("Disk usage after updates {}".format(
disk_usage))
_res = disk_usage[0]
msg = "Disk Usage = {}MB exceeds 2.2 times \
from Actual disk usage = {}MB"
self.assertIs(
_res > 2.2 * self.disk_usage[
self.disk_usage.keys()[0]],
False,
msg.format(_res, self.disk_usage[self.disk_usage.keys()[0]]))
# Space amplification check ends
success, fail = self.client.get_multi([key],
self.wait_timeout)
self.assertIs(key in success, True,
msg="key {} doesn't exist\
".format(key))
actual_val = dict()
expected_val = Json.loads(val.toString())
actual_val = Json.loads(success[key][
'value'].toString())
self.log.debug("Expected_val= {} and actual_val = {}\
".format(expected_val, actual_val))
self.assertIs(expected_val == actual_val, True,
msg="expected_val-{} != Actual_val-{}\
".format(expected_val, actual_val))
self.change_swap_space(self.cluster.nodes_in_cluster,
disable=False)
self.log.info("====test_update_single_doc_n_times ends====")
def test_move_val_btwn_key_and_seq_trees(self):
"""
Test Focus: Update items such that values moves
Sequence Tree and Key Trees.
STEPS:
-- Update items with new size , so that
items move from sequence tree to key
tree or vice versa
-- Do data validation
-- Again update items with initial size
-- Check space amplification
-- Again validate documents
"""
self.log.info("test_move_val_btwn_key_and_seq_trees starts")
msg_stats = "Fragmentation value for {} stats exceeds\
the configured value"
count = 0
keyTree, seqTree = (self.get_disk_usage(
self.buckets[0],
self.cluster.nodes_in_cluster)[2:4])
self.log.debug("Disk usage after pure creates {}".format((
self.disk_usage, keyTree, seqTree)))
initial_doc_size = self.doc_size
upsert_size = 0
if self.doc_size < 32:
upsert_size = 2048
while count < self.test_itr:
self.log.info("Update Iteration count == {}".format(count))
for node in self.cluster.nodes_in_cluster:
shell = RemoteMachineShellConnection(node)
shell.kill_memcached()
shell.disconnect()
self.assertTrue(self.bucket_util._wait_warmup_completed(
[self.cluster_util.cluster.master],
self.bucket_util.buckets[0],
wait_time=self.wait_timeout * 10))
#######################################################################
'''
STEP - 1, Update items with changed/new size
'''
self.log.info("Step 1, Iteration= {}".format(count+1))
self.doc_ops = "update"
self.update_start = 0
self.update_end = self.num_items
if self.rev_update:
self.update_start = -int(self.num_items - 1)
self.update_end = 1
self.doc_size = upsert_size
self.generate_docs()
_ = self.loadgen_docs(self.retry_exceptions,
self.ignore_exceptions,
_sync=True)
self.bucket_util._wait_for_stats_all_buckets(timeout=3600)
if upsert_size > 32:
seqTree_update = (self.get_disk_usage(
self.buckets[0],
self.cluster.nodes_in_cluster)[-1])
self.log.info("For upsert_size > 32 seqIndex usage-{}\
".format(seqTree_update))
#######################################################################
'''
STEP - 2, Validate data after initial upsert
'''
self.log.info("Step 2, Iteration= {}".format(count+1))
self.validate_data("update", self.gen_update)
#######################################################################
'''
STEP - 3, Updating items with changed doc size
to move between tress
'''
self.log.info("Step 3, Iteration= {}".format(count+1))
self.doc_size = initial_doc_size
self.generate_docs()
_ = self.loadgen_docs(self.retry_exceptions,
self.ignore_exceptions,
_sync=True)
self.bucket_util._wait_for_stats_all_buckets(timeout=3600)
#######################################################################
'''
STEP - 4, Space Amplification Checks
'''
_result = self.check_fragmentation_using_magma_stats(
self.buckets[0],
self.cluster.nodes_in_cluster)
self.assertIs(_result, True,
msg_stats.format("magma"))
_r = self.check_fragmentation_using_bucket_stats(
self.buckets[0], self.cluster.nodes_in_cluster)
self.assertIs(_r, True,
msg_stats.format("KV"))
disk_usage = self.get_disk_usage(
self.buckets[0], self.cluster.nodes_in_cluster)
_res = disk_usage[0]
self.log.info("disk usage after upsert count {} is {}MB \
".format(count+1, _res))
if self.doc_size > 32:
self.assertIs(
_res > 1.5 * self.disk_usage[self.disk_usage.keys()[0]],
False, "Disk Usage {} After \
update count {} exceeds \
Actual disk usage {} by 1.5 \
times".format(_res, count+1,
self.disk_usage[self.disk_usage.keys()[0]]))
else:
self.assertIs(disk_usage[3] > 0.5 * seqTree_update,
False, " Current seqTree usage-{} exceeds by'\n'\
0.5 times from the earlier '\n' \
seqTree usage (after update) -{} \
".format(disk_usage[3], seqTree_update))
count += 1
#######################################################################
'''
STEP - 5, Data validation
'''
self.log.info("Step 5, Iteration= {}".format(count+1))
self.validate_data("update", self.gen_update)
#######################################################################
self.change_swap_space(self.cluster.nodes_in_cluster, disable=False)
self.log.info("====test_move_docs_btwn_key_and_seq_trees ends====")
def test_parallel_create_update(self):
"""
STEPS:
-- Create new items and update already
existing items
-- Check disk_usage after each Iteration
-- Data validation for last iteration
"""
self.log.info("test_parallel_create_update starts")
count = 0
init_items = self.num_items
self.doc_ops = "create:update"
self.update_start = 0
self.update_end = self.num_items
while count < self.test_itr:
self.log.info("Iteration {}".format(count+1))
self.create_start = self.num_items
self.create_end = self.num_items+init_items
if self.rev_write:
self.create_start = -int(self.num_items+init_items - 1)
self.create_end = -int(self.num_items - 1)
self.generate_docs()
_ = self.loadgen_docs(self.retry_exceptions,
self.ignore_exceptions,
_sync=True)
self.bucket_util._wait_for_stats_all_buckets(timeout=3600)
self.bucket_util.verify_stats_all_buckets(self.num_items)
if count == self.test_itr - 1:
self.validate_data("update", self.gen_update)
self.update_start = self.num_items
self.update_end = self.num_items+init_items
if self.rev_update:
self.update_start = -int(self.num_items+init_items - 1)
self.update_end = -int(self.num_items - 1)
disk_usage = self.get_disk_usage(
self.buckets[0],
self.cluster.nodes_in_cluster)
if self.doc_size <= 32:
self.assertIs(
disk_usage[2] >= disk_usage[3], True,
"seqIndex usage = {}MB'\n' \
after Iteration {}'\n' \
exceeds keyIndex usage={}MB'\n' \
".format(disk_usage[3],
count+1,
disk_usage[2]))
self.assertIs(
disk_usage[0] > 2.2 * (2 * self.disk_usage[
self.disk_usage.keys()[0]]),
False, "Disk Usage {}MB After '\n\'\
Updates exceeds '\n\'\
Actual disk usage {}MB by '\n'\
2.2 times".format(disk_usage[0],
(2 * self.disk_usage[
self.disk_usage.keys()[0]])))
count += 1
self.change_swap_space(self.cluster.nodes_in_cluster, disable=False)
self.log.info("====test_parallel_create_update ends====")
