def incremental_rebalance_in_out_with_mutation_and_deletion(self):
self.cluster.rebalance(self.servers[:self.num_servers],
self.servers[1:self.num_servers], [])
gen_delete = BlobGenerator('mike',
'mike-',
self.value_size,
start=self.num_items / 2,
end=self.num_items)
for i in reversed(range(self.num_servers)[self.num_servers / 2:]):
tasks = self._async_load_all_buckets(self.master, self.gen_update,
"update", 0)
tasks.extend(
self._async_load_all_buckets(self.master, gen_delete, "delete",
0))
self.cluster.rebalance(self.servers[:i], [],
self.servers[i:self.num_servers])
time.sleep(10)
self.cluster.rebalance(self.servers[:self.num_servers],
self.servers[i:self.num_servers], [])
for task in tasks:
task.result()
self._load_all_buckets(self.master, gen_delete, "create", 0)
self._wait_for_stats_all_buckets(self.servers[:self.num_servers])
self._verify_all_buckets(self.master, max_verify=self.max_verify)
self._verify_stats_all_buckets(self.servers[:self.num_servers])
def checks_tap_connections_tests(self):
servs_init = self.servers[:self.nodes_init]
servs_in = [self.servers[i + self.nodes_init] for i in range(self.nodes_in)]
servs_out = [self.servers[self.nodes_init - i - 1] for i in range(self.nodes_out)]
rest = RestConnection(self.master)
buckets_stats_before = {}
for bucket in self.buckets:
_, result = rest.get_bucket_stats_json(bucket)
buckets_stats_before[bucket.name] = result["op"]["samples"]["ep_tap_user_count"];
self.log.info("current nodes : {0}".format([node.id for node in rest.node_statuses()]))
self.log.info("adding nodes {0} to cluster".format(servs_in))
self.log.info("removing nodes {0} from cluster".format(servs_out))
result_nodes = set(servs_init + servs_in) - set(servs_out)
self.cluster.rebalance(servs_init[:self.nodes_init], servs_in, servs_out)
gen = BlobGenerator('mike2', 'mike2-', self.value_size, end=self.num_items)
self._load_all_buckets(self.master, gen, "create", 0)
self.verify_cluster_stats(result_nodes)
buckets_stats_after = {}
for bucket in self.buckets:
_, result = rest.get_bucket_stats_json(bucket)
buckets_stats_after[bucket.name] = result["op"]["samples"]["ep_tap_user_count"];
for stat in buckets_stats_after[bucket.name][len(buckets_stats_before[bucket.name]) - 1:]:
if stat != 0:
self.log.error("'ep_tap_user_count' for bucket '{0}' before test:{1}".format(bucket.name, buckets_stats_before[bucket.name]))
self.log.error("'ep_tap_user_count' for bucket '{0}' after test:{1}".format(bucket.name, buckets_stats_after[bucket.name]))
self.log.error("'ep_tap_user_count' != 0 as expected");
self.log.info("'ep_tap_user_count' for bucket '{0}' = 0 for the entire test".format(bucket.name));
def incremental_rebalance_out_in_with_mutation(self):
init_num_nodes = self.input.param("init_num_nodes", 1)
self.cluster.rebalance(self.servers[:self.num_servers],
self.servers[1:init_num_nodes], [])
gen = BlobGenerator('mike',
'mike-',
self.value_size,
end=self.num_items)
self._load_all_buckets(self.master, gen, "create", 0)
for i in range(self.num_servers):
tasks = self._async_load_all_buckets(self.master, gen, "update", 0)
self.cluster.rebalance(
self.servers[:self.num_servers],
self.servers[init_num_nodes:init_num_nodes + i + 1], [])
time.sleep(10)
self.cluster.rebalance(
self.servers[:self.num_servers], [],
self.servers[init_num_nodes:init_num_nodes + i + 1])
for task in tasks:
task.result()
self._wait_for_stats_all_buckets(self.servers[:init_num_nodes])
self._verify_all_buckets(self.master, max_verify=self.max_verify)
self._verify_stats_all_buckets(self.servers[:init_num_nodes])
def _load_doc_data_all_buckets(self,
op_type='create',
start=0,
end=0,
expiry=0):
loaded = False
count = 0
gen_load = BlobGenerator('observe',
'observe',
1024,
start=start,
end=end)
while not loaded and count < 60:
try:
self._load_all_buckets(self.servers[0], gen_load, op_type,
expiry)
loaded = True
except MemcachedError as error:
if error.status == 134:
loaded = False
self.log.error(
"Memcached error 134, wait for 5 seconds and then try again"
)
count += 1
time.sleep(5)
def collectinfo_test(self):
"""We use cbcollect_info to automatically collect the logs for server node
First we load some items to the node. Optionally you can do some mutation
against these items. Then we use cbcollect_info the automatically generate
the zip file containing all the logs about the node. We want to verify we have
all the log files according to the LOG_FILE_NAME_LIST and in stats.log, we have
stats for all the buckets we have created"""
gen_load = BlobGenerator('nosql',
'nosql-',
self.value_size,
end=self.num_items)
gen_update = BlobGenerator('nosql',
'nosql-',
self.value_size,
end=(self.num_items / 2 - 1))
gen_expire = BlobGenerator('nosql',
'nosql-',
self.value_size,
start=self.num_items / 2,
end=(self.num_items * 3 / 4 - 1))
gen_delete = BlobGenerator('nosql',
'nosql-',
self.value_size,
start=self.num_items * 3 / 4,
end=self.num_items)
self._load_all_buckets(self.master, gen_load, "create", 0)
if (self.doc_ops is not None):
if ("update" in self.doc_ops):
self._load_all_buckets(self.master, gen_update, "update", 0)
if ("delete" in self.doc_ops):
self._load_all_buckets(self.master, gen_delete, "delete", 0)
if ("expire" in self.doc_ops):
self._load_all_buckets(self.master, gen_expire, "update",
self.expire_time)
time.sleep(self.expire_time + 1)
self._wait_for_stats_all_buckets(self.servers[:self.num_servers])
self.shell.delete_files("%s.zip" % (self.log_filename))
self.shell.delete_files(
"cbcollect_info*"
) #This is the folder generated after unzip the log package
self.shell.execute_cbcollect_info("%s.zip" % (self.log_filename))
self.verify_results(self.log_filename)
def setUp(self):
super(BucketFlushTests, self).setUp()
self.nodes_in = self.input.param("nodes_in", 0)
self.value_size = self.input.param("value_size", 256)
self.gen_create = BlobGenerator('bucketflush',
'bucketflush-',
self.value_size,
end=self.num_items)
def _async_load_doc_data_all_buckets(self, op_type='create', start=0):
gen_load = BlobGenerator('warmup',
'warmup-',
self.data_size,
start=start,
end=self.num_items)
tasks = self._async_load_all_buckets(self.servers[0], gen_load,
op_type, 0)
return tasks
def rebalance_in_with_ops_batch(self):
gen_delete = BlobGenerator('mike', 'mike-', self.value_size, start=(self.num_items / 2 - 1), end=self.num_items)
gen_create = BlobGenerator('mike', 'mike-', self.value_size, start=self.num_items + 1, end=self.num_items * 3 / 2)
servs_in = [self.servers[i + 1] for i in range(self.nodes_in)]
rebalance = self.cluster.async_rebalance(self.servers[:1], servs_in, [])
if(self.doc_ops is not None):
# define which doc's ops will be performed during rebalancing
# allows multiple of them but one by one
if("update" in self.doc_ops):
self._load_all_buckets(self.servers[0], self.gen_update, "update", 0, 1, 4294967295, True, batch_size=20000, pause_secs=5, timeout_secs=180)
if("create" in self.doc_ops):
self._load_all_buckets(self.servers[0], gen_create, "create", 0, 1, 4294967295, True, batch_size=20000, pause_secs=5, timeout_secs=180)
if("delete" in self.doc_ops):
self._load_all_buckets(self.servers[0], gen_delete, "delete", 0, 1, 4294967295, True, batch_size=20000, pause_secs=5, timeout_secs=180)
rebalance.result()
self._wait_for_stats_all_buckets(self.servers[:self.nodes_in + 1])
self._verify_all_buckets(self.servers[0], 1, 1000, None, only_store_hash=True, batch_size=5000)
self._verify_stats_all_buckets(self.servers[:self.nodes_in + 1])
def rebalance_in_with_ops(self):
gen_delete = BlobGenerator('mike', 'mike-', self.value_size, start=self.num_items / 2, end=self.num_items)
gen_create = BlobGenerator('mike', 'mike-', self.value_size, start=self.num_items + 1, end=self.num_items * 3 / 2)
servs_in = [self.servers[i + 1] for i in range(self.nodes_in)]
rebalance = self.cluster.async_rebalance(self.servers[:1], servs_in, [])
if(self.doc_ops is not None):
# define which doc's ops will be performed during rebalancing
# allows multiple of them but one by one
if("update" in self.doc_ops):
self._load_all_buckets(self.master, self.gen_update, "update", 0)
if("create" in self.doc_ops):
self._load_all_buckets(self.master, gen_create, "create", 0)
if("delete" in self.doc_ops):
self._load_all_buckets(self.master, gen_delete, "delete", 0)
rebalance.result()
self._wait_for_stats_all_buckets(self.servers[:self.nodes_in + 1])
self._verify_all_buckets(self.master, max_verify=self.max_verify)
self._verify_stats_all_buckets(self.servers[:self.nodes_in + 1])
def online_cluster_upgrade(self):
self._install(self.servers[:self.src_init + self.dest_init ])
self.initial_version = self.upgrade_versions[0]
self._install(self.servers[self.src_init + self.dest_init:])
self.cluster.shutdown()
XDCRReplicationBaseTest.setUp(self)
bucket_default = self._get_bucket('default', self.src_master)
bucket_sasl = self._get_bucket('bucket0', self.src_master)
bucket_standard = self._get_bucket('standard_bucket0', self.dest_master)
self._load_bucket(bucket_default, self.src_master, self.gen_create, 'create', exp=0)
self._load_bucket(bucket_sasl, self.src_master, self.gen_create, 'create', exp=0)
self._load_bucket(bucket_standard, self.dest_master, self.gen_create, 'create', exp=0)
gen_create2 = BlobGenerator('loadTwo', 'loadTwo-', self._value_size, end=self._num_items)
self._load_bucket(bucket_sasl, self.dest_master, gen_create2, 'create', exp=0)
self._online_upgrade(self.src_nodes, self.servers[self.src_init + self.dest_init:])
self._install(self.src_nodes)
self._online_upgrade(self.servers[self.src_init + self.dest_init:], self.src_nodes, False)
self._load_bucket(bucket_default, self.src_master, self.gen_delete, 'delete', exp=0)
self._load_bucket(bucket_default, self.src_master, self.gen_update, 'create', exp=self._expires)
self._load_bucket(bucket_sasl, self.src_master, self.gen_delete, 'delete', exp=0)
self._load_bucket(bucket_sasl, self.src_master, self.gen_update, 'create', exp=self._expires)
self._online_upgrade(self.dest_nodes, self.servers[self.src_init + self.dest_init:])
self._install(self.dest_nodes)
self._online_upgrade(self.servers[self.src_init + self.dest_init:], self.dest_nodes, False)
self._load_bucket(bucket_standard, self.dest_master, self.gen_delete, 'delete', exp=0)
self._load_bucket(bucket_standard, self.dest_master, self.gen_update, 'create', exp=self._expires)
self.do_merge_bucket(self.src_master, self.dest_master, True, bucket_sasl)
bucket_sasl = self._get_bucket('bucket0', self.dest_master)
gen_delete2 = BlobGenerator('loadTwo', 'loadTwo-', self._value_size,
start=int((self._num_items) * (float)(100 - self._percent_delete) / 100), end=self._num_items)
gen_update2 = BlobGenerator('loadTwo', 'loadTwo-', self._value_size, start=0,
end=int(self._num_items * (float)(self._percent_update) / 100))
self._load_bucket(bucket_sasl, self.dest_master, gen_delete2, 'delete', exp=0)
self._load_bucket(bucket_sasl, self.dest_master, gen_update2, 'create', exp=self._expires)
self.do_merge_bucket(self.dest_master, self.src_master, False, bucket_sasl)
self.do_merge_bucket(self.src_master, self.dest_master, False, bucket_default)
self.do_merge_bucket(self.dest_master, self.src_master, False, bucket_standard)
self.verify_xdcr_stats(self.src_nodes, self.dest_nodes, True)
def rebalance_in_out_at_once_persistence_stopped(self):
num_nodes_with_stopped_persistence = self.input.param(
"num_nodes_with_stopped_persistence", 1)
servs_init = self.servers[:self.nodes_init]
servs_in = [
self.servers[i + self.nodes_init] for i in range(self.nodes_in)
]
servs_out = [
self.servers[self.nodes_init - i - 1]
for i in range(self.nodes_out)
]
rest = RestConnection(self.master)
self._wait_for_stats_all_buckets(servs_init)
for server in servs_init[:min(num_nodes_with_stopped_persistence, self.
nodes_init)]:
shell = RemoteMachineShellConnection(server)
for bucket in self.buckets:
shell.execute_cbepctl(bucket, "stop", "", "", "")
self.sleep(5)
self.num_items_without_persistence = self.input.param(
"num_items_without_persistence", 100000)
gen_extra = BlobGenerator('mike', 'mike-', self.value_size, start=self.num_items / 2\
, end=self.num_items / 2 + self.num_items_without_persistence)
self.log.info("current nodes : {0}".format(
[node.id for node in rest.node_statuses()]))
self.log.info("adding nodes {0} to cluster".format(servs_in))
self.log.info("removing nodes {0} from cluster".format(servs_out))
tasks = self._async_load_all_buckets(self.master,
gen_extra,
"create",
0,
batch_size=1000)
result_nodes = set(servs_init + servs_in) - set(servs_out)
# wait timeout in 60 min because MB-7386 rebalance stuck
self.cluster.rebalance(servs_init[:self.nodes_init],
servs_in,
servs_out,
timeout=self.wait_timeout * 60)
for task in tasks:
task.result()
self._wait_for_stats_all_buckets(servs_init[:self.nodes_init - self.nodes_out], \
ep_queue_size=self.num_items_without_persistence * 0.9, ep_queue_size_cond='>')
self._wait_for_stats_all_buckets(servs_in)
self._verify_all_buckets(self.master, timeout=None)
self._verify_stats_all_buckets(result_nodes)
#verify that curr_items_tot corresponds to sum of curr_items from all nodes
verified = True
for bucket in self.buckets:
verified &= RebalanceHelper.wait_till_total_numbers_match(
self.master, bucket)
self.assertTrue(
verified,
"Lost items!!! Replication was completed but sum(curr_items) don't match the curr_items_total"
)
def ops_change_cas(self):
"""CAS value manipulation by update, delete, expire test.
We load a certain number of items. Then for half of them, we use
MemcachedClient cas() method to mutate those item values in order
to change CAS value of those items.
We use MemcachedClient set() method to set a quarter of the items expired.
We also use MemcachedClient delete() to delete a quarter of the items"""
gen_load_mysql = BlobGenerator('nosql',
'nosql-',
self.value_size,
end=self.num_items)
gen_update = BlobGenerator('nosql',
'nosql-',
self.value_size,
end=(self.num_items / 2 - 1))
gen_delete = BlobGenerator('nosql',
'nosql-',
self.value_size,
start=self.num_items / 2,
end=(3 * self.num_items / 4 - 1))
gen_expire = BlobGenerator('nosql',
'nosql-',
self.value_size,
start=3 * self.num_items / 4,
end=self.num_items)
self._load_all_buckets(self.master,
gen_load_mysql,
"create",
0,
flag=self.item_flag)
if (self.doc_ops is not None):
if ("update" in self.doc_ops):
self.verify_cas("update", gen_update)
if ("delete" in self.doc_ops):
self.verify_cas("delete", gen_delete)
if ("expire" in self.doc_ops):
self.verify_cas("expire", gen_expire)
self._wait_for_stats_all_buckets(
[self.master]) #we only need 1 node to do cas test
def offline_cluster_upgrade(self):
self._install(self.servers[:self.src_init + self.dest_init ])
upgrade_nodes = self.input.param('upgrade_nodes', "src").split(";")
self.cluster.shutdown()
XDCRReplicationBaseTest.setUp(self)
bucket = self._get_bucket('default', self.src_master)
self._load_bucket(bucket, self.src_master, self.gen_create, 'create', exp=0)
bucket = self._get_bucket('bucket0', self.src_master)
self._load_bucket(bucket, self.src_master, self.gen_create, 'create', exp=0)
bucket = self._get_bucket('bucket0', self.dest_master)
gen_create2 = BlobGenerator('loadTwo', 'loadTwo', self._value_size, end=self._num_items)
self._load_bucket(bucket, self.dest_master, gen_create2, 'create', exp=0)
nodes_to_upgrade = []
if "src" in upgrade_nodes :
nodes_to_upgrade += self.src_nodes
if "dest" in upgrade_nodes :
nodes_to_upgrade += self.dest_nodes
for upgrade_version in self.upgrade_versions:
for server in nodes_to_upgrade:
remote = RemoteMachineShellConnection(server)
remote.stop_server()
remote.disconnect()
upgrade_threads = self._async_update(upgrade_version, nodes_to_upgrade)
#wait upgrade statuses
for upgrade_thread in upgrade_threads:
upgrade_thread.join()
success_upgrade = True
while not self.queue.empty():
success_upgrade &= self.queue.get()
if not success_upgrade:
self.fail("Upgrade failed!")
self.sleep(self.expire_time)
bucket = self._get_bucket('bucket0', self.src_master)
gen_create3 = BlobGenerator('loadThree', 'loadThree', self._value_size, end=self._num_items)
self._load_bucket(bucket, self.src_master, gen_create3, 'create', exp=0)
self.do_merge_bucket(self.src_master, self.dest_master, True, bucket)
bucket = self._get_bucket('default', self.src_master)
self._load_bucket(bucket, self.src_master, gen_create2, 'create', exp=0)
self.do_merge_bucket(self.src_master, self.dest_master, False, bucket)
self.verify_xdcr_stats(self.src_nodes, self.dest_nodes, True)
def setUp(self):
super(Rebalance, self).setUp()
if self._replication_direction_str in "bidirection":
self.gen_create2 = BlobGenerator('LoadTwo',
'LoadTwo',
self._value_size,
end=self._num_items)
self.gen_delete2 = BlobGenerator(
'LoadTwo',
'LoadTwo-',
self._value_size,
start=int((self._num_items) *
(float)(100 - self._percent_delete) / 100),
end=self._num_items)
self.gen_update2 = BlobGenerator(
'LoadTwo',
'LoadTwo-',
self._value_size,
start=0,
end=int(self._num_items * (float)(self._percent_update) / 100))
def setUp(self):
super(bidirectional, self).setUp()
self.gen_create2 = BlobGenerator('loadTwo',
'loadTwo',
self._value_size,
end=self._num_items)
self.gen_delete2 = BlobGenerator(
'loadTwo',
'loadTwo-',
self._value_size,
start=int(
(self._num_items) * (float)(100 - self._percent_delete) / 100),
end=self._num_items)
self.gen_update2 = BlobGenerator(
'loadTwo',
'loadTwo-',
self._value_size,
start=0,
end=int(self._num_items * (float)(self._percent_update) / 100))
def rebalance_in_with_ops(self):
gen_delete = BlobGenerator('mike',
'mike-',
self.value_size,
start=self.num_items / 2,
end=self.num_items)
gen_create = BlobGenerator('mike',
'mike-',
self.value_size,
start=self.num_items + 1,
end=self.num_items * 3 / 2)
servs_in = [
self.servers[i + self.nodes_init] for i in range(self.nodes_in)
]
if self.output_time:
start_time = time.time()
rebalance = self.cluster.async_rebalance(
self.servers[:self.nodes_init], servs_in, [])
if (self.doc_ops is not None):
tasks = []
# define which doc's ops will be performed during rebalancing
# allows multiple of them but one by one
if ("update" in self.doc_ops):
tasks += self._async_load_all_buckets(self.master,
self.gen_update,
"update", 0)
if ("create" in self.doc_ops):
tasks += self._async_load_all_buckets(self.master, gen_create,
"create", 0)
if ("delete" in self.doc_ops):
tasks += self._async_load_all_buckets(self.master, gen_delete,
"delete", 0)
for task in tasks:
task.result()
rebalance.result()
if self.output_time:
delta_time = time.time() - start_time
self.log.info("TIME FOR REBALANCE IS %s SECS (%s MINS)" %
(delta_time, delta_time / 60))
self.verify_cluster_stats(self.servers[:self.nodes_in +
self.nodes_init])
def _async_load_doc_data_all_buckets(self,
op_type='create',
start=0,
end=0):
gen_load = BlobGenerator('observe',
'observe',
1024,
start=start,
end=end)
tasks = self._async_load_all_buckets(self.servers[0], gen_load,
op_type, 0)
return tasks
def setUp(self):
super(BucketFlushTests, self).setUp()
self.nodes_in = self.input.param("nodes_in", 0)
self.value_size = self.input.param("value_size", 256)
self.data_op = self.input.param("data_op", "create")
self.use_ascii = self.input.param("use_ascii", "False")
self.gen_create = BlobGenerator('bucketflush', 'bucketflush-', self.value_size, end=self.num_items)
try:
self.default_test_setup()
except Exception, e:
self.cluster.shutdown()
self.fail(e)
def incremental_rebalance_out_with_mutation_and_deletion(self):
gen_2 = BlobGenerator('mike', 'mike-', self.value_size, start=self.num_items / 2,
end=self.num_items)
for i in reversed(range(self.num_servers)[1:]):
rebalance = self.cluster.async_rebalance(self.servers[:i], [], [self.servers[i]])
self._load_all_buckets(self.master, self.gen_update, "update", 0)
self._load_all_buckets(self.master, gen_2, "delete", 0)
rebalance.result()
self._load_all_buckets(self.master, gen_2, "create", 0)
self._wait_for_stats_all_buckets(self.servers[:i])
self._verify_all_buckets(self.master, max_verify=self.max_verify)
self._verify_stats_all_buckets(self.servers[:i])
def checkpoint_failover_master(self):
"""Load N items. During the load, failover Master.
Verify backfill doesn't happen on R1, R2."""
param = 'checkpoint'
stat_key = 'vb_0:open_checkpoint_id'
rest = RestConnection(self.master)
nodes = rest.node_statuses()
failover_node = None
for node in nodes:
if node.id.find(self.master.ip) >= 0:
failover_node = node
self._set_checkpoint_size(self.servers[:self.num_servers], self.bucket,
self.checkpoint_size)
generate_load = BlobGenerator('nosql',
'nosql-',
self.value_size,
end=self.num_items)
data_load_thread = Thread(target=self._load_all_buckets,
name="load_data",
args=(self.master, generate_load, "create",
0, 1, 0, True, self.checkpoint_size, 5,
180))
data_load_thread.start()
time.sleep(5)
prev_backfill_timestamp_R1 = self._get_backfill_timestamp(
self.replica1, self.replica2)
prev_backfill_timestamp_R2 = self._get_backfill_timestamp(
self.replica2, self.replica3)
failed_over = rest.fail_over(failover_node.id)
if not failed_over:
self.log.info(
"unable to failover the node the first time. try again in 60 seconds.."
)
#try again in 60 seconds
time.sleep(75)
failed_over = rest.fail_over(failover_node.id)
self.assertTrue(failed_over,
"unable to failover node %s" % (self.master.ip))
self.log.info("failed over node : {0}".format(failover_node.id))
data_load_thread.join()
self._verify_backfill_happen(self.replica1, self.replica2,
prev_backfill_timestamp_R1)
self._verify_backfill_happen(self.replica2, self.replica3,
prev_backfill_timestamp_R2)
self.cluster.rebalance(self.servers[:self.num_servers], [],
[self.master])
self.cluster.rebalance(self.servers[1:self.num_servers], [self.master],
[])
def checkpoint_replication_pause(self):
"""With 3 replicas load data. pause replication to R2. Let checkpoints close on Master and R1.
Restart replication of R2 and R3, backfill should not be seen on R1 and R2."""
param = 'checkpoint'
stat_key = 'vb_0:last_closed_checkpoint_id'
self._set_checkpoint_size(self.servers[:self.num_servers], self.bucket,
str(self.checkpoint_size))
time.sleep(5)
prev_backfill_timestamp_R1 = self._get_backfill_timestamp(
self.replica1, self.replica2)
prev_backfill_timestamp_R2 = self._get_backfill_timestamp(
self.replica2, self.replica3)
generate_load = BlobGenerator('nosql',
'nosql-',
self.value_size,
end=self.num_items)
data_load_thread = Thread(target=self._load_all_buckets,
name="load_data",
args=(self.master, generate_load, "create",
0, 1, 0, True, self.checkpoint_size, 5,
180))
data_load_thread.start()
self._stop_replication(self.replica2, self.bucket)
m_stats = StatsCommon.get_stats([self.master], self.bucket, param,
stat_key)
chk_pnt = int(m_stats[m_stats.keys()[0]]) + 2
tasks = []
tasks.append(
self.cluster.async_wait_for_stats([self.master], self.bucket,
param, stat_key, '>=', chk_pnt))
tasks.append(
self.cluster.async_wait_for_stats([self.replica1], self.bucket,
param, stat_key, '>=', chk_pnt))
for task in tasks:
try:
task.result(60)
except TimeoutError:
self.fail("Checkpoint not closed")
data_load_thread.join()
self._start_replication(self.replica2, self.bucket)
self._verify_checkpoint_id(param, stat_key, m_stats)
self._verify_stats_all_buckets(self.servers[:self.num_servers])
self._verify_backfill_happen(self.replica1, self.replica2,
prev_backfill_timestamp_R1)
self._verify_backfill_happen(self.replica2, self.replica3,
prev_backfill_timestamp_R2)
def incremental_rebalance_in_with_ops(self):
for i in range(1, self.num_servers, 2):
rebalance = self.cluster.async_rebalance(self.servers[:i], self.servers[i:i + 2], [])
if self.doc_ops is not None:
# define which doc's operation will be performed during rebalancing
#only one type of ops can be passed
if("update" in self.doc_ops):
# 1/2th of data will be updated in each iteration
self._load_all_buckets(self.master, self.gen_update, "update", 0)
elif("create" in self.doc_ops):
# 1/2th of initial data will be added in each iteration
gen_create = BlobGenerator('mike', 'mike-', self.value_size, start=self.num_items * (1 + i) / 2.0 , end=self.num_items * (1 + i / 2.0))
self._load_all_buckets(self.master, gen_create, "create", 0)
elif("delete" in self.doc_ops):
# 1/(num_servers) of initial data will be removed after each iteration
# at the end we should get empty base( or couple items)
gen_delete = BlobGenerator('mike', 'mike-', self.value_size, start=int(self.num_items * (1 - i / (self.num_servers - 1.0))) + 1, end=int(self.num_items * (1 - (i - 1) / (self.num_servers - 1.0))))
self._load_all_buckets(self.master, gen_delete, "delete", 0)
rebalance.result()
self._wait_for_stats_all_buckets(self.servers[:i + 2])
self._verify_all_buckets(self.master, max_verify=self.max_verify)
self._verify_stats_all_buckets(self.servers[:i + 2])
def setUp(self):
log = logger.Logger.get_logger()
self._input = TestInputSingleton.input
self._keys_count = self._input.param("keys_count", DEFAULT_KEY_COUNT)
self._num_replicas = self._input.param("replica", DEFAULT_REPLICA)
self.bidirectional = self._input.param("bidirectional", False)
self.case_number = self._input.param("case_number", 0)
self._value_size = self._input.param("value_size", 256)
self.wait_timeout = self._input.param("wait_timeout", 60)
self._servers = self._input.servers
self.master = self._servers[0]
self._failed_nodes = []
num_buckets = 0
self.buckets = []
self.default_bucket = self._input.param("default_bucket", True)
if self.default_bucket:
self.default_bucket_name = "default"
num_buckets += 1
self._standard_buckets = self._input.param("standard_buckets", 0)
self._sasl_buckets = self._input.param("sasl_buckets", 0)
num_buckets += self._standard_buckets + self._sasl_buckets
self.dgm_run = self._input.param("dgm_run", True)
self.log = logger.Logger().get_logger()
self._cluster_helper = Cluster()
self.disabled_consistent_view = self._input.param(
"disabled_consistent_view", None)
self._quota = self._initialize_nodes(self._cluster_helper,
self._servers,
self.disabled_consistent_view)
if self.dgm_run:
self.quota = 256
self.bucket_size = int(
(2.0 / 3.0) / float(num_buckets) * float(self._quota))
self.gen_create = BlobGenerator('loadOne',
'loadOne_',
self._value_size,
end=self._keys_count)
self.add_back_flag = False
self._cleanup_nodes = []
log.info("============== setup was started for test #{0} {1}=============="\
.format(self.case_number, self._testMethodName))
RemoteUtilHelper.common_basic_setup(self._servers)
BucketOperationHelper.delete_all_buckets_or_assert(self._servers, self)
for server in self._servers:
ClusterOperationHelper.cleanup_cluster([server])
ClusterHelper.wait_for_ns_servers_or_assert(self._servers, self)
self._setup_cluster()
self._create_buckets_()
log.info("============== setup was finished for test #{0} {1} =============="\
.format(self.case_number, self._testMethodName))
def incremental_rebalance_in_with_mutation_and_expiration(self):
gen_2 = BlobGenerator('mike',
'mike-',
self.value_size,
start=self.num_items / 2,
end=self.num_items)
for i in range(self.num_servers)[1:]:
rebalance = self.cluster.async_rebalance(self.servers[:i],
[self.servers[i]], [])
self._load_all_buckets(self.master, self.gen_update, "update", 0)
self._load_all_buckets(self.master, gen_2, "update", 5)
self.sleep(5)
rebalance.result()
self._load_all_buckets(self.master, gen_2, "create", 0)
self.verify_cluster_stats(self.servers[:i + 1])
def rebalance_out_with_ops(self):
gen_delete = BlobGenerator('mike',
'mike-',
self.value_size,
start=self.num_items / 2,
end=self.num_items)
gen_create = BlobGenerator('mike',
'mike-',
self.value_size,
start=self.num_items + 1,
end=self.num_items * 3 / 2)
servs_out = [
self.servers[self.num_servers - i - 1]
for i in range(self.nodes_out)
]
rebalance = self.cluster.async_rebalance(self.servers[:1], [],
servs_out)
# define which doc's ops will be performed during rebalancing
# allows multiple of them but one by one
tasks = []
if (self.doc_ops is not None):
if ("update" in self.doc_ops):
tasks += self._async_load_all_buckets(self.master,
self.gen_update,
"update", 0)
if ("create" in self.doc_ops):
tasks += self._async_load_all_buckets(self.master, gen_create,
"create", 0)
if ("delete" in self.doc_ops):
tasks += self._async_load_all_buckets(self.master, gen_delete,
"delete", 0)
for task in tasks:
task.result()
rebalance.result()
self.verify_cluster_stats(self.servers[:self.num_servers -
self.nodes_out])
def _load_gen_data(self, cname, node):
for op_type in self._seed_data_ops_lst:
num_items_ratio = self._get_num_items_ratio(op_type)
load_gen = BlobGenerator(cname,
cname,
self._value_size,
end=num_items_ratio)
self._log.info(
"Starting Load operation '{0}' for items (ratio) '{1}' on node '{2}'...."
.format(op_type, num_items_ratio, cname))
if self._seed_data_mode_str == XDCRConstants.SEED_DATA_MODE_SYNC:
self._load_all_buckets(node, load_gen, op_type, 0)
self._log.info("Completed Load of {0}".format(op_type))
else:
self._async_load_all_buckets(node, load_gen, op_type, 0)
self._log.info("Started async Load of {0}".format(op_type))
def rebalance_in_out_at_once_with_max_buckets_number(self):
servs_init = self.servers[:self.nodes_init]
servs_in = [
self.servers[i + self.nodes_init] for i in range(self.nodes_in)
]
servs_out = [
self.servers[self.nodes_init - i - 1]
for i in range(self.nodes_out)
]
rest = RestConnection(self.master)
self._wait_for_stats_all_buckets(servs_init)
self.log.info("current nodes : {0}".format(
[node.id for node in rest.node_statuses()]))
self.log.info("adding nodes {0} to cluster".format(servs_in))
self.log.info("removing nodes {0} from cluster".format(servs_out))
result_nodes = set(servs_init + servs_in) - set(servs_out)
rest = RestConnection(self.master)
bucket_num = rest.get_internalSettings("maxBucketCount")
self.bucket_size = self.quota / bucket_num
self.log.info('total %s buckets will be created with size %s MB' %
(bucket_num, self.bucket_size))
self.cluster.create_default_bucket(self.master, self.bucket_size,
self.num_replicas)
self.buckets.append(
Bucket(name="default",
authType="sasl",
saslPassword="",
num_replicas=self.num_replicas,
bucket_size=self.bucket_size))
self._create_sasl_buckets(self.master, (bucket_num - 1) / 2)
self._create_standard_buckets(self.master,
bucket_num - 1 - (bucket_num - 1) / 2)
gen = BlobGenerator('mike',
'mike-',
self.value_size,
end=self.num_items)
self._load_all_buckets(self.master, gen, "create", 0)
self._wait_for_stats_all_buckets(servs_init)
rebalance = self.cluster.async_rebalance(servs_init, servs_in,
servs_out)
self._async_load_all_buckets(self.master, gen, "update", 0)
rebalance.result()
self.verify_cluster_stats(result_nodes)
def checkpoint_server_down(self):
"""Load N items. Shut down server R2. Then Restart R2 and
verify backfill happens on R1 and R2."""
param = 'checkpoint'
stat_key = 'vb_0:open_checkpoint_id'
rest = RestConnection(self.master)
self._set_checkpoint_size(self.servers[:self.num_servers], self.bucket,
self.checkpoint_size)
generate_load_one = BlobGenerator('nosql',
'nosql-',
self.value_size,
end=self.num_items)
self._load_all_buckets(self.master,
generate_load_one,
"create",
0,
1,
0,
True,
batch_size=self.checkpoint_size,
pause_secs=5,
timeout_secs=180)
self._wait_for_stats_all_buckets(self.servers[:self.num_servers])
prev_backfill_timestamp_R1 = self._get_backfill_timestamp(
self.replica1, self.replica2)
prev_backfill_timestamp_R2 = self._get_backfill_timestamp(
self.replica2, self.replica3)
m_stats = StatsCommon.get_stats([self.master], self.bucket, param,
stat_key)
self._stop_server(self.replica2)
time.sleep(5)
data_load_thread = Thread(target=self._load_data_use_workloadgen,
name="load_data",
args=(self.master, ))
data_load_thread.start()
data_load_thread.join()
self._start_server(self.replica2)
time.sleep(5)
self._verify_checkpoint_id(param, stat_key, m_stats)
self._verify_backfill_happen(self.replica1, self.replica2,
prev_backfill_timestamp_R1, True)
self._verify_backfill_happen(self.replica2, self.replica3,
prev_backfill_timestamp_R2, True)
def incremental_rebalance_in_out_with_mutation(self):
self.cluster.rebalance(self.servers[:self.num_servers],
self.servers[1:self.num_servers], [])
gen = BlobGenerator('mike', 'mike-', self.value_size, end=self.num_items)
self._load_all_buckets(self.master, gen, "create", 0)
for i in reversed(range(self.num_servers)[self.num_servers / 2:]):
tasks = self._async_load_all_buckets(self.master, gen, "update", 0)
self.cluster.rebalance(self.servers[:i], [], self.servers[i:self.num_servers])
time.sleep(5)
for task in tasks:
task.result(self.wait_timeout * 20)
tasks = self._async_load_all_buckets(self.master, gen, "update", 0)
self.cluster.rebalance(self.servers[:self.num_servers],
self.servers[i:self.num_servers], [])
for task in tasks:
task.result(self.wait_timeout * 20)
self.verify_cluster_stats(self.servers[:self.num_servers])
def test_start_compaction(self):
# disable auto compaction
self.disable_compaction()
# create ddoc and add views
ddoc_name = "ddoc1"
views = self.make_default_views("test_add_views", 3)
server = self.servers[0]
self.create_views(server, ddoc_name, views)
# load initial documents
gen_load = BlobGenerator('test_view_compaction',
'test_view_compaction-',
self.value_size,
end=self.num_items)
self._load_all_buckets(server, gen_load, "create", 0)
# start fragmentation monitor
fragmentation_monitor = \
self.cluster.async_monitor_view_fragmentation(server,
ddoc_name,
self.fragmentation_value,
timeout = 20)
# generate load until fragmentation reached
while fragmentation_monitor.state != "FINISHED":
# update docs to create fragmentation
self._load_all_buckets(server, gen_load, "update", 0)
for view in views:
# run queries to create indexes
self.cluster.query_view(server, ddoc_name, view.name, {})
fragmentation_monitor.result()
# compact ddoc and make sure fragmentation is less than high_mark
# will throw exception if failed
result = self.cluster.compact_view(server, ddoc_name)
self.assertTrue(result)
