def online_cluster_upgrade(self): self._install(self.servers[:self.src_init + self.dest_init ]) self.initial_version = self.upgrade_versions[0] self._install(self.servers[self.src_init + self.dest_init:]) self.cluster.shutdown(force=True) XDCRReplicationBaseTest.setUp(self) bucket_default = self._get_bucket(self, 'default', self.src_master) bucket_sasl = self._get_bucket(self, 'bucket0', self.src_master) bucket_standard = self._get_bucket(self, 'standard_bucket0', self.dest_master) self._load_bucket(bucket_default, self.src_master, self.gen_create, 'create', exp=0) self._load_bucket(bucket_sasl, self.src_master, self.gen_create, 'create', exp=0) self._load_bucket(bucket_standard, self.dest_master, self.gen_create, 'create', exp=0) gen_create2 = BlobGenerator('loadTwo', 'loadTwo-', self._value_size, end=self._num_items) self._load_bucket(bucket_sasl, self.dest_master, gen_create2, 'create', exp=0) self._online_upgrade(self.src_nodes, self.servers[self.src_init + self.dest_init:]) self._install(self.src_nodes) self._online_upgrade(self.servers[self.src_init + self.dest_init:], self.src_nodes, False) self._load_bucket(bucket_default, self.src_master, self.gen_delete, 'delete', exp=0) self._load_bucket(bucket_default, self.src_master, self.gen_update, 'create', exp=self._expires) self._load_bucket(bucket_sasl, self.src_master, self.gen_delete, 'delete', exp=0) self._load_bucket(bucket_sasl, self.src_master, self.gen_update, 'create', exp=self._expires) self.sleep(120) self._online_upgrade(self.dest_nodes, self.servers[self.src_init + self.dest_init:]) self._install(self.dest_nodes) self._online_upgrade(self.servers[self.src_init + self.dest_init:], self.dest_nodes, False) self._load_bucket(bucket_standard, self.dest_master, self.gen_delete, 'delete', exp=0) self._load_bucket(bucket_standard, self.dest_master, self.gen_update, 'create', exp=self._expires) self.do_merge_bucket(self.src_master, self.dest_master, True, bucket_sasl) bucket_sasl = self._get_bucket(self, 'bucket0', self.dest_master) gen_delete2 = BlobGenerator('loadTwo', 'loadTwo-', self._value_size, start=int((self._num_items) * (float)(100 - self._percent_delete) / 100), end=self._num_items) gen_update2 = BlobGenerator('loadTwo', 'loadTwo-', self._value_size, start=0, end=int(self._num_items * (float)(self._percent_update) / 100)) self._load_bucket(bucket_sasl, self.dest_master, gen_delete2, 'delete', exp=0) self._load_bucket(bucket_sasl, self.dest_master, gen_update2, 'create', exp=self._expires) self.do_merge_bucket(self.dest_master, self.src_master, False, bucket_sasl) self.do_merge_bucket(self.src_master, self.dest_master, False, bucket_default) self.do_merge_bucket(self.dest_master, self.src_master, False, bucket_standard) self.sleep(120) self._post_upgrade_ops() self.sleep(120) self.verify_xdcr_stats(self.src_nodes, self.dest_nodes, True) self.max_verify = None if self.ddocs_src: for bucket_name in self.buckets_on_src: bucket = self._get_bucket(self, bucket_name, self.src_master) expected_rows = sum([len(kv_store) for kv_store in bucket.kvs.values()]) self._verify_ddocs(expected_rows, [bucket_name], self.ddocs_src, self.src_master) if self.ddocs_dest: for bucket_name in self.buckets_on_dest: bucket = self._get_bucket(self, bucket_name, self.dest_master) expected_rows = sum([len(kv_store) for kv_store in bucket.kvs.values()]) self._verify_ddocs(expected_rows, [bucket_name], self.ddocs_dest, self.dest_master)
def offline_cluster_upgrade(self): self._install(self.servers[:self.src_init + self.dest_init ]) upgrade_nodes = self.input.param('upgrade_nodes', "src").split(";") self.cluster.shutdown(force=True) XDCRReplicationBaseTest.setUp(self) self.set_xdcr_param('xdcrFailureRestartInterval', 1) self.sleep(60) bucket = self._get_bucket(self, 'default', self.src_master) self._operations() self._load_bucket(bucket, self.src_master, self.gen_create, 'create', exp=0) bucket = self._get_bucket(self, 'bucket0', self.src_master) self._load_bucket(bucket, self.src_master, self.gen_create, 'create', exp=0) bucket = self._get_bucket(self, 'bucket0', self.dest_master) gen_create2 = BlobGenerator('loadTwo', 'loadTwo', self._value_size, end=self._num_items) self._load_bucket(bucket, self.dest_master, gen_create2, 'create', exp=0) nodes_to_upgrade = [] if "src" in upgrade_nodes : nodes_to_upgrade += self.src_nodes if "dest" in upgrade_nodes : nodes_to_upgrade += self.dest_nodes self.sleep(60) self._wait_for_replication_to_catchup() self._offline_upgrade(nodes_to_upgrade) if self._use_encryption_after_upgrade and "src" in upgrade_nodes and "dest" in upgrade_nodes and self.upgrade_versions[0] >= "2.5.0": if "src" in self._use_encryption_after_upgrade: src_remote_clusters = RestConnection(self.src_master).get_remote_clusters() for remote_cluster in src_remote_clusters: self._modify_clusters(None, self.src_master, remote_cluster['name'], self.dest_master, require_encryption=1) if "dest" in self._use_encryption_after_upgrade: dest_remote_clusters = RestConnection(self.dest_master).get_remote_clusters() for remote_cluster in dest_remote_clusters: self._modify_clusters(None, self.dest_master, remote_cluster['name'], self.src_master, require_encryption=1) self.set_xdcr_param('xdcrFailureRestartInterval', 1) self.sleep(60) bucket = self._get_bucket(self, 'bucket0', self.src_master) gen_create3 = BlobGenerator('loadThree', 'loadThree', self._value_size, end=self._num_items) self._load_bucket(bucket, self.src_master, gen_create3, 'create', exp=0) self.do_merge_bucket(self.src_master, self.dest_master, True, bucket) bucket = self._get_bucket(self, 'default', self.src_master) self._load_bucket(bucket, self.src_master, gen_create2, 'create', exp=0) self.do_merge_bucket(self.src_master, self.dest_master, False, bucket) self.sleep(60) self._post_upgrade_ops() self.sleep(60) self.verify_xdcr_stats(self.src_nodes, self.dest_nodes, True) self.max_verify = None if self.ddocs_src: for bucket_name in self.buckets_on_src: bucket = self._get_bucket(self, bucket_name, self.src_master) expected_rows = sum([len(kv_store) for kv_store in bucket.kvs.values()]) self._verify_ddocs(expected_rows, [bucket_name], self.ddocs_src, self.src_master) if self.ddocs_dest: for bucket_name in self.buckets_on_dest: bucket = self._get_bucket(self, bucket_name, self.dest_master) expected_rows = sum([len(kv_store) for kv_store in bucket.kvs.values()]) self._verify_ddocs(expected_rows, [bucket_name], self.ddocs_dest, self.dest_master)
def offline_cluster_upgrade(self): self._install(self.servers[:self.src_init + self.dest_init ]) upgrade_nodes = self.input.param('upgrade_nodes', "src").split(";") self.cluster.shutdown() XDCRReplicationBaseTest.setUp(self) self.set_xdcr_param('xdcrFailureRestartInterval', 1) self.sleep(60) bucket = self._get_bucket('default', self.src_master) self._load_bucket(bucket, self.src_master, self.gen_create, 'create', exp=0) bucket = self._get_bucket('bucket0', self.src_master) self._load_bucket(bucket, self.src_master, self.gen_create, 'create', exp=0) bucket = self._get_bucket('bucket0', self.dest_master) gen_create2 = BlobGenerator('loadTwo', 'loadTwo', self._value_size, end=self._num_items) self._load_bucket(bucket, self.dest_master, gen_create2, 'create', exp=0) nodes_to_upgrade = [] if "src" in upgrade_nodes : nodes_to_upgrade += self.src_nodes if "dest" in upgrade_nodes : nodes_to_upgrade += self.dest_nodes self.sleep(60) self._wait_for_replication_to_catchup() for upgrade_version in self.upgrade_versions: for server in nodes_to_upgrade: remote = RemoteMachineShellConnection(server) remote.stop_server() remote.disconnect() upgrade_threads = self._async_update(upgrade_version, nodes_to_upgrade) #wait upgrade statuses for upgrade_thread in upgrade_threads: upgrade_thread.join() success_upgrade = True while not self.queue.empty(): success_upgrade &= self.queue.get() if not success_upgrade: self.fail("Upgrade failed!") self.sleep(self.expire_time) self.set_xdcr_param('xdcrFailureRestartInterval', 1) self.sleep(60) bucket = self._get_bucket('bucket0', self.src_master) gen_create3 = BlobGenerator('loadThree', 'loadThree', self._value_size, end=self._num_items) self._load_bucket(bucket, self.src_master, gen_create3, 'create', exp=0) self.do_merge_bucket(self.src_master, self.dest_master, True, bucket) bucket = self._get_bucket('default', self.src_master) self._load_bucket(bucket, self.src_master, gen_create2, 'create', exp=0) self.do_merge_bucket(self.src_master, self.dest_master, False, bucket) self.sleep(60) self.verify_xdcr_stats(self.src_nodes, self.dest_nodes, True)
def incremental_offline_upgrade(self): upgrade_seq = self.input.param("upgrade_seq", "src>dest") self._install(self.servers[:self.src_init + self.dest_init ]) self.cluster.shutdown(force=True) XDCRReplicationBaseTest.setUp(self) self.set_xdcr_param('xdcrFailureRestartInterval', 1) self.sleep(60) bucket = self._get_bucket(self, 'default', self.src_master) self._load_bucket(bucket, self.src_master, self.gen_create, 'create', exp=0) bucket = self._get_bucket(self, 'bucket0', self.src_master) self._load_bucket(bucket, self.src_master, self.gen_create, 'create', exp=0) bucket = self._get_bucket(self, 'bucket0', self.dest_master) gen_create2 = BlobGenerator('loadTwo', 'loadTwo', self._value_size, end=self._num_items) self._load_bucket(bucket, self.dest_master, gen_create2, 'create', exp=0) self.sleep(self._timeout) self._wait_for_replication_to_catchup() nodes_to_upgrade = [] if upgrade_seq == "src>dest": nodes_to_upgrade = copy.copy(self.src_nodes) nodes_to_upgrade.extend(self.dest_nodes) elif upgrade_seq == "src<dest": nodes_to_upgrade = copy.copy(self.dest_nodes) nodes_to_upgrade.extend(self.src_nodes) elif upgrade_seq == "src><dest": min_cluster = min(len(self.src_nodes), len(self.dest_nodes)) for i in xrange(min_cluster): nodes_to_upgrade.append(self.src_nodes[i]) nodes_to_upgrade.append(self.dest_nodes[i]) for _seq, node in enumerate(nodes_to_upgrade): self._offline_upgrade([node]) self.set_xdcr_param('xdcrFailureRestartInterval', 1) self.sleep(60) bucket = self._get_bucket(self, 'bucket0', self.src_master) itemPrefix = "loadThree" + _seq * 'a' gen_create3 = BlobGenerator(itemPrefix, itemPrefix, self._value_size, end=self._num_items) self._load_bucket(bucket, self.src_master, gen_create3, 'create', exp=0) bucket = self._get_bucket(self, 'default', self.src_master) itemPrefix = "loadFour" + _seq * 'a' gen_create4 = BlobGenerator(itemPrefix, itemPrefix, self._value_size, end=self._num_items) self._load_bucket(bucket, self.src_master, gen_create4, 'create', exp=0) self.sleep(60) bucket = self._get_bucket(self, 'bucket0', self.src_master) self.do_merge_bucket(self.src_master, self.dest_master, True, bucket) bucket = self._get_bucket(self, 'default', self.src_master) self.do_merge_bucket(self.src_master, self.dest_master, False, bucket) self.verify_xdcr_stats(self.src_nodes, self.dest_nodes, True) self.sleep(self.wait_timeout * 5, "Let clusters work for some time")
def online_cluster_upgrade(self): self._install(self.servers[:self.src_init + self.dest_init ]) self.initial_version = self.upgrade_versions[0] self._install(self.servers[self.src_init + self.dest_init:]) self.cluster.shutdown() XDCRReplicationBaseTest.setUp(self) bucket_default = self._get_bucket('default', self.src_master) bucket_sasl = self._get_bucket('bucket0', self.src_master) bucket_standard = self._get_bucket('standard_bucket0', self.dest_master) self._load_bucket(bucket_default, self.src_master, self.gen_create, 'create', exp=0) self._load_bucket(bucket_sasl, self.src_master, self.gen_create, 'create', exp=0) self._load_bucket(bucket_standard, self.dest_master, self.gen_create, 'create', exp=0) gen_create2 = BlobGenerator('loadTwo', 'loadTwo-', self._value_size, end=self._num_items) self._load_bucket(bucket_sasl, self.dest_master, gen_create2, 'create', exp=0) self._online_upgrade(self.src_nodes, self.servers[self.src_init + self.dest_init:]) self._install(self.src_nodes) self._online_upgrade(self.servers[self.src_init + self.dest_init:], self.src_nodes, False) self._load_bucket(bucket_default, self.src_master, self.gen_delete, 'delete', exp=0) self._load_bucket(bucket_default, self.src_master, self.gen_update, 'create', exp=self._expires) self._load_bucket(bucket_sasl, self.src_master, self.gen_delete, 'delete', exp=0) self._load_bucket(bucket_sasl, self.src_master, self.gen_update, 'create', exp=self._expires) self._online_upgrade(self.dest_nodes, self.servers[self.src_init + self.dest_init:]) self._install(self.dest_nodes) self._online_upgrade(self.servers[self.src_init + self.dest_init:], self.dest_nodes, False) self._load_bucket(bucket_standard, self.dest_master, self.gen_delete, 'delete', exp=0) self._load_bucket(bucket_standard, self.dest_master, self.gen_update, 'create', exp=self._expires) self.do_merge_bucket(self.src_master, self.dest_master, True, bucket_sasl) bucket_sasl = self._get_bucket('bucket0', self.dest_master) gen_delete2 = BlobGenerator('loadTwo', 'loadTwo-', self._value_size, start=int((self._num_items) * (float)(100 - self._percent_delete) / 100), end=self._num_items) gen_update2 = BlobGenerator('loadTwo', 'loadTwo-', self._value_size, start=0, end=int(self._num_items * (float)(self._percent_update) / 100)) self._load_bucket(bucket_sasl, self.dest_master, gen_delete2, 'delete', exp=0) self._load_bucket(bucket_sasl, self.dest_master, gen_update2, 'create', exp=self._expires) self.do_merge_bucket(self.dest_master, self.src_master, False, bucket_sasl) self.do_merge_bucket(self.src_master, self.dest_master, False, bucket_default) self.do_merge_bucket(self.dest_master, self.src_master, False, bucket_standard) self.verify_xdcr_stats(self.src_nodes, self.dest_nodes, True)
def offline_cluster_upgrade(self): self._install(self.servers[:self.src_init + self.dest_init ]) upgrade_nodes = self.input.param('upgrade_nodes', "src").split(";") self.cluster.shutdown() XDCRReplicationBaseTest.setUp(self) bucket = self._get_bucket('default', self.src_master) self._load_bucket(bucket, self.src_master, self.gen_create, 'create', exp=0) bucket = self._get_bucket('bucket0', self.src_master) self._load_bucket(bucket, self.src_master, self.gen_create, 'create', exp=0) bucket = self._get_bucket('bucket0', self.dest_master) gen_create2 = BlobGenerator('loadTwo', 'loadTwo', self._value_size, end=self._num_items) self._load_bucket(bucket, self.dest_master, gen_create2, 'create', exp=0) nodes_to_upgrade = [] if "src" in upgrade_nodes : nodes_to_upgrade += self.src_nodes if "dest" in upgrade_nodes : nodes_to_upgrade += self.dest_nodes for upgrade_version in self.upgrade_versions: for server in nodes_to_upgrade: remote = RemoteMachineShellConnection(server) remote.stop_server() remote.disconnect() upgrade_threads = self._async_update(upgrade_version, nodes_to_upgrade) #wait upgrade statuses for upgrade_thread in upgrade_threads: upgrade_thread.join() success_upgrade = True while not self.queue.empty(): success_upgrade &= self.queue.get() if not success_upgrade: self.fail("Upgrade failed!") self.sleep(self.expire_time) bucket = self._get_bucket('bucket0', self.src_master) gen_create3 = BlobGenerator('loadThree', 'loadThree', self._value_size, end=self._num_items) self._load_bucket(bucket, self.src_master, gen_create3, 'create', exp=0) self.do_merge_bucket(self.src_master, self.dest_master, True, bucket) bucket = self._get_bucket('default', self.src_master) self._load_bucket(bucket, self.src_master, gen_create2, 'create', exp=0) self.do_merge_bucket(self.src_master, self.dest_master, False, bucket) self.verify_xdcr_stats(self.src_nodes, self.dest_nodes, True)
def offline_cluster_upgrade(self): self._install(self.servers[:self.src_init + self.dest_init ]) upgrade_nodes = self.input.param('upgrade_nodes', "src").split(";") self.cluster.shutdown() XDCRReplicationBaseTest.setUp(self) self.set_xdcr_param('xdcrFailureRestartInterval', 1) self.sleep(60) bucket = self._get_bucket(self, 'default', self.src_master) self._operations() self._load_bucket(bucket, self.src_master, self.gen_create, 'create', exp=0) bucket = self._get_bucket(self, 'bucket0', self.src_master) self._load_bucket(bucket, self.src_master, self.gen_create, 'create', exp=0) bucket = self._get_bucket(self, 'bucket0', self.dest_master) gen_create2 = BlobGenerator('loadTwo', 'loadTwo', self._value_size, end=self._num_items) self._load_bucket(bucket, self.dest_master, gen_create2, 'create', exp=0) nodes_to_upgrade = [] if "src" in upgrade_nodes : nodes_to_upgrade += self.src_nodes if "dest" in upgrade_nodes : nodes_to_upgrade += self.dest_nodes self.sleep(60) self._wait_for_replication_to_catchup() self._offline_upgrade(nodes_to_upgrade) self.set_xdcr_param('xdcrFailureRestartInterval', 1) self.sleep(60) bucket = self._get_bucket(self, 'bucket0', self.src_master) gen_create3 = BlobGenerator('loadThree', 'loadThree', self._value_size, end=self._num_items) self._load_bucket(bucket, self.src_master, gen_create3, 'create', exp=0) self.do_merge_bucket(self.src_master, self.dest_master, True, bucket) bucket = self._get_bucket(self, 'default', self.src_master) self._load_bucket(bucket, self.src_master, gen_create2, 'create', exp=0) self.do_merge_bucket(self.src_master, self.dest_master, False, bucket) self.sleep(60) self._post_upgrade_ops() self.verify_xdcr_stats(self.src_nodes, self.dest_nodes, True) self._verify(self.gen_create.end + gen_create2.end + gen_create3.end)