def cleanup_cluster(servers, wait_for_rebalance=True, master = None): log = logger.Logger.get_logger() if master == None: master = servers[0] rest = RestConnection(master) helper = RestHelper(rest) helper.is_ns_server_running(timeout_in_seconds=testconstants.NS_SERVER_TIMEOUT) nodes = rest.node_statuses() master_id = rest.get_nodes_self().id for node in nodes: if int(node.port) in xrange(9091, 9991): rest.eject_node(node) nodes.remove(node) if len(nodes) > 1: log.info("rebalancing all nodes in order to remove nodes") rest.log_client_error("Starting rebalance from test, ejected nodes %s" % \ [node.id for node in nodes if node.id != master_id]) removed = helper.remove_nodes(knownNodes=[node.id for node in nodes], ejectedNodes=[node.id for node in nodes if node.id != master_id], wait_for_rebalance=wait_for_rebalance) success_cleaned = [] for removed in [node for node in nodes if (node.id != master_id)]: removed.rest_password = servers[0].rest_password removed.rest_username = servers[0].rest_username try: rest = RestConnection(removed) except Exception as ex: log.error("can't create rest connection after rebalance out for ejected nodes,\ will retry after 10 seconds according to MB-8430: {0} ".format(ex)) time.sleep(10) rest = RestConnection(removed) start = time.time() while time.time() - start < 30: if len(rest.get_pools_info()["pools"]) == 0: success_cleaned.append(removed) break else: time.sleep(0.1) if time.time() - start > 10: log.error("'pools' on node {0}:{1} - {2}".format( removed.ip, removed.port, rest.get_pools_info()["pools"])) for node in set([node for node in nodes if (node.id != master_id)]) - set(success_cleaned): log.error("node {0}:{1} was not cleaned after removing from cluster".format( removed.ip, removed.port)) try: rest = RestConnection(node) rest.force_eject_node() except Exception as ex: log.error("force_eject_node {0}:{1} failed: {2}".format(removed.ip, removed.port, ex)) if len(set([node for node in nodes if (node.id != master_id)])\ - set(success_cleaned)) != 0: raise Exception("not all ejected nodes were cleaned successfully") log.info("removed all the nodes from cluster associated with {0} ? {1}".format(servers[0], \ [(node.id, node.port) for node in nodes if (node.id != master_id)]))
def cleanup_cluster(servers, wait_for_rebalance=True, master = None): log = logger.Logger.get_logger() if master is None: master = servers[0] rest = RestConnection(master) helper = RestHelper(rest) helper.is_ns_server_running(timeout_in_seconds=testconstants.NS_SERVER_TIMEOUT) nodes = rest.node_statuses() master_id = rest.get_nodes_self().id for node in nodes: if int(node.port) in xrange(9091, 9991): rest.eject_node(node) nodes.remove(node) if len(nodes) > 1: log.info("rebalancing all nodes in order to remove nodes") rest.log_client_error("Starting rebalance from test, ejected nodes %s" % \ [node.id for node in nodes if node.id != master_id]) removed = helper.remove_nodes(knownNodes=[node.id for node in nodes], ejectedNodes=[node.id for node in nodes if node.id != master_id], wait_for_rebalance=wait_for_rebalance) success_cleaned = [] for removed in [node for node in nodes if (node.id != master_id)]: removed.rest_password = servers[0].rest_password removed.rest_username = servers[0].rest_username try: rest = RestConnection(removed) except Exception as ex: log.error("can't create rest connection after rebalance out for ejected nodes,\ will retry after 10 seconds according to MB-8430: {0} ".format(ex)) time.sleep(10) rest = RestConnection(removed) start = time.time() while time.time() - start < 30: if len(rest.get_pools_info()["pools"]) == 0: success_cleaned.append(removed) break else: time.sleep(0.1) if time.time() - start > 10: log.error("'pools' on node {0}:{1} - {2}".format( removed.ip, removed.port, rest.get_pools_info()["pools"])) for node in set([node for node in nodes if (node.id != master_id)]) - set(success_cleaned): log.error("node {0}:{1} was not cleaned after removing from cluster".format( removed.ip, removed.port)) try: rest = RestConnection(node) rest.force_eject_node() except Exception as ex: log.error("force_eject_node {0}:{1} failed: {2}".format(removed.ip, removed.port, ex)) if len(set([node for node in nodes if (node.id != master_id)])\ - set(success_cleaned)) != 0: raise Exception("not all ejected nodes were cleaned successfully") log.info("removed all the nodes from cluster associated with {0} ? {1}".format(servers[0], \ [(node.id, node.port) for node in nodes if (node.id != master_id)]))
def cleanup_cluster(servers, wait_for_rebalance=True): log = logger.Logger.get_logger() rest = RestConnection(servers[0]) helper = RestHelper(rest) helper.is_ns_server_running(timeout_in_seconds=testconstants.NS_SERVER_TIMEOUT) nodes = rest.node_statuses() master_id = rest.get_nodes_self().id if len(nodes) > 1: log.info("rebalancing all nodes in order to remove nodes") removed = helper.remove_nodes(knownNodes=[node.id for node in nodes], ejectedNodes=[node.id for node in nodes if node.id != master_id], wait_for_rebalance=wait_for_rebalance) log.info("removed all the nodes from cluster associated with {0} ? {1}".format(servers[0], removed))
def test_crash_entire_cluster(self): self.cluster.rebalance( [self.master], self.servers[1:], []) vbucket = 0 nodeA = self.servers[0] n = 10000 self.load_docs(nodeA, vbucket, n) dcp_client = self.dcp_client(nodeA, PRODUCER) stream = dcp_client.stream_req(vbucket, 0, 0, 2*n, 0) self.load_docs(nodeA, vbucket, n) # stop all nodes node_range = list(range(len(self.servers))) for i in node_range: assert self.stop_node(i) time.sleep(2) # start all nodes in reverse order node_range.reverse() for i in node_range: assert self.start_node(i) rest = RestHelper(RestConnection(nodeA)) assert rest.is_ns_server_running() _, _, high_seqno = self.vb_info(nodeA, vbucket) dcp_client = self.dcp_client(nodeA, PRODUCER) stream = dcp_client.stream_req(vbucket, 0, 0, high_seqno, 0) stream.run() assert stream.last_by_seqno == high_seqno
def test_crash_entire_cluster(self): self.cluster.rebalance( [self.master], self.servers[1:], []) vbucket = 0 nodeA = self.servers[0] n = 10000 self.load_docs(nodeA, vbucket, n) dcp_client = self.dcp_client(nodeA, PRODUCER) stream = dcp_client.stream_req(vbucket, 0, 0, 2*n, 0) self.load_docs(nodeA, vbucket, n) # stop all nodes node_range = range(len(self.servers)) for i in node_range: assert self.stop_node(i) time.sleep(2) # start all nodes in reverse order node_range.reverse() for i in node_range: assert self.start_node(i) rest = RestHelper(RestConnection(nodeA)) assert rest.is_ns_server_running() _, _, high_seqno = self.vb_info(nodeA, vbucket) dcp_client = self.dcp_client(nodeA, PRODUCER) stream = dcp_client.stream_req(vbucket, 0, 0, high_seqno, 0) stream.run() assert stream.last_by_seqno == high_seqno
def test_stream_after_warmup(self): nodeA = self.servers[0] bucket = 'standard_bucket' + str(self.standard_buckets - 1) originalVbInfo = self.all_vb_info(nodeA, bucket=bucket) expectedVbSeqno = {} # load all buckets doc_gen = BlobGenerator('dcpdata', 'dcpdata-', self.value_size, end=self.num_items) self._load_all_buckets(self.master, doc_gen, "create", 0) self._wait_for_stats_all_buckets([nodeA]) # store expected vb seqnos originalVbInfo = self.all_vb_info(nodeA, bucket=bucket) # restart node assert self.stop_node(0) time.sleep(5) assert self.start_node(0) rest = RestHelper(RestConnection(nodeA)) assert rest.is_ns_server_running() time.sleep(2) # verify original vbInfo can be streamed dcp_client = self.dcp_client(nodeA, PRODUCER, bucket_name=bucket) for vbucket in originalVbInfo: vb_uuid, _, high_seqno = originalVbInfo[vbucket] stream = dcp_client.stream_req(vbucket, 0, 0, high_seqno, vb_uuid) responses = stream.run() assert high_seqno == stream.last_by_seqno
def test_crash_while_streaming(self): bucket = self.bucket_util.buckets[0] vbucket = randint(0, self.vbuckets) nodeA = self.servers[0] self.load_docs(bucket, vbucket, 0, self.num_items, "create") shell_conn = RemoteMachineShellConnection(nodeA) cb_stat_obj = Cbstats(shell_conn) dcp_client = self.dcp_client(nodeA, dcp.constants.PRODUCER) _ = dcp_client.stream_req(vbucket, 0, 0, 2 * self.num_items, 0) self.load_docs(nodeA, vbucket, self.num_items) self.assertTrue(self.stop_node(0), msg="Failed during stop_node") self.sleep(2, "Sleep after stop_node") self.assertTrue(self.start_node(0), msg="Failed during start_node") rest = RestHelper(RestConnection(nodeA)) self.assertTrue(rest.is_ns_server_running(), msg="Failed while is_ns_server_running check") self.sleep(30, "Sleep to wait for ns_server to run") vb_info = cb_stat_obj.vbucket_seqno(bucket.name) dcp_client = self.dcp_client(nodeA, dcp.constants.PRODUCER) stream = dcp_client.stream_req(vbucket, 0, 0, vb_info[vbucket]["high_seqno"], 0) stream.run() self.assertTrue(stream.last_by_seqno == vb_info[vbucket]["high_seqno"], msg="Mismatch in high_seqno. {0} == {1}".format( vb_info[vbucket]["high_seqno"], stream.last_by_seqno)) # Disconnect shell Connection for the node shell_conn.disconnect()
def test_stream_after_warmup(self): nodeA = self.servers[0] bucket = 'standard_bucket'+str(self.standard_buckets-1) originalVbInfo = self.all_vb_info(nodeA, bucket = bucket) expectedVbSeqno = {} # load all buckets doc_gen = BlobGenerator( 'dcpdata', 'dcpdata-', self.value_size, end=self.num_items) self._load_all_buckets(self.master, doc_gen, "create", 0) self._wait_for_stats_all_buckets([nodeA]) # store expected vb seqnos originalVbInfo = self.all_vb_info(nodeA, bucket = bucket) # restart node assert self.stop_node(0) time.sleep(5) assert self.start_node(0) rest = RestHelper(RestConnection(nodeA)) assert rest.is_ns_server_running() time.sleep(2) # verify original vbInfo can be streamed dcp_client = self.dcp_client(nodeA, PRODUCER, auth_user = bucket) for vbucket in originalVbInfo: vb_uuid, _, high_seqno = originalVbInfo[vbucket] stream = dcp_client.stream_req(vbucket, 0, 0, high_seqno, vb_uuid) responses = stream.run() assert high_seqno == stream.last_by_seqno
def cleanup_cluster(servers, wait_for_rebalance=True): log = logger.Logger.get_logger() rest = RestConnection(servers[0]) helper = RestHelper(rest) helper.is_ns_server_running( timeout_in_seconds=testconstants.NS_SERVER_TIMEOUT) nodes = rest.node_statuses() master_id = rest.get_nodes_self().id if len(nodes) > 1: log.info("rebalancing all nodes in order to remove nodes") removed = helper.remove_nodes( knownNodes=[node.id for node in nodes], ejectedNodes=[ node.id for node in nodes if node.id != master_id ], wait_for_rebalance=wait_for_rebalance) log.info( "removed all the nodes from cluster associated with {0} ? {1}". format(servers[0], removed))
def cleanup_cluster(servers, wait_for_rebalance=True): log = logger.Logger.get_logger() rest = RestConnection(servers[0]) helper = RestHelper(rest) helper.is_ns_server_running(timeout_in_seconds=testconstants.NS_SERVER_TIMEOUT) nodes = rest.node_statuses() master_id = rest.get_nodes_self().id if len(nodes) > 1: log.info("rebalancing all nodes in order to remove nodes") rest.log_client_error("Starting rebalance from test, ejected nodes %s" % \ [node.id for node in nodes if node.id != master_id]) removed = helper.remove_nodes(knownNodes=[node.id for node in nodes], ejectedNodes=[node.id for node in nodes if node.id != master_id], wait_for_rebalance=wait_for_rebalance) success_cleaned = [] for removed in [node for node in nodes if (node.id != master_id)]: removed.rest_password = servers[0].rest_password removed.rest_username = servers[0].rest_username rest = RestConnection(removed) start = time.time() while time.time() - start < 30: if len(rest.get_pools_info()["pools"]) == 0: success_cleaned.append(removed) break else: time.sleep(0.1) if time.time() - start > 10: log.error("'pools' on node {0}:{1} - {2}".format( removed.ip, removed.port, rest.get_pools_info()["pools"])) for node in set([node for node in nodes if (node.id != master_id)]) - set(success_cleaned): log.error("node {0}:{1} was not cleaned after removing from cluster".format( removed.ip, removed.port)) if len(set([node for node in nodes if (node.id != master_id)])\ - set(success_cleaned)) != 0: raise Exception("not all ejected nodes were cleaned successfully") log.info("removed all the nodes from cluster associated with {0} ? {1}".format(servers[0], \ [(node.id, node.port) for node in nodes if (node.id != master_id)]))
def test_stream_after_n_crashes(self): crashes = self.input.param("crash_num", 5) vbucket = randint(0, self.vbuckets) bucket = self.bucket_util.buckets[0] self.log.info("Chosen vbucket {0} for {1} crashes".format( vbucket, crashes)) start = 0 end = self.num_items nodeA = self.cluster.servers[0] shell_conn = RemoteMachineShellConnection(nodeA) cb_stat_obj = Cbstats(shell_conn) rest = RestHelper(RestConnection(nodeA)) for _ in xrange(crashes): # Load data into the selected vbucket self.load_docs(bucket, vbucket, start, end, "create") self.assertTrue(self.stop_node(0), msg="Failed during stop_node") self.sleep(5, "Sleep after stop_node") self.assertTrue(self.start_node(0), msg="Failed during start_node") self.assertTrue(rest.is_ns_server_running(), msg="Failed while is_ns_server_running check") self.sleep(5, "Waiting after ns_server started") # Fetch vbucket seqno stats vb_stat = cb_stat_obj.vbucket_seqno(bucket.name) dcp_client = self.dcp_client(nodeA, dcp.constants.PRODUCER) stream = dcp_client.stream_req(vbucket, 0, 0, vb_stat[vbucket]["high_seqno"], vb_stat[vbucket]["uuid"]) stream.run() self.assertTrue( stream.last_by_seqno == vb_stat[vbucket]["high_seqno"], msg="Mismatch in high_seqno. {0} == {1}".format( vb_stat[vbucket]["high_seqno"], stream.last_by_seqno)) # Update start/end values for next loop start = end end += self.num_items # Disconnect shell Connection for the node shell_conn.disconnect()
def test_crash_entire_cluster(self): self.cluster.rebalance([self.master], self.servers[1:], []) bucket = self.bucket_util.buckets[0] vbucket = randint(0, self.vbuckets) nodeA = self.servers[0] self.load_docs(bucket, vbucket, 0, self.num_items, "create") shell_conn = RemoteMachineShellConnection(nodeA) cb_stat_obj = Cbstats(shell_conn) dcp_client = self.dcp_client(nodeA, dcp.constants.PRODUCER) _ = dcp_client.stream_req(vbucket, 0, 0, 2 * self.num_items, 0) self.load_docs(nodeA, vbucket, self.num_items) # stop all nodes node_range = range(len(self.servers)) for i in node_range: self.assertTrue(self.stop_node(i), msg="Failed during stoip_node") self.sleep(2, "Wait after stop_node") # start all nodes in reverse order node_range.reverse() for i in node_range: self.assertTrue(self.start_node(i), msg="Failed during start_node") rest = RestHelper(RestConnection(nodeA)) self.assertTrue(rest.is_ns_server_running(), msg="Failed while is_ns_server_running check") vb_info = cb_stat_obj.vbucket_seqno(bucket.name) dcp_client = self.dcp_client(nodeA, dcp.constants.PRODUCER) stream = dcp_client.stream_req(vbucket, 0, 0, vb_info[vbucket]["high_seqno"], 0) stream.run() self.assertTrue(stream.last_by_seqno == vb_info[vbucket]["high_seqno"], msg="Seq-no mismatch. {0} != {1}".format( stream.last_by_seqno, vb_info[vbucket]["high_seqno"])) # Disconnect shell Connection for the node shell_conn.disconnect()
def test_crash_while_streaming(self): vbucket = 0 nodeA = self.servers[0] n = 10000 self.load_docs(nodeA, vbucket, n) dcp_client = self.dcp_client(nodeA, PRODUCER) stream = dcp_client.stream_req(vbucket, 0, 0, 2*n, 0) self.load_docs(nodeA, vbucket, n) assert self.stop_node(0) time.sleep(2) assert self.start_node(0) rest = RestHelper(RestConnection(nodeA)) assert rest.is_ns_server_running() time.sleep(2) _, _, high_seqno = self.vb_info(nodeA, vbucket) dcp_client = self.dcp_client(nodeA, PRODUCER) stream = dcp_client.stream_req(vbucket, 0, 0, high_seqno, 0) stream.run() assert stream.last_by_seqno == high_seqno
def test_stream_after_n_crashes(self): crashes = 5 vbucket = 0 # load some data nodeA = self.servers[0] rest = RestHelper(RestConnection(nodeA)) for i in xrange(crashes): self.load_docs(nodeA, vbucket, self.num_items) assert self.stop_node(0) time.sleep(5) assert self.start_node(0) assert rest.is_ns_server_running() time.sleep(2) vb_uuid, _, high_seqno = self.vb_info(nodeA, vbucket) dcp_client = self.dcp_client(nodeA, PRODUCER) stream = dcp_client.stream_req(vbucket, 0, 0, high_seqno, vb_uuid) stream.run() assert stream.last_by_seqno == high_seqno
def test_crash_while_streaming(self): vbucket = 0 nodeA = self.servers[0] n = 10000 self.load_docs(nodeA, vbucket, n) dcp_client = self.dcp_client(nodeA, PRODUCER) stream = dcp_client.stream_req(vbucket, 0, 0, 2*n, 0) self.load_docs(nodeA, vbucket, n) assert self.stop_node(0) time.sleep(2) assert self.start_node(0) rest = RestHelper(RestConnection(nodeA)) assert rest.is_ns_server_running() time.sleep(30) _, _, high_seqno = self.vb_info(nodeA, vbucket) dcp_client = self.dcp_client(nodeA, PRODUCER) stream = dcp_client.stream_req(vbucket, 0, 0, high_seqno, 0) stream.run() assert stream.last_by_seqno == high_seqno
def test_stream_after_n_crashes(self): crashes = 5 vbucket = 0 # load some data nodeA = self.servers[0] rest = RestHelper(RestConnection(nodeA)) for i in xrange(crashes): self.load_docs(nodeA, vbucket, self.num_items) assert self.stop_node(0) time.sleep(5) assert self.start_node(0) assert rest.is_ns_server_running() time.sleep(2) vb_uuid, _, high_seqno = self.vb_info(nodeA, vbucket) dcp_client = self.dcp_client(nodeA, PRODUCER) stream = dcp_client.stream_req( vbucket, 0, 0, high_seqno, vb_uuid) stream.run() assert stream.last_by_seqno == high_seqno
def test_stream_after_warmup(self): nodeA = self.cluster.servers[0] bucket = self.bucket_util.buckets[1] shell_conn = RemoteMachineShellConnection(nodeA) cb_stat_obj = Cbstats(shell_conn) # load all buckets doc_gen = doc_generator(self.key, 0, self.num_items) self._load_all_buckets(self.master, doc_gen, "create", 0) self._wait_for_stats_all_buckets() # store expected vb seqnos originalVbInfo = self.all_vb_info(cb_stat_obj, bucket.name) # restart node self.assertTrue(self.stop_node(0), msg="Failed during stop_node") self.sleep(5, "Wait after stop_node") self.assertTrue(self.start_node(0), msg="Failed during start_node") rest = RestHelper(RestConnection(nodeA)) self.assertTrue(rest.is_ns_server_running(), msg="Failed while is_ns_server_running") self.sleep(2, "Wait after ns_server_start") # verify original vbInfo can be streamed dcp_client = self.dcp_client(nodeA, PRODUCER, bucket_name=bucket.name) for vbucket in originalVbInfo: vb_uuid, _, high_seqno = originalVbInfo[vbucket] stream = dcp_client.stream_req(vbucket, 0, 0, high_seqno, vb_uuid) _ = stream.run() self.assertTrue(high_seqno == stream.last_by_seqno, msg="Mismatch in high_seqno. {0} == {1}".format( high_seqno, stream.last_by_seqno)) # Disconnect the shell_conn shell_conn.disconnect()
class NewUpgradeBaseTest(BaseTestCase): def setUp(self): super(NewUpgradeBaseTest, self).setUp() self.product = self.input.param('product', 'couchbase-server') self.initial_version = self.input.param('initial_version', '1.8.1-942-rel') self.initial_vbuckets = self.input.param('initial_vbuckets', 64) self.rest_settings = self.input.membase_settings self.rest = RestConnection(self.master) self.rest_helper = RestHelper(self.rest) self.sleep_time = 10 self.data_size = self.input.param('data_size', 1024) self.op_types = self.input.param('op_types', 'bucket') self.item_flag = self.input.param('item_flag', 4042322160) self.expire_time = self.input.param('expire_time', 0) def tearDown(self): super(NewUpgradeBaseTest, self).tearDown() def _install(self, servers): params = {} params['num_nodes'] = len(servers) params['product'] = self.product params['version'] = self.initial_version params['vbuckets'] = [self.initial_vbuckets] InstallerJob().parallel_install(servers, params) if self.product in ["couchbase", "couchbase-server", "cb"]: success = True for server in servers: success &= RemoteMachineShellConnection( server).is_couchbase_installed() if not success: self.log.info("some nodes were not install successfully!") sys.exit(1) def operations(self, multi_nodes=False): self.quota = self._initialize_nodes(self.cluster, self.servers, self.disabled_consistent_view) self.buckets = [] gc.collect() if self.total_buckets > 0: self.bucket_size = self._get_bucket_size(self.quota, self.total_buckets) if self.default_bucket: self.cluster.create_default_bucket(self.master, self.bucket_size, self.num_replicas) self.buckets.append( Bucket(name="default", authType="sasl", saslPassword="", num_replicas=self.num_replicas, bucket_size=self.bucket_size)) self._create_sasl_buckets(self.master, self.sasl_buckets) self._create_standard_buckets(self.master, self.standard_buckets) if multi_nodes: servers_in = [ self.servers[i + 1] for i in range(self.initial_num_servers - 1) ] self.cluster.rebalance(self.servers[:1], servers_in, []) if self.op_types == "data": self._load_data_all_buckets("create") if multi_nodes: self._wait_for_stats_all_buckets( self.servers[:self.initial_num_servers]) else: self._wait_for_stats_all_buckets([self.master]) def _load_data_all_buckets(self, op_type='create', start=0): loaded = False count = 0 gen_load = BlobGenerator('upgrade-', 'upgrade-', self.data_size, start=start, end=self.num_items) while not loaded and count < 60: try: self._load_all_buckets(self.master, gen_load, op_type, self.expire_time, 1, self.item_flag, True, batch_size=20000, pause_secs=5, timeout_secs=180) loaded = True except MemcachedError as error: if error.status == 134: loaded = False self.log.error( "Memcached error 134, wait for 5 seconds and then try again" ) count += 1 time.sleep(self.sleep_time) def _get_build(self, server, version, remote, is_amazon=False): info = remote.extract_remote_info() builds, changes = BuildQuery().get_all_builds() self.log.info("finding build %s for machine %s" % (version, server)) result = re.search('r', version) if result is None: appropriate_build = BuildQuery().\ find_membase_release_build('%s-enterprise' % (self.product), info.deliverable_type, info.architecture_type, version.strip(), is_amazon=is_amazon) else: appropriate_build = BuildQuery().\ find_membase_build(builds, '%s-enterprise' % (self.product), info.deliverable_type, info.architecture_type, version.strip(), is_amazon=is_amazon) return appropriate_build def _upgrade(self, upgrade_version, server, remote): appropriate_build = self._get_build(server, upgrade_version, remote) self.assertTrue(appropriate_build.url, msg="unable to find build {0}".format(upgrade_version)) remote.download_build(appropriate_build) remote.membase_upgrade(appropriate_build, save_upgrade_config=False) self.rest_helper.is_ns_server_running(testconstants.NS_SERVER_TIMEOUT) self.rest.init_cluster_port(self.rest_settings.rest_username, self.rest_settings.rest_password) time.sleep(self.sleep_time) def verification(self, multi_nodes=False): for bucket in self.buckets: if self.rest_helper.bucket_exists(bucket.name): continue else: raise Exception("bucket:- %s not found" % bucket.name) if self.op_types == "bucket": bucketinfo = self.rest.get_bucket(bucket.name) self.log.info("bucket info :- %s" % bucketinfo) if self.op_types == "data": if multi_nodes: self._wait_for_stats_all_buckets( self.servers[:self.num_servers]) self._verify_all_buckets(self.master, 1, self.wait_timeout * 50, self.max_verify, True, 1) self._verify_stats_all_buckets(self.servers[:self.num_servers]) else: self._wait_for_stats_all_buckets([self.master]) self._verify_all_buckets(self.master, 1, self.wait_timeout * 50, self.max_verify, True, 1) self._verify_stats_all_buckets([self.master])
class NewUpgradeBaseTest(BaseTestCase): def setUp(self): super(NewUpgradeBaseTest, self).setUp() self.product = self.input.param('product', 'couchbase-server') self.initial_version = self.input.param('initial_version', '1.8.1-942-rel') self.initial_vbuckets = self.input.param('initial_vbuckets', 64) self.rest_settings = self.input.membase_settings self.rest = RestConnection(self.master) self.rest_helper = RestHelper(self.rest) self.sleep_time = 10 self.data_size = self.input.param('data_size', 1024) self.op_types = self.input.param('op_types', 'bucket') self.item_flag = self.input.param('item_flag', 4042322160) self.expire_time = self.input.param('expire_time', 0) def tearDown(self): super(NewUpgradeBaseTest, self).tearDown() def _install(self, servers): params = {} params['num_nodes'] = len(servers) params['product'] = self.product params['version'] = self.initial_version params['vbuckets'] = [self.initial_vbuckets] InstallerJob().parallel_install(servers, params) if self.product in ["couchbase", "couchbase-server", "cb"]: success = True for server in servers: success &= RemoteMachineShellConnection(server).is_couchbase_installed() if not success: self.log.info("some nodes were not install successfully!") sys.exit(1) def operations(self, multi_nodes=False): self.quota = self._initialize_nodes(self.cluster, self.servers, self.disabled_consistent_view) self.buckets = [] gc.collect() if self.total_buckets > 0: self.bucket_size = self._get_bucket_size(self.quota, self.total_buckets) if self.default_bucket: self.cluster.create_default_bucket(self.master, self.bucket_size, self.num_replicas) self.buckets.append(Bucket(name="default", authType="sasl", saslPassword="", num_replicas=self.num_replicas, bucket_size=self.bucket_size)) self._create_sasl_buckets(self.master, self.sasl_buckets) self._create_standard_buckets(self.master, self.standard_buckets) if multi_nodes: servers_in = [self.servers[i+1] for i in range(self.initial_num_servers-1)] self.cluster.rebalance(self.servers[:1], servers_in, []) if self.op_types == "data": self._load_data_all_buckets("create") if multi_nodes: self._wait_for_stats_all_buckets(self.servers[:self.initial_num_servers]) else: self._wait_for_stats_all_buckets([self.master]) def _load_data_all_buckets(self, op_type='create', start=0): loaded = False count = 0 gen_load = BlobGenerator('upgrade-', 'upgrade-', self.data_size, start=start, end=self.num_items) while not loaded and count < 60: try : self._load_all_buckets(self.master, gen_load, op_type, self.expire_time, 1, self.item_flag, True, batch_size=20000, pause_secs=5, timeout_secs=180) loaded = True except MemcachedError as error: if error.status == 134: loaded = False self.log.error("Memcached error 134, wait for 5 seconds and then try again") count += 1 time.sleep(self.sleep_time) def _get_build(self, server, version, remote, is_amazon=False): info = remote.extract_remote_info() builds, changes = BuildQuery().get_all_builds() self.log.info("finding build %s for machine %s" % (version, server)) result = re.search('r', version) if result is None: appropriate_build = BuildQuery().\ find_membase_release_build('%s-enterprise' % (self.product), info.deliverable_type, info.architecture_type, version.strip(), is_amazon=is_amazon) else: appropriate_build = BuildQuery().\ find_membase_build(builds, '%s-enterprise' % (self.product), info.deliverable_type, info.architecture_type, version.strip(), is_amazon=is_amazon) return appropriate_build def _upgrade(self, upgrade_version, server, remote): appropriate_build = self._get_build(server, upgrade_version, remote) self.assertTrue(appropriate_build.url, msg="unable to find build {0}".format(upgrade_version)) remote.download_build(appropriate_build) remote.membase_upgrade(appropriate_build, save_upgrade_config=False) self.rest_helper.is_ns_server_running(testconstants.NS_SERVER_TIMEOUT) self.rest.init_cluster_port(self.rest_settings.rest_username, self.rest_settings.rest_password) time.sleep(self.sleep_time) def verification(self, multi_nodes=False): for bucket in self.buckets: if self.rest_helper.bucket_exists(bucket.name): continue else: raise Exception("bucket:- %s not found" % bucket.name) if self.op_types == "bucket": bucketinfo = self.rest.get_bucket(bucket.name) self.log.info("bucket info :- %s" % bucketinfo) if self.op_types == "data": if multi_nodes: self._wait_for_stats_all_buckets(self.servers[:self.num_servers]) self._verify_all_buckets(self.master, 1, self.wait_timeout*50, self.max_verify, True, 1) self._verify_stats_all_buckets(self.servers[:self.num_servers]) else: self._wait_for_stats_all_buckets([self.master]) self._verify_all_buckets(self.master, 1, self.wait_timeout*50, self.max_verify, True, 1) self._verify_stats_all_buckets([self.master])