def cleanup_cluster(servers, wait_for_rebalance=True, master = None):
log = logger.Logger.get_logger()
if master == None:
master = servers[0]
rest = RestConnection(master)
helper = RestHelper(rest)
helper.is_ns_server_running(timeout_in_seconds=testconstants.NS_SERVER_TIMEOUT)
nodes = rest.node_statuses()
master_id = rest.get_nodes_self().id
for node in nodes:
if int(node.port) in xrange(9091, 9991):
rest.eject_node(node)
nodes.remove(node)
if len(nodes) > 1:
log.info("rebalancing all nodes in order to remove nodes")
rest.log_client_error("Starting rebalance from test, ejected nodes %s" % \
[node.id for node in nodes if node.id != master_id])
removed = helper.remove_nodes(knownNodes=[node.id for node in nodes],
ejectedNodes=[node.id for node in nodes if node.id != master_id],
wait_for_rebalance=wait_for_rebalance)
success_cleaned = []
for removed in [node for node in nodes if (node.id != master_id)]:
removed.rest_password = servers[0].rest_password
removed.rest_username = servers[0].rest_username
try:
rest = RestConnection(removed)
except Exception as ex:
log.error("can't create rest connection after rebalance out for ejected nodes,\
will retry after 10 seconds according to MB-8430: {0} ".format(ex))
time.sleep(10)
rest = RestConnection(removed)
start = time.time()
while time.time() - start < 30:
if len(rest.get_pools_info()["pools"]) == 0:
success_cleaned.append(removed)
break
else:
time.sleep(0.1)
if time.time() - start > 10:
log.error("'pools' on node {0}:{1} - {2}".format(
removed.ip, removed.port, rest.get_pools_info()["pools"]))
for node in set([node for node in nodes if (node.id != master_id)]) - set(success_cleaned):
log.error("node {0}:{1} was not cleaned after removing from cluster".format(
removed.ip, removed.port))
try:
rest = RestConnection(node)
rest.force_eject_node()
except Exception as ex:
log.error("force_eject_node {0}:{1} failed: {2}".format(removed.ip, removed.port, ex))
if len(set([node for node in nodes if (node.id != master_id)])\
- set(success_cleaned)) != 0:
raise Exception("not all ejected nodes were cleaned successfully")
log.info("removed all the nodes from cluster associated with {0} ? {1}".format(servers[0], \
[(node.id, node.port) for node in nodes if (node.id != master_id)]))
def cleanup_cluster(servers, wait_for_rebalance=True, master = None):
log = logger.Logger.get_logger()
if master is None:
master = servers[0]
rest = RestConnection(master)
helper = RestHelper(rest)
helper.is_ns_server_running(timeout_in_seconds=testconstants.NS_SERVER_TIMEOUT)
nodes = rest.node_statuses()
master_id = rest.get_nodes_self().id
for node in nodes:
if int(node.port) in xrange(9091, 9991):
rest.eject_node(node)
nodes.remove(node)
if len(nodes) > 1:
log.info("rebalancing all nodes in order to remove nodes")
rest.log_client_error("Starting rebalance from test, ejected nodes %s" % \
[node.id for node in nodes if node.id != master_id])
removed = helper.remove_nodes(knownNodes=[node.id for node in nodes],
ejectedNodes=[node.id for node in nodes if node.id != master_id],
wait_for_rebalance=wait_for_rebalance)
success_cleaned = []
for removed in [node for node in nodes if (node.id != master_id)]:
removed.rest_password = servers[0].rest_password
removed.rest_username = servers[0].rest_username
try:
rest = RestConnection(removed)
except Exception as ex:
log.error("can't create rest connection after rebalance out for ejected nodes,\
will retry after 10 seconds according to MB-8430: {0} ".format(ex))
time.sleep(10)
rest = RestConnection(removed)
start = time.time()
while time.time() - start < 30:
if len(rest.get_pools_info()["pools"]) == 0:
success_cleaned.append(removed)
break
else:
time.sleep(0.1)
if time.time() - start > 10:
log.error("'pools' on node {0}:{1} - {2}".format(
removed.ip, removed.port, rest.get_pools_info()["pools"]))
for node in set([node for node in nodes if (node.id != master_id)]) - set(success_cleaned):
log.error("node {0}:{1} was not cleaned after removing from cluster".format(
removed.ip, removed.port))
try:
rest = RestConnection(node)
rest.force_eject_node()
except Exception as ex:
log.error("force_eject_node {0}:{1} failed: {2}".format(removed.ip, removed.port, ex))
if len(set([node for node in nodes if (node.id != master_id)])\
- set(success_cleaned)) != 0:
raise Exception("not all ejected nodes were cleaned successfully")
log.info("removed all the nodes from cluster associated with {0} ? {1}".format(servers[0], \
[(node.id, node.port) for node in nodes if (node.id != master_id)]))
def cleanup_cluster(servers, wait_for_rebalance=True):
log = logger.Logger.get_logger()
rest = RestConnection(servers[0])
helper = RestHelper(rest)
helper.is_ns_server_running(timeout_in_seconds=testconstants.NS_SERVER_TIMEOUT)
nodes = rest.node_statuses()
master_id = rest.get_nodes_self().id
if len(nodes) > 1:
log.info("rebalancing all nodes in order to remove nodes")
removed = helper.remove_nodes(knownNodes=[node.id for node in nodes],
ejectedNodes=[node.id for node in nodes if node.id != master_id],
wait_for_rebalance=wait_for_rebalance)
log.info("removed all the nodes from cluster associated with {0} ? {1}".format(servers[0], removed))
def test_crash_entire_cluster(self):
self.cluster.rebalance(
[self.master],
self.servers[1:], [])
vbucket = 0
nodeA = self.servers[0]
n = 10000
self.load_docs(nodeA, vbucket, n)
dcp_client = self.dcp_client(nodeA, PRODUCER)
stream = dcp_client.stream_req(vbucket, 0, 0, 2*n, 0)
self.load_docs(nodeA, vbucket, n)
# stop all nodes
node_range = list(range(len(self.servers)))
for i in node_range:
assert self.stop_node(i)
time.sleep(2)
# start all nodes in reverse order
node_range.reverse()
for i in node_range:
assert self.start_node(i)
rest = RestHelper(RestConnection(nodeA))
assert rest.is_ns_server_running()
_, _, high_seqno = self.vb_info(nodeA, vbucket)
dcp_client = self.dcp_client(nodeA, PRODUCER)
stream = dcp_client.stream_req(vbucket, 0, 0, high_seqno, 0)
stream.run()
assert stream.last_by_seqno == high_seqno
def test_crash_entire_cluster(self):
self.cluster.rebalance(
[self.master],
self.servers[1:], [])
vbucket = 0
nodeA = self.servers[0]
n = 10000
self.load_docs(nodeA, vbucket, n)
dcp_client = self.dcp_client(nodeA, PRODUCER)
stream = dcp_client.stream_req(vbucket, 0, 0, 2*n, 0)
self.load_docs(nodeA, vbucket, n)
# stop all nodes
node_range = range(len(self.servers))
for i in node_range:
assert self.stop_node(i)
time.sleep(2)
# start all nodes in reverse order
node_range.reverse()
for i in node_range:
assert self.start_node(i)
rest = RestHelper(RestConnection(nodeA))
assert rest.is_ns_server_running()
_, _, high_seqno = self.vb_info(nodeA, vbucket)
dcp_client = self.dcp_client(nodeA, PRODUCER)
stream = dcp_client.stream_req(vbucket, 0, 0, high_seqno, 0)
stream.run()
assert stream.last_by_seqno == high_seqno
def test_stream_after_warmup(self):
nodeA = self.servers[0]
bucket = 'standard_bucket' + str(self.standard_buckets - 1)
originalVbInfo = self.all_vb_info(nodeA, bucket=bucket)
expectedVbSeqno = {}
# load all buckets
doc_gen = BlobGenerator('dcpdata',
'dcpdata-',
self.value_size,
end=self.num_items)
self._load_all_buckets(self.master, doc_gen, "create", 0)
self._wait_for_stats_all_buckets([nodeA])
# store expected vb seqnos
originalVbInfo = self.all_vb_info(nodeA, bucket=bucket)
# restart node
assert self.stop_node(0)
time.sleep(5)
assert self.start_node(0)
rest = RestHelper(RestConnection(nodeA))
assert rest.is_ns_server_running()
time.sleep(2)
# verify original vbInfo can be streamed
dcp_client = self.dcp_client(nodeA, PRODUCER, bucket_name=bucket)
for vbucket in originalVbInfo:
vb_uuid, _, high_seqno = originalVbInfo[vbucket]
stream = dcp_client.stream_req(vbucket, 0, 0, high_seqno, vb_uuid)
responses = stream.run()
assert high_seqno == stream.last_by_seqno
def test_crash_while_streaming(self):
bucket = self.bucket_util.buckets[0]
vbucket = randint(0, self.vbuckets)
nodeA = self.servers[0]
self.load_docs(bucket, vbucket, 0, self.num_items, "create")
shell_conn = RemoteMachineShellConnection(nodeA)
cb_stat_obj = Cbstats(shell_conn)
dcp_client = self.dcp_client(nodeA, dcp.constants.PRODUCER)
_ = dcp_client.stream_req(vbucket, 0, 0, 2 * self.num_items, 0)
self.load_docs(nodeA, vbucket, self.num_items)
self.assertTrue(self.stop_node(0), msg="Failed during stop_node")
self.sleep(2, "Sleep after stop_node")
self.assertTrue(self.start_node(0), msg="Failed during start_node")
rest = RestHelper(RestConnection(nodeA))
self.assertTrue(rest.is_ns_server_running(),
msg="Failed while is_ns_server_running check")
self.sleep(30, "Sleep to wait for ns_server to run")
vb_info = cb_stat_obj.vbucket_seqno(bucket.name)
dcp_client = self.dcp_client(nodeA, dcp.constants.PRODUCER)
stream = dcp_client.stream_req(vbucket, 0, 0,
vb_info[vbucket]["high_seqno"], 0)
stream.run()
self.assertTrue(stream.last_by_seqno == vb_info[vbucket]["high_seqno"],
msg="Mismatch in high_seqno. {0} == {1}".format(
vb_info[vbucket]["high_seqno"],
stream.last_by_seqno))
# Disconnect shell Connection for the node
shell_conn.disconnect()
def test_stream_after_warmup(self):
nodeA = self.servers[0]
bucket = 'standard_bucket'+str(self.standard_buckets-1)
originalVbInfo = self.all_vb_info(nodeA, bucket = bucket)
expectedVbSeqno = {}
# load all buckets
doc_gen = BlobGenerator(
'dcpdata', 'dcpdata-', self.value_size, end=self.num_items)
self._load_all_buckets(self.master, doc_gen, "create", 0)
self._wait_for_stats_all_buckets([nodeA])
# store expected vb seqnos
originalVbInfo = self.all_vb_info(nodeA, bucket = bucket)
# restart node
assert self.stop_node(0)
time.sleep(5)
assert self.start_node(0)
rest = RestHelper(RestConnection(nodeA))
assert rest.is_ns_server_running()
time.sleep(2)
# verify original vbInfo can be streamed
dcp_client = self.dcp_client(nodeA, PRODUCER, auth_user = bucket)
for vbucket in originalVbInfo:
vb_uuid, _, high_seqno = originalVbInfo[vbucket]
stream = dcp_client.stream_req(vbucket, 0, 0, high_seqno, vb_uuid)
responses = stream.run()
assert high_seqno == stream.last_by_seqno
def cleanup_cluster(servers, wait_for_rebalance=True):
log = logger.Logger.get_logger()
rest = RestConnection(servers[0])
helper = RestHelper(rest)
helper.is_ns_server_running(
timeout_in_seconds=testconstants.NS_SERVER_TIMEOUT)
nodes = rest.node_statuses()
master_id = rest.get_nodes_self().id
if len(nodes) > 1:
log.info("rebalancing all nodes in order to remove nodes")
removed = helper.remove_nodes(
knownNodes=[node.id for node in nodes],
ejectedNodes=[
node.id for node in nodes if node.id != master_id
],
wait_for_rebalance=wait_for_rebalance)
log.info(
"removed all the nodes from cluster associated with {0} ? {1}".
format(servers[0], removed))
def cleanup_cluster(servers, wait_for_rebalance=True):
log = logger.Logger.get_logger()
rest = RestConnection(servers[0])
helper = RestHelper(rest)
helper.is_ns_server_running(timeout_in_seconds=testconstants.NS_SERVER_TIMEOUT)
nodes = rest.node_statuses()
master_id = rest.get_nodes_self().id
if len(nodes) > 1:
log.info("rebalancing all nodes in order to remove nodes")
rest.log_client_error("Starting rebalance from test, ejected nodes %s" % \
[node.id for node in nodes if node.id != master_id])
removed = helper.remove_nodes(knownNodes=[node.id for node in nodes],
ejectedNodes=[node.id for node in nodes if node.id != master_id],
wait_for_rebalance=wait_for_rebalance)
success_cleaned = []
for removed in [node for node in nodes if (node.id != master_id)]:
removed.rest_password = servers[0].rest_password
removed.rest_username = servers[0].rest_username
rest = RestConnection(removed)
start = time.time()
while time.time() - start < 30:
if len(rest.get_pools_info()["pools"]) == 0:
success_cleaned.append(removed)
break
else:
time.sleep(0.1)
if time.time() - start > 10:
log.error("'pools' on node {0}:{1} - {2}".format(
removed.ip, removed.port, rest.get_pools_info()["pools"]))
for node in set([node for node in nodes if (node.id != master_id)]) - set(success_cleaned):
log.error("node {0}:{1} was not cleaned after removing from cluster".format(
removed.ip, removed.port))
if len(set([node for node in nodes if (node.id != master_id)])\
- set(success_cleaned)) != 0:
raise Exception("not all ejected nodes were cleaned successfully")
log.info("removed all the nodes from cluster associated with {0} ? {1}".format(servers[0], \
[(node.id, node.port) for node in nodes if (node.id != master_id)]))
def test_stream_after_n_crashes(self):
crashes = self.input.param("crash_num", 5)
vbucket = randint(0, self.vbuckets)
bucket = self.bucket_util.buckets[0]
self.log.info("Chosen vbucket {0} for {1} crashes".format(
vbucket, crashes))
start = 0
end = self.num_items
nodeA = self.cluster.servers[0]
shell_conn = RemoteMachineShellConnection(nodeA)
cb_stat_obj = Cbstats(shell_conn)
rest = RestHelper(RestConnection(nodeA))
for _ in xrange(crashes):
# Load data into the selected vbucket
self.load_docs(bucket, vbucket, start, end, "create")
self.assertTrue(self.stop_node(0), msg="Failed during stop_node")
self.sleep(5, "Sleep after stop_node")
self.assertTrue(self.start_node(0), msg="Failed during start_node")
self.assertTrue(rest.is_ns_server_running(),
msg="Failed while is_ns_server_running check")
self.sleep(5, "Waiting after ns_server started")
# Fetch vbucket seqno stats
vb_stat = cb_stat_obj.vbucket_seqno(bucket.name)
dcp_client = self.dcp_client(nodeA, dcp.constants.PRODUCER)
stream = dcp_client.stream_req(vbucket, 0, 0,
vb_stat[vbucket]["high_seqno"],
vb_stat[vbucket]["uuid"])
stream.run()
self.assertTrue(
stream.last_by_seqno == vb_stat[vbucket]["high_seqno"],
msg="Mismatch in high_seqno. {0} == {1}".format(
vb_stat[vbucket]["high_seqno"], stream.last_by_seqno))
# Update start/end values for next loop
start = end
end += self.num_items
# Disconnect shell Connection for the node
shell_conn.disconnect()
def test_crash_entire_cluster(self):
self.cluster.rebalance([self.master], self.servers[1:], [])
bucket = self.bucket_util.buckets[0]
vbucket = randint(0, self.vbuckets)
nodeA = self.servers[0]
self.load_docs(bucket, vbucket, 0, self.num_items, "create")
shell_conn = RemoteMachineShellConnection(nodeA)
cb_stat_obj = Cbstats(shell_conn)
dcp_client = self.dcp_client(nodeA, dcp.constants.PRODUCER)
_ = dcp_client.stream_req(vbucket, 0, 0, 2 * self.num_items, 0)
self.load_docs(nodeA, vbucket, self.num_items)
# stop all nodes
node_range = range(len(self.servers))
for i in node_range:
self.assertTrue(self.stop_node(i), msg="Failed during stoip_node")
self.sleep(2, "Wait after stop_node")
# start all nodes in reverse order
node_range.reverse()
for i in node_range:
self.assertTrue(self.start_node(i), msg="Failed during start_node")
rest = RestHelper(RestConnection(nodeA))
self.assertTrue(rest.is_ns_server_running(),
msg="Failed while is_ns_server_running check")
vb_info = cb_stat_obj.vbucket_seqno(bucket.name)
dcp_client = self.dcp_client(nodeA, dcp.constants.PRODUCER)
stream = dcp_client.stream_req(vbucket, 0, 0,
vb_info[vbucket]["high_seqno"], 0)
stream.run()
self.assertTrue(stream.last_by_seqno == vb_info[vbucket]["high_seqno"],
msg="Seq-no mismatch. {0} != {1}".format(
stream.last_by_seqno,
vb_info[vbucket]["high_seqno"]))
# Disconnect shell Connection for the node
shell_conn.disconnect()
def test_crash_while_streaming(self):
vbucket = 0
nodeA = self.servers[0]
n = 10000
self.load_docs(nodeA, vbucket, n)
dcp_client = self.dcp_client(nodeA, PRODUCER)
stream = dcp_client.stream_req(vbucket, 0, 0, 2*n, 0)
self.load_docs(nodeA, vbucket, n)
assert self.stop_node(0)
time.sleep(2)
assert self.start_node(0)
rest = RestHelper(RestConnection(nodeA))
assert rest.is_ns_server_running()
time.sleep(2)
_, _, high_seqno = self.vb_info(nodeA, vbucket)
dcp_client = self.dcp_client(nodeA, PRODUCER)
stream = dcp_client.stream_req(vbucket, 0, 0, high_seqno, 0)
stream.run()
assert stream.last_by_seqno == high_seqno
def test_stream_after_n_crashes(self):
crashes = 5
vbucket = 0
# load some data
nodeA = self.servers[0]
rest = RestHelper(RestConnection(nodeA))
for i in xrange(crashes):
self.load_docs(nodeA, vbucket, self.num_items)
assert self.stop_node(0)
time.sleep(5)
assert self.start_node(0)
assert rest.is_ns_server_running()
time.sleep(2)
vb_uuid, _, high_seqno = self.vb_info(nodeA, vbucket)
dcp_client = self.dcp_client(nodeA, PRODUCER)
stream = dcp_client.stream_req(vbucket, 0, 0, high_seqno, vb_uuid)
stream.run()
assert stream.last_by_seqno == high_seqno
def test_crash_while_streaming(self):
vbucket = 0
nodeA = self.servers[0]
n = 10000
self.load_docs(nodeA, vbucket, n)
dcp_client = self.dcp_client(nodeA, PRODUCER)
stream = dcp_client.stream_req(vbucket, 0, 0, 2*n, 0)
self.load_docs(nodeA, vbucket, n)
assert self.stop_node(0)
time.sleep(2)
assert self.start_node(0)
rest = RestHelper(RestConnection(nodeA))
assert rest.is_ns_server_running()
time.sleep(30)
_, _, high_seqno = self.vb_info(nodeA, vbucket)
dcp_client = self.dcp_client(nodeA, PRODUCER)
stream = dcp_client.stream_req(vbucket, 0, 0, high_seqno, 0)
stream.run()
assert stream.last_by_seqno == high_seqno
def test_stream_after_n_crashes(self):
crashes = 5
vbucket = 0
# load some data
nodeA = self.servers[0]
rest = RestHelper(RestConnection(nodeA))
for i in xrange(crashes):
self.load_docs(nodeA, vbucket, self.num_items)
assert self.stop_node(0)
time.sleep(5)
assert self.start_node(0)
assert rest.is_ns_server_running()
time.sleep(2)
vb_uuid, _, high_seqno = self.vb_info(nodeA, vbucket)
dcp_client = self.dcp_client(nodeA, PRODUCER)
stream = dcp_client.stream_req(
vbucket, 0, 0,
high_seqno, vb_uuid)
stream.run()
assert stream.last_by_seqno == high_seqno
def test_stream_after_warmup(self):
nodeA = self.cluster.servers[0]
bucket = self.bucket_util.buckets[1]
shell_conn = RemoteMachineShellConnection(nodeA)
cb_stat_obj = Cbstats(shell_conn)
# load all buckets
doc_gen = doc_generator(self.key, 0, self.num_items)
self._load_all_buckets(self.master, doc_gen, "create", 0)
self._wait_for_stats_all_buckets()
# store expected vb seqnos
originalVbInfo = self.all_vb_info(cb_stat_obj, bucket.name)
# restart node
self.assertTrue(self.stop_node(0), msg="Failed during stop_node")
self.sleep(5, "Wait after stop_node")
self.assertTrue(self.start_node(0), msg="Failed during start_node")
rest = RestHelper(RestConnection(nodeA))
self.assertTrue(rest.is_ns_server_running(),
msg="Failed while is_ns_server_running")
self.sleep(2, "Wait after ns_server_start")
# verify original vbInfo can be streamed
dcp_client = self.dcp_client(nodeA, PRODUCER, bucket_name=bucket.name)
for vbucket in originalVbInfo:
vb_uuid, _, high_seqno = originalVbInfo[vbucket]
stream = dcp_client.stream_req(vbucket, 0, 0, high_seqno, vb_uuid)
_ = stream.run()
self.assertTrue(high_seqno == stream.last_by_seqno,
msg="Mismatch in high_seqno. {0} == {1}".format(
high_seqno, stream.last_by_seqno))
# Disconnect the shell_conn
shell_conn.disconnect()
class NewUpgradeBaseTest(BaseTestCase):
def setUp(self):
super(NewUpgradeBaseTest, self).setUp()
self.product = self.input.param('product', 'couchbase-server')
self.initial_version = self.input.param('initial_version',
'1.8.1-942-rel')
self.initial_vbuckets = self.input.param('initial_vbuckets', 64)
self.rest_settings = self.input.membase_settings
self.rest = RestConnection(self.master)
self.rest_helper = RestHelper(self.rest)
self.sleep_time = 10
self.data_size = self.input.param('data_size', 1024)
self.op_types = self.input.param('op_types', 'bucket')
self.item_flag = self.input.param('item_flag', 4042322160)
self.expire_time = self.input.param('expire_time', 0)
def tearDown(self):
super(NewUpgradeBaseTest, self).tearDown()
def _install(self, servers):
params = {}
params['num_nodes'] = len(servers)
params['product'] = self.product
params['version'] = self.initial_version
params['vbuckets'] = [self.initial_vbuckets]
InstallerJob().parallel_install(servers, params)
if self.product in ["couchbase", "couchbase-server", "cb"]:
success = True
for server in servers:
success &= RemoteMachineShellConnection(
server).is_couchbase_installed()
if not success:
self.log.info("some nodes were not install successfully!")
sys.exit(1)
def operations(self, multi_nodes=False):
self.quota = self._initialize_nodes(self.cluster, self.servers,
self.disabled_consistent_view)
self.buckets = []
gc.collect()
if self.total_buckets > 0:
self.bucket_size = self._get_bucket_size(self.quota,
self.total_buckets)
if self.default_bucket:
self.cluster.create_default_bucket(self.master, self.bucket_size,
self.num_replicas)
self.buckets.append(
Bucket(name="default",
authType="sasl",
saslPassword="",
num_replicas=self.num_replicas,
bucket_size=self.bucket_size))
self._create_sasl_buckets(self.master, self.sasl_buckets)
self._create_standard_buckets(self.master, self.standard_buckets)
if multi_nodes:
servers_in = [
self.servers[i + 1]
for i in range(self.initial_num_servers - 1)
]
self.cluster.rebalance(self.servers[:1], servers_in, [])
if self.op_types == "data":
self._load_data_all_buckets("create")
if multi_nodes:
self._wait_for_stats_all_buckets(
self.servers[:self.initial_num_servers])
else:
self._wait_for_stats_all_buckets([self.master])
def _load_data_all_buckets(self, op_type='create', start=0):
loaded = False
count = 0
gen_load = BlobGenerator('upgrade-',
'upgrade-',
self.data_size,
start=start,
end=self.num_items)
while not loaded and count < 60:
try:
self._load_all_buckets(self.master,
gen_load,
op_type,
self.expire_time,
1,
self.item_flag,
True,
batch_size=20000,
pause_secs=5,
timeout_secs=180)
loaded = True
except MemcachedError as error:
if error.status == 134:
loaded = False
self.log.error(
"Memcached error 134, wait for 5 seconds and then try again"
)
count += 1
time.sleep(self.sleep_time)
def _get_build(self, server, version, remote, is_amazon=False):
info = remote.extract_remote_info()
builds, changes = BuildQuery().get_all_builds()
self.log.info("finding build %s for machine %s" % (version, server))
result = re.search('r', version)
if result is None:
appropriate_build = BuildQuery().\
find_membase_release_build('%s-enterprise' % (self.product), info.deliverable_type,
info.architecture_type, version.strip(), is_amazon=is_amazon)
else:
appropriate_build = BuildQuery().\
find_membase_build(builds, '%s-enterprise' % (self.product), info.deliverable_type,
info.architecture_type, version.strip(), is_amazon=is_amazon)
return appropriate_build
def _upgrade(self, upgrade_version, server, remote):
appropriate_build = self._get_build(server, upgrade_version, remote)
self.assertTrue(appropriate_build.url,
msg="unable to find build {0}".format(upgrade_version))
remote.download_build(appropriate_build)
remote.membase_upgrade(appropriate_build, save_upgrade_config=False)
self.rest_helper.is_ns_server_running(testconstants.NS_SERVER_TIMEOUT)
self.rest.init_cluster_port(self.rest_settings.rest_username,
self.rest_settings.rest_password)
time.sleep(self.sleep_time)
def verification(self, multi_nodes=False):
for bucket in self.buckets:
if self.rest_helper.bucket_exists(bucket.name):
continue
else:
raise Exception("bucket:- %s not found" % bucket.name)
if self.op_types == "bucket":
bucketinfo = self.rest.get_bucket(bucket.name)
self.log.info("bucket info :- %s" % bucketinfo)
if self.op_types == "data":
if multi_nodes:
self._wait_for_stats_all_buckets(
self.servers[:self.num_servers])
self._verify_all_buckets(self.master, 1,
self.wait_timeout * 50,
self.max_verify, True, 1)
self._verify_stats_all_buckets(self.servers[:self.num_servers])
else:
self._wait_for_stats_all_buckets([self.master])
self._verify_all_buckets(self.master, 1,
self.wait_timeout * 50,
self.max_verify, True, 1)
self._verify_stats_all_buckets([self.master])
class NewUpgradeBaseTest(BaseTestCase):
def setUp(self):
super(NewUpgradeBaseTest, self).setUp()
self.product = self.input.param('product', 'couchbase-server')
self.initial_version = self.input.param('initial_version', '1.8.1-942-rel')
self.initial_vbuckets = self.input.param('initial_vbuckets', 64)
self.rest_settings = self.input.membase_settings
self.rest = RestConnection(self.master)
self.rest_helper = RestHelper(self.rest)
self.sleep_time = 10
self.data_size = self.input.param('data_size', 1024)
self.op_types = self.input.param('op_types', 'bucket')
self.item_flag = self.input.param('item_flag', 4042322160)
self.expire_time = self.input.param('expire_time', 0)
def tearDown(self):
super(NewUpgradeBaseTest, self).tearDown()
def _install(self, servers):
params = {}
params['num_nodes'] = len(servers)
params['product'] = self.product
params['version'] = self.initial_version
params['vbuckets'] = [self.initial_vbuckets]
InstallerJob().parallel_install(servers, params)
if self.product in ["couchbase", "couchbase-server", "cb"]:
success = True
for server in servers:
success &= RemoteMachineShellConnection(server).is_couchbase_installed()
if not success:
self.log.info("some nodes were not install successfully!")
sys.exit(1)
def operations(self, multi_nodes=False):
self.quota = self._initialize_nodes(self.cluster, self.servers, self.disabled_consistent_view)
self.buckets = []
gc.collect()
if self.total_buckets > 0:
self.bucket_size = self._get_bucket_size(self.quota, self.total_buckets)
if self.default_bucket:
self.cluster.create_default_bucket(self.master, self.bucket_size, self.num_replicas)
self.buckets.append(Bucket(name="default", authType="sasl", saslPassword="",
num_replicas=self.num_replicas, bucket_size=self.bucket_size))
self._create_sasl_buckets(self.master, self.sasl_buckets)
self._create_standard_buckets(self.master, self.standard_buckets)
if multi_nodes:
servers_in = [self.servers[i+1] for i in range(self.initial_num_servers-1)]
self.cluster.rebalance(self.servers[:1], servers_in, [])
if self.op_types == "data":
self._load_data_all_buckets("create")
if multi_nodes:
self._wait_for_stats_all_buckets(self.servers[:self.initial_num_servers])
else:
self._wait_for_stats_all_buckets([self.master])
def _load_data_all_buckets(self, op_type='create', start=0):
loaded = False
count = 0
gen_load = BlobGenerator('upgrade-', 'upgrade-', self.data_size, start=start, end=self.num_items)
while not loaded and count < 60:
try :
self._load_all_buckets(self.master, gen_load, op_type, self.expire_time, 1,
self.item_flag, True, batch_size=20000, pause_secs=5, timeout_secs=180)
loaded = True
except MemcachedError as error:
if error.status == 134:
loaded = False
self.log.error("Memcached error 134, wait for 5 seconds and then try again")
count += 1
time.sleep(self.sleep_time)
def _get_build(self, server, version, remote, is_amazon=False):
info = remote.extract_remote_info()
builds, changes = BuildQuery().get_all_builds()
self.log.info("finding build %s for machine %s" % (version, server))
result = re.search('r', version)
if result is None:
appropriate_build = BuildQuery().\
find_membase_release_build('%s-enterprise' % (self.product), info.deliverable_type,
info.architecture_type, version.strip(), is_amazon=is_amazon)
else:
appropriate_build = BuildQuery().\
find_membase_build(builds, '%s-enterprise' % (self.product), info.deliverable_type,
info.architecture_type, version.strip(), is_amazon=is_amazon)
return appropriate_build
def _upgrade(self, upgrade_version, server, remote):
appropriate_build = self._get_build(server, upgrade_version, remote)
self.assertTrue(appropriate_build.url, msg="unable to find build {0}".format(upgrade_version))
remote.download_build(appropriate_build)
remote.membase_upgrade(appropriate_build, save_upgrade_config=False)
self.rest_helper.is_ns_server_running(testconstants.NS_SERVER_TIMEOUT)
self.rest.init_cluster_port(self.rest_settings.rest_username, self.rest_settings.rest_password)
time.sleep(self.sleep_time)
def verification(self, multi_nodes=False):
for bucket in self.buckets:
if self.rest_helper.bucket_exists(bucket.name):
continue
else:
raise Exception("bucket:- %s not found" % bucket.name)
if self.op_types == "bucket":
bucketinfo = self.rest.get_bucket(bucket.name)
self.log.info("bucket info :- %s" % bucketinfo)
if self.op_types == "data":
if multi_nodes:
self._wait_for_stats_all_buckets(self.servers[:self.num_servers])
self._verify_all_buckets(self.master, 1, self.wait_timeout*50, self.max_verify, True, 1)
self._verify_stats_all_buckets(self.servers[:self.num_servers])
else:
self._wait_for_stats_all_buckets([self.master])
self._verify_all_buckets(self.master, 1, self.wait_timeout*50, self.max_verify, True, 1)
self._verify_stats_all_buckets([self.master])
