def test_update_controller_ip_port_list(self):
"""Test update_controller_ip_port_list.
Add a unit and then delete a unit, then query the list of ports to
check that the port has been deleted.
"""
raise unittest.SkipTest("Skipping because of lp:1951858")
app = self.test_config['charm_name']
logging.info("test_update_controller_ip_port_list: start test")
logging.info("Wait till model is idle ...")
zaza.model.block_until_all_units_idle()
ips = self.get_port_ips()
num = len(ips)
logging.info('initial hm port num is {}: {}'.format(num, ips))
logging.info("test_update_controller_ip_port_list: add one unit")
logging.info("Adding one unit ...")
zaza.model.add_unit(app)
logging.info("Wait until one unit is added ...")
zaza.model.block_until_unit_count(app, num+1)
zaza.model.wait_for_application_states()
ips = self.get_port_ips()
logging.info('hm ports are {} after adding one unit'.format(ips))
self.assertTrue(len(ips) == num+1)
logging.info("test_update_controller_ip_port_list: remove one unit")
logging.info("Removing one unit ...")
_, nons = generic_utils.get_leaders_and_non_leaders(app)
zaza.model.destroy_unit(app, nons[0])
logging.info("Wait until one unit is deleted ...")
zaza.model.block_until_unit_count(app, num)
zaza.model.wait_for_application_states()
ips = self.get_port_ips()
logging.info('hm ports are {} after deleting one unit'.format(ips))
self.assertTrue(len(ips) == num)
def test_803_remove_fourth(self):
"""Remove mysql-innodb-cluster node.
We start with a four node full cluster, remove one, down to a three
node full cluster.
"""
leader, nons = generic_utils.get_leaders_and_non_leaders(
self.application_name)
non_leader_unit = zaza.model.get_unit_from_name(nons[0])
non_leader_unit_ip = zaza.model.get_unit_public_address(
non_leader_unit)
# Wait until we are idle in the hopes clients are not running
# update-status hooks
logging.info("Wait till model is idle ...")
zaza.model.block_until_all_units_idle()
zaza.model.destroy_unit(self.application_name, nons[0])
logging.info("Scale in test: back down to three")
logging.info("Wait until 3 units ...")
zaza.model.block_until_unit_count(self.application, 3)
logging.info("Wait for status ready ...")
zaza.model.wait_for_application_states(states=self.states)
# Show status
logging.info(self.get_cluster_status())
logging.info(
"Removing old unit from cluster: {} ".format(non_leader_unit_ip))
action = zaza.model.run_action(leader,
"remove-instance",
action_params={
"address": non_leader_unit_ip,
"force": True
})
assert action.data.get("results") is not None, (
"Remove instance action failed: No results: {}".format(
action.data))
def test_800_remove_leader(self):
"""Remove leader node.
We start with a three node cluster, remove one, down to two.
The cluster will be in waiting state.
"""
logging.info("Scale in test: remove leader")
leader, nons = generic_utils.get_leaders_and_non_leaders(
self.application_name)
leader_unit = zaza.model.get_unit_from_name(leader)
# Wait until we are idle in the hopes clients are not running
# update-status hooks
logging.info("Wait till model is idle ...")
zaza.model.block_until_all_units_idle()
zaza.model.destroy_unit(self.application_name, leader)
logging.info("Wait until all only 2 units ...")
zaza.model.block_until_unit_count(self.application, 2)
logging.info("Wait until all units are cluster incomplete ...")
zaza.model.block_until_wl_status_info_starts_with(
self.application, "'cluster' incomplete")
# Show status
logging.info(self.get_cluster_status())
logging.info(
"Removing old unit from cluster: {} "
.format(zaza.model.get_unit_public_address(leader_unit)))
action = zaza.model.run_action(
nons[0],
"remove-instance",
action_params={
"address": zaza.model.get_unit_public_address(leader_unit),
"force": True})
assert action.data.get("results") is not None, (
"Remove instance action failed: No results: {}"
.format(action.data))
def test_100_cold_start_bootstrap(self):
"""Bootstrap a non-leader node.
After bootstrapping a non-leader node, notify bootstrapped on the
leader node.
"""
_machines = sorted(
juju_utils.get_machine_uuids_for_application(self.application))
# Stop Nodes
# Avoid hitting an update-status hook
logging.info("Wait till model is idle ...")
zaza.model.block_until_all_units_idle()
logging.info("Stopping instances: {}".format(_machines))
for uuid in _machines:
self.nova_client.servers.stop(uuid)
logging.info("Wait till all machines are shutoff ...")
for uuid in _machines:
openstack_utils.resource_reaches_status(self.nova_client.servers,
uuid,
expected_status='SHUTOFF',
stop_after_attempt=16)
# Start nodes
_machines.sort(reverse=True)
logging.info("Starting instances: {}".format(_machines))
for uuid in _machines:
self.nova_client.servers.start(uuid)
for unit in zaza.model.get_units(self.application):
zaza.model.block_until_unit_wl_status(
unit.entity_id,
'unknown',
negate_match=True)
logging.info("Wait till model is idle ...")
# XXX If a hook was executing on a unit when it was powered off
# it comes back in an error state.
try:
zaza.model.block_until_all_units_idle()
except zaza.model.UnitError:
self.resolve_update_status_errors()
zaza.model.block_until_all_units_idle()
logging.info("Wait for application states ...")
for unit in zaza.model.get_units(self.application):
try:
zaza.model.run_on_unit(unit.entity_id, "hooks/update-status")
except zaza.model.UnitError:
self.resolve_update_status_errors()
zaza.model.run_on_unit(unit.entity_id, "hooks/update-status")
states = {"percona-cluster": {
"workload-status": "blocked",
"workload-status-message": "MySQL is down"}}
zaza.model.wait_for_application_states(states=states)
# Update which node is the leader and which are not
_leader, _non_leaders = generic_utils.get_leaders_and_non_leaders(
self.application_name)
# We want to test the worst possible scenario which is the
# non-leader with the highest sequence number. We will use the leader
# for the notify-bootstrapped after. They just need to be different
# units.
logging.info("Execute bootstrap-pxc action after cold boot ...")
zaza.model.run_action(
_non_leaders[0],
"bootstrap-pxc",
action_params={})
logging.info("Wait for application states ...")
for unit in zaza.model.get_units(self.application):
zaza.model.run_on_unit(unit.entity_id, "hooks/update-status")
states = {"percona-cluster": {
"workload-status": "waiting",
"workload-status-message": "Unit waiting for cluster bootstrap"}}
zaza.model.wait_for_application_states(
states=states)
logging.info("Execute notify-bootstrapped action after cold boot on "
"the leader node ...")
zaza.model.run_action_on_leader(
self.application,
"notify-bootstrapped",
action_params={})
logging.info("Wait for application states ...")
for unit in zaza.model.get_units(self.application):
zaza.model.run_on_unit(unit.entity_id, "hooks/update-status")
test_config = lifecycle_utils.get_charm_config(fatal=False)
zaza.model.wait_for_application_states(
states=test_config.get("target_deploy_status", {}))