def test_get_machine_uuids_for_application(self):
self.patch_object(juju_utils, "get_machines_for_application")
self.get_machines_for_application.return_value = [self.machine]
self.assertEqual(
next(juju_utils.get_machine_uuids_for_application(
self.application)), self.machine_data.get("instance-id"))
self.get_machines_for_application.assert_called_once_with(
self.application, model_name=None)
def setUpClass(cls):
"""Run class setup for running percona-cluster cold start tests."""
super(PerconaClusterColdStartTest, cls).setUpClass()
cls.overcloud_keystone_session = (
openstack_utils.get_undercloud_keystone_session())
cls.nova_client = openstack_utils.get_nova_session_client(
cls.overcloud_keystone_session)
cls.machines = (juju_utils.get_machine_uuids_for_application(
cls.application))
def test_100_reboot_cluster_from_complete_outage(self):
"""Reboot cluster from complete outage.
After a cold start, reboot cluster from complete outage.
"""
_machines = sorted(
juju_utils.get_machine_uuids_for_application(self.application))
# Stop Nodes
# Avoid hitting an update-status hook
logging.info("Wait till model is idle ...")
zaza.model.block_until_all_units_idle()
logging.info("Stopping instances: {}".format(_machines))
for uuid in _machines:
self.nova_client.servers.stop(uuid)
logging.info("Wait till all machines are shutoff ...")
for uuid in _machines:
openstack_utils.resource_reaches_status(self.nova_client.servers,
uuid,
expected_status='SHUTOFF',
stop_after_attempt=16)
# Start nodes
_machines.sort(reverse=True)
logging.info("Starting instances: {}".format(_machines))
for uuid in _machines:
self.nova_client.servers.start(uuid)
logging.info("Wait till all {} units are in state 'unkown' ...".format(
self.application))
for unit in zaza.model.get_units(self.application):
zaza.model.block_until_unit_wl_status(unit.entity_id,
'unknown',
negate_match=True)
logging.info("Wait till model is idle ...")
try:
zaza.model.block_until_all_units_idle()
except zaza.model.UnitError:
self.resolve_update_status_errors()
zaza.model.block_until_all_units_idle()
logging.info("Clear error hooks after reboot ...")
for unit in zaza.model.get_units(self.application):
try:
zaza.model.run_on_unit(unit.entity_id, "hooks/update-status")
except zaza.model.UnitError:
self.resolve_update_status_errors()
zaza.model.run_on_unit(unit.entity_id, "hooks/update-status")
logging.info(
"Wait till all {} units are in state 'blocked' ...".format(
self.application))
for unit in zaza.model.get_units(self.application):
zaza.model.block_until_unit_wl_status(unit.entity_id, 'blocked')
logging.info("Execute reboot-cluster-from-complete-outage "
"action after cold boot ...")
# We do not know which unit has the most up to date data
# run reboot-cluster-from-complete-outage until we get a success.
for unit in zaza.model.get_units(self.application):
action = zaza.model.run_action(
unit.entity_id,
"reboot-cluster-from-complete-outage",
action_params={})
if "Success" in action.data.get("results", {}).get("outcome", ""):
break
else:
logging.info(action.data.get("results", {}).get("output", ""))
assert "Success" in action.data["results"]["outcome"], (
"Reboot cluster from complete outage action failed: {}".format(
action.data))
logging.info("Wait for application states ...")
for unit in zaza.model.get_units(self.application):
zaza.model.run_on_unit(unit.entity_id, "hooks/update-status")
test_config = lifecycle_utils.get_charm_config(fatal=False)
zaza.model.wait_for_application_states(
states=test_config.get("target_deploy_status", {}))
def test_100_cold_start_bootstrap(self):
"""Bootstrap a non-leader node.
After bootstrapping a non-leader node, notify bootstrapped on the
leader node.
"""
_machines = sorted(
juju_utils.get_machine_uuids_for_application(self.application))
# Stop Nodes
# Avoid hitting an update-status hook
logging.info("Wait till model is idle ...")
zaza.model.block_until_all_units_idle()
logging.info("Stopping instances: {}".format(_machines))
for uuid in _machines:
self.nova_client.servers.stop(uuid)
logging.info("Wait till all machines are shutoff ...")
for uuid in _machines:
openstack_utils.resource_reaches_status(self.nova_client.servers,
uuid,
expected_status='SHUTOFF',
stop_after_attempt=16)
# Start nodes
_machines.sort(reverse=True)
logging.info("Starting instances: {}".format(_machines))
for uuid in _machines:
self.nova_client.servers.start(uuid)
for unit in zaza.model.get_units(self.application):
zaza.model.block_until_unit_wl_status(unit.entity_id,
'unknown',
negate_match=True)
logging.info("Wait till model is idle ...")
# XXX If a hook was executing on a unit when it was powered off
# it comes back in an error state.
try:
zaza.model.block_until_all_units_idle()
except zaza.model.UnitError:
self.resolve_update_status_errors()
zaza.model.block_until_all_units_idle()
logging.info("Wait for application states ...")
for unit in zaza.model.get_units(self.application):
try:
zaza.model.run_on_unit(unit.entity_id, "hooks/update-status")
except zaza.model.UnitError:
self.resolve_update_status_errors()
zaza.model.run_on_unit(unit.entity_id, "hooks/update-status")
states = {
"percona-cluster": {
"workload-status": "blocked",
"workload-status-message": "MySQL is down"
}
}
zaza.model.wait_for_application_states(states=states)
# Update which node is the leader and which are not
_leader, _non_leaders = self.get_leaders_and_non_leaders()
# We want to test the worst possible scenario which is the
# non-leader with the highest sequence number. We will use the leader
# for the notify-bootstrapped after. They just need to be different
# units.
logging.info("Execute bootstrap-pxc action after cold boot ...")
zaza.model.run_action(_non_leaders[0],
"bootstrap-pxc",
action_params={})
logging.info("Wait for application states ...")
for unit in zaza.model.get_units(self.application):
zaza.model.run_on_unit(unit.entity_id, "hooks/update-status")
states = {
"percona-cluster": {
"workload-status": "waiting",
"workload-status-message": "Unit waiting for cluster bootstrap"
}
}
zaza.model.wait_for_application_states(states=states)
logging.info("Execute notify-bootstrapped action after cold boot on "
"the leader node ...")
zaza.model.run_action_on_leader(self.application,
"notify-bootstrapped",
action_params={})
logging.info("Wait for application states ...")
for unit in zaza.model.get_units(self.application):
zaza.model.run_on_unit(unit.entity_id, "hooks/update-status")
test_config = lifecycle_utils.get_charm_config(fatal=False)
zaza.model.wait_for_application_states(
states=test_config.get("target_deploy_status", {}))
def test_100_reboot_cluster_from_complete_outage(self):
"""Reboot cluster from complete outage.
After a cold start, reboot cluster from complete outage.
"""
_machines = list(
juju_utils.get_machine_uuids_for_application(self.application))
# Stop Nodes
_machines.sort()
# Avoid hitting an update-status hook
logging.debug("Wait till model is idle ...")
zaza.model.block_until_all_units_idle()
logging.info("Stopping instances: {}".format(_machines))
for uuid in _machines:
self.nova_client.servers.stop(uuid)
logging.debug("Wait till all machines are shutoff ...")
for uuid in _machines:
openstack_utils.resource_reaches_status(self.nova_client.servers,
uuid,
expected_status='SHUTOFF',
stop_after_attempt=16)
# Start nodes
_machines.sort(reverse=True)
logging.info("Starting instances: {}".format(_machines))
for uuid in _machines:
self.nova_client.servers.start(uuid)
for unit in zaza.model.get_units(self.application):
zaza.model.block_until_unit_wl_status(
unit.entity_id,
'unknown',
negate_match=True)
logging.debug("Wait till model is idle ...")
try:
zaza.model.block_until_all_units_idle()
except zaza.model.UnitError:
self.resolve_update_status_errors()
zaza.model.block_until_all_units_idle()
logging.debug("Wait for application states ...")
for unit in zaza.model.get_units(self.application):
try:
zaza.model.run_on_unit(unit.entity_id, "hooks/update-status")
except zaza.model.UnitError:
self.resolve_update_status_errors()
zaza.model.run_on_unit(unit.entity_id, "hooks/update-status")
states = {self.application: {
"workload-status": "blocked",
"workload-status-message":
"MySQL InnoDB Cluster not healthy: None"}}
zaza.model.wait_for_application_states(states=states)
logging.info("Execute reboot-cluster-from-complete-outage "
"action after cold boot ...")
action = zaza.model.run_action_on_leader(
self.application,
"reboot-cluster-from-complete-outage",
action_params={})
assert "Success" in action.data["results"]["outcome"], (
"Reboot cluster from complete outage action failed: {}"
.format(action.data))
logging.debug("Wait for application states ...")
for unit in zaza.model.get_units(self.application):
zaza.model.run_on_unit(unit.entity_id, "hooks/update-status")
test_config = lifecycle_utils.get_charm_config(fatal=False)
zaza.model.wait_for_application_states(
states=test_config.get("target_deploy_status", {}))
