def test_get_machine_uuids_for_application(self): self.patch_object(juju_utils, "get_machines_for_application") self.get_machines_for_application.return_value = [self.machine] self.assertEqual( next(juju_utils.get_machine_uuids_for_application( self.application)), self.machine_data.get("instance-id")) self.get_machines_for_application.assert_called_once_with( self.application, model_name=None)
def setUpClass(cls): """Run class setup for running percona-cluster cold start tests.""" super(PerconaClusterColdStartTest, cls).setUpClass() cls.overcloud_keystone_session = ( openstack_utils.get_undercloud_keystone_session()) cls.nova_client = openstack_utils.get_nova_session_client( cls.overcloud_keystone_session) cls.machines = (juju_utils.get_machine_uuids_for_application( cls.application))
def test_100_reboot_cluster_from_complete_outage(self): """Reboot cluster from complete outage. After a cold start, reboot cluster from complete outage. """ _machines = sorted( juju_utils.get_machine_uuids_for_application(self.application)) # Stop Nodes # Avoid hitting an update-status hook logging.info("Wait till model is idle ...") zaza.model.block_until_all_units_idle() logging.info("Stopping instances: {}".format(_machines)) for uuid in _machines: self.nova_client.servers.stop(uuid) logging.info("Wait till all machines are shutoff ...") for uuid in _machines: openstack_utils.resource_reaches_status(self.nova_client.servers, uuid, expected_status='SHUTOFF', stop_after_attempt=16) # Start nodes _machines.sort(reverse=True) logging.info("Starting instances: {}".format(_machines)) for uuid in _machines: self.nova_client.servers.start(uuid) logging.info("Wait till all {} units are in state 'unkown' ...".format( self.application)) for unit in zaza.model.get_units(self.application): zaza.model.block_until_unit_wl_status(unit.entity_id, 'unknown', negate_match=True) logging.info("Wait till model is idle ...") try: zaza.model.block_until_all_units_idle() except zaza.model.UnitError: self.resolve_update_status_errors() zaza.model.block_until_all_units_idle() logging.info("Clear error hooks after reboot ...") for unit in zaza.model.get_units(self.application): try: zaza.model.run_on_unit(unit.entity_id, "hooks/update-status") except zaza.model.UnitError: self.resolve_update_status_errors() zaza.model.run_on_unit(unit.entity_id, "hooks/update-status") logging.info( "Wait till all {} units are in state 'blocked' ...".format( self.application)) for unit in zaza.model.get_units(self.application): zaza.model.block_until_unit_wl_status(unit.entity_id, 'blocked') logging.info("Execute reboot-cluster-from-complete-outage " "action after cold boot ...") # We do not know which unit has the most up to date data # run reboot-cluster-from-complete-outage until we get a success. for unit in zaza.model.get_units(self.application): action = zaza.model.run_action( unit.entity_id, "reboot-cluster-from-complete-outage", action_params={}) if "Success" in action.data.get("results", {}).get("outcome", ""): break else: logging.info(action.data.get("results", {}).get("output", "")) assert "Success" in action.data["results"]["outcome"], ( "Reboot cluster from complete outage action failed: {}".format( action.data)) logging.info("Wait for application states ...") for unit in zaza.model.get_units(self.application): zaza.model.run_on_unit(unit.entity_id, "hooks/update-status") test_config = lifecycle_utils.get_charm_config(fatal=False) zaza.model.wait_for_application_states( states=test_config.get("target_deploy_status", {}))
def test_100_cold_start_bootstrap(self): """Bootstrap a non-leader node. After bootstrapping a non-leader node, notify bootstrapped on the leader node. """ _machines = sorted( juju_utils.get_machine_uuids_for_application(self.application)) # Stop Nodes # Avoid hitting an update-status hook logging.info("Wait till model is idle ...") zaza.model.block_until_all_units_idle() logging.info("Stopping instances: {}".format(_machines)) for uuid in _machines: self.nova_client.servers.stop(uuid) logging.info("Wait till all machines are shutoff ...") for uuid in _machines: openstack_utils.resource_reaches_status(self.nova_client.servers, uuid, expected_status='SHUTOFF', stop_after_attempt=16) # Start nodes _machines.sort(reverse=True) logging.info("Starting instances: {}".format(_machines)) for uuid in _machines: self.nova_client.servers.start(uuid) for unit in zaza.model.get_units(self.application): zaza.model.block_until_unit_wl_status(unit.entity_id, 'unknown', negate_match=True) logging.info("Wait till model is idle ...") # XXX If a hook was executing on a unit when it was powered off # it comes back in an error state. try: zaza.model.block_until_all_units_idle() except zaza.model.UnitError: self.resolve_update_status_errors() zaza.model.block_until_all_units_idle() logging.info("Wait for application states ...") for unit in zaza.model.get_units(self.application): try: zaza.model.run_on_unit(unit.entity_id, "hooks/update-status") except zaza.model.UnitError: self.resolve_update_status_errors() zaza.model.run_on_unit(unit.entity_id, "hooks/update-status") states = { "percona-cluster": { "workload-status": "blocked", "workload-status-message": "MySQL is down" } } zaza.model.wait_for_application_states(states=states) # Update which node is the leader and which are not _leader, _non_leaders = self.get_leaders_and_non_leaders() # We want to test the worst possible scenario which is the # non-leader with the highest sequence number. We will use the leader # for the notify-bootstrapped after. They just need to be different # units. logging.info("Execute bootstrap-pxc action after cold boot ...") zaza.model.run_action(_non_leaders[0], "bootstrap-pxc", action_params={}) logging.info("Wait for application states ...") for unit in zaza.model.get_units(self.application): zaza.model.run_on_unit(unit.entity_id, "hooks/update-status") states = { "percona-cluster": { "workload-status": "waiting", "workload-status-message": "Unit waiting for cluster bootstrap" } } zaza.model.wait_for_application_states(states=states) logging.info("Execute notify-bootstrapped action after cold boot on " "the leader node ...") zaza.model.run_action_on_leader(self.application, "notify-bootstrapped", action_params={}) logging.info("Wait for application states ...") for unit in zaza.model.get_units(self.application): zaza.model.run_on_unit(unit.entity_id, "hooks/update-status") test_config = lifecycle_utils.get_charm_config(fatal=False) zaza.model.wait_for_application_states( states=test_config.get("target_deploy_status", {}))
def test_100_reboot_cluster_from_complete_outage(self): """Reboot cluster from complete outage. After a cold start, reboot cluster from complete outage. """ _machines = list( juju_utils.get_machine_uuids_for_application(self.application)) # Stop Nodes _machines.sort() # Avoid hitting an update-status hook logging.debug("Wait till model is idle ...") zaza.model.block_until_all_units_idle() logging.info("Stopping instances: {}".format(_machines)) for uuid in _machines: self.nova_client.servers.stop(uuid) logging.debug("Wait till all machines are shutoff ...") for uuid in _machines: openstack_utils.resource_reaches_status(self.nova_client.servers, uuid, expected_status='SHUTOFF', stop_after_attempt=16) # Start nodes _machines.sort(reverse=True) logging.info("Starting instances: {}".format(_machines)) for uuid in _machines: self.nova_client.servers.start(uuid) for unit in zaza.model.get_units(self.application): zaza.model.block_until_unit_wl_status( unit.entity_id, 'unknown', negate_match=True) logging.debug("Wait till model is idle ...") try: zaza.model.block_until_all_units_idle() except zaza.model.UnitError: self.resolve_update_status_errors() zaza.model.block_until_all_units_idle() logging.debug("Wait for application states ...") for unit in zaza.model.get_units(self.application): try: zaza.model.run_on_unit(unit.entity_id, "hooks/update-status") except zaza.model.UnitError: self.resolve_update_status_errors() zaza.model.run_on_unit(unit.entity_id, "hooks/update-status") states = {self.application: { "workload-status": "blocked", "workload-status-message": "MySQL InnoDB Cluster not healthy: None"}} zaza.model.wait_for_application_states(states=states) logging.info("Execute reboot-cluster-from-complete-outage " "action after cold boot ...") action = zaza.model.run_action_on_leader( self.application, "reboot-cluster-from-complete-outage", action_params={}) assert "Success" in action.data["results"]["outcome"], ( "Reboot cluster from complete outage action failed: {}" .format(action.data)) logging.debug("Wait for application states ...") for unit in zaza.model.get_units(self.application): zaza.model.run_on_unit(unit.entity_id, "hooks/update-status") test_config = lifecycle_utils.get_charm_config(fatal=False) zaza.model.wait_for_application_states( states=test_config.get("target_deploy_status", {}))