def node_stop_scenario(self, instance_kill_count, node, timeout): for _ in range(instance_kill_count): try: logging.info("Starting node_stop_scenario injection") instance_id = self.aws.get_instance_id(node) logging.info("Stopping the node %s with instance ID: %s " % (node, instance_id)) self.aws.stop_instances(instance_id) self.aws.wait_until_stopped(instance_id) logging.info("Node with instance ID: %s is in stopped state" % (instance_id)) nodeaction.wait_for_unknown_status(node, timeout) except Exception as e: logging.error("Failed to stop node instance. Encountered following exception: %s. " "Test Failed" % (e)) logging.error("node_stop_scenario injection failed!") sys.exit(1)
def node_reboot_scenario(self, instance_kill_count, node, timeout): for _ in range(instance_kill_count): try: logging.info("Starting node_reboot_scenario injection" + str(node)) instance_id = self.aws.get_instance_id(node) logging.info("Rebooting the node %s with instance ID: %s " % (node, instance_id)) self.aws.reboot_instances(instance_id) nodeaction.wait_for_unknown_status(node, timeout) nodeaction.wait_for_ready_status(node, timeout) logging.info("Node with instance ID: %s has been rebooted" % (instance_id)) logging.info("node_reboot_scenario has been successfuly injected!") except Exception as e: logging.error( "Failed to reboot node instance. Encountered following exception:" " %s. Test Failed" % (e) ) logging.error("node_reboot_scenario injection failed!") sys.exit(1)
def node_stop_scenario(self, instance_kill_count, node, timeout): for _ in range(instance_kill_count): try: logging.info("Starting node_stop_scenario injection") bmc_addr = self.bm.get_bmc_addr(node) logging.info("Stopping the node %s with bmc address: %s " % (node, bmc_addr)) self.bm.stop_instances(bmc_addr, node) self.bm.wait_until_stopped(bmc_addr, node) logging.info("Node with bmc address: %s is in stopped state" % (bmc_addr)) nodeaction.wait_for_unknown_status(node, timeout) except Exception as e: logging.error( "Failed to stop node instance. Encountered following exception: %s. " "Test Failed. Most errors are caused by " "an incorrect ipmi address or login" % (e) ) logging.error("node_stop_scenario injection failed!") sys.exit(1)
def stop_kubelet_scenario(self, instance_kill_count, node, timeout): for _ in range(instance_kill_count): try: logging.info("Starting stop_kubelet_scenario injection") logging.info("Stopping the kubelet of the node %s" % (node)) runcommand.run("oc debug node/" + node + " -- chroot /host systemctl stop kubelet") nodeaction.wait_for_unknown_status(node, timeout) logging.info("The kubelet of the node %s has been stopped" % (node)) logging.info( "stop_kubelet_scenario has been successfuly injected!") except Exception as e: logging.error( "Failed to stop the kubelet of the node. Encountered following " "exception: %s. Test Failed" % (e)) logging.error("stop_kubelet_scenario injection failed!") sys.exit(1)
def node_stop_scenario(self, instance_kill_count, node, timeout): for _ in range(instance_kill_count): try: logging.info("Starting node_stop_scenario injection") vm_name, resource_group = self.azure.get_instance_id(node) logging.info("Stopping the node %s with instance ID: %s " % (vm_name, resource_group)) self.azure.stop_instances(resource_group, vm_name) self.azure.wait_until_stopped(resource_group, vm_name, timeout) logging.info("Node with instance ID: %s is in stopped state" % vm_name) nodeaction.wait_for_unknown_status(vm_name, timeout) except Exception as e: logging.error( "Failed to stop node instance. Encountered following exception: %s. " "Test Failed" % e) logging.error("node_stop_scenario injection failed!") sys.exit(1)
def node_reboot_scenario(self, instance_kill_count, node, timeout): for _ in range(instance_kill_count): try: logging.info("Starting node_reboot_scenario injection") bmc_addr = self.bm.get_bmc_addr(node) logging.info("BMC Addr: %s" % (bmc_addr)) logging.info("Rebooting the node %s with bmc address: %s " % (node, bmc_addr)) self.bm.reboot_instances(bmc_addr, node) nodeaction.wait_for_unknown_status(node, timeout) nodeaction.wait_for_ready_status(node, timeout) logging.info("Node with bmc address: %s has been rebooted" % (bmc_addr)) logging.info("node_reboot_scenario has been successfuly injected!") except Exception as e: logging.error( "Failed to reboot node instance. Encountered following exception:" " %s. Test Failed. Most errors are caused by " "an incorrect ipmi address or login" % (e) ) traceback.print_exc() logging.error("node_reboot_scenario injection failed!") sys.exit(1)
def node_reboot_scenario(self, instance_kill_count, node, timeout): for _ in range(instance_kill_count): try: logging.info("Starting node_reboot_scenario injection") logging.info("Rebooting the node %s" % (node)) openstack_node_ip = nodeaction.get_node_ip(node) openstack_node_name = self.openstackcloud.get_openstack_nodename( openstack_node_ip) self.openstackcloud.reboot_instances(openstack_node_name) nodeaction.wait_for_unknown_status(node, timeout) nodeaction.wait_for_ready_status(node, timeout) logging.info("Node with instance name: %s has been rebooted" % (node)) logging.info( "node_reboot_scenario has been successfuly injected!") except Exception as e: logging.error( "Failed to reboot node instance. Encountered following exception:" " %s. Test Failed" % (e)) logging.error("node_reboot_scenario injection failed!") sys.exit(1)