def node_disruption(self, sync=None, finish_execution=None): self.sync = sync self.finish_execution = finish_execution infra.display_on_terminal(self, "Entering Node Disruption plugin") table_name = "Node Disruption" infra.create_report_table(self, table_name) infra.add_table_headers(self, table_name, ["Node", "IP", "TimeStamp", "Status of Disruption"]) infra.display_on_terminal(self, "Entering Process Disruption plugin") input_args_dict = self.get_input_arguments() node_name = input_args_dict.keys()[0] input_args = input_args_dict.get(node_name, None) host_config = infra.get_openstack_config() if input_args: print "Inpt " + str(input_args) role = input_args.get('role', None) nodes_to_be_disrupted = [] for node in host_config: if role in host_config[node].get('role', None): infra.display_on_terminal(self, node, " will be disrupted ") nodes_to_be_disrupted.append(node) node_reboot_command = "reboot -f " if self.sync: infra.display_on_terminal(self, "Waiting for notification") infra.wait_for_notification(sync) infra.display_on_terminal(self, "Received notification, Starting") ha_interval = self.get_ha_interval() for i in range(1): #while infra.is_execution_completed(self.finish_execution) is False: # for node in nodes_to_be_disrupted: node = nodes_to_be_disrupted[0] ip = host_config.get(node, None).get('ip', None) user = host_config.get(node, None).get('user', None) password = host_config.get(node, None).get('password', None) infra.display_on_terminal(self, "IP: ", ip, " User: "******" Pwd: ", password) infra.display_on_terminal(self, "Executing ", node_reboot_command) code, out, error = infra.ssh_and_execute_command(ip, user, password, node_reboot_command) if error: infra.display_on_terminal(self, "Error ", error, "color=red") infra.display_on_terminal(self, "waiting for ", ip, " to " "come " "online") if infra.wait_for_ping(ip, 240, 10): infra.display_on_terminal(self, "Node ", ip, " is online", "color=green") infra.display_on_terminal(self, "Will sleep for interval ", str(ha_interval)) #time.sleep(ha_interval) infra.add_table_rows(self, table_name, [[node, ip, utils.get_timestamp(), HAConstants.OKGREEN + 'Rebooted' + HAConstants.ENDC]]) # bring it back to stable state infra.display_on_terminal(self, "Waiting for the node to become stable") if infra.wait_for_ping(ip, 240, 10): infra.display_on_terminal(self, "Node ", ip, " is in stable state", "color=green") infra.display_on_terminal(self, "Finishing Node Disruption")
def process_disruption(self, sync=None, finish_execution=None): self.sync = sync self.finish_execution = finish_execution table_name = "Process Disruption" infra.create_report_table(self, table_name) infra.add_table_headers( self, table_name, ["Host", "Process", "TimeStamp", "Status of Disruption"]) infra.display_on_terminal(self, "Entering Process Disruption plugin") input_args_dict = self.get_input_arguments() node_name = input_args_dict.keys()[0] input_args = input_args_dict.get(node_name, None) host_config = infra.get_openstack_config() if input_args: print "Inpt " + str(input_args) process_name = input_args.get('process_name', None) role = input_args.get('role', None) type = input_args.get('type', None) infra.display_on_terminal(self, "Process ", process_name, " will be disrupted") nodes_to_be_disrupted = [] for node in host_config: if 'controller' in host_config[node].get('role', None): infra.display_on_terminal(self, node, " will be disrupted ") nodes_to_be_disrupted.append(node) self.expected_failures.append(node + "::" + process_name) self.set_expected_failures(self.expected_failures) rhel_stop_command = "systemctl stop " + process_name rhel_start_command = "systemctl start " + process_name if sync: infra.display_on_terminal(self, "Waiting for notification") infra.wait_for_notification(sync) infra.display_on_terminal(self, "Received notification, Starting") ha_interval = self.get_ha_interval() disruption_count = self.get_disruption_count() infra.display_on_terminal(self, "Process will be disrupted ", str(disruption_count)) while infra.is_execution_completed(self.finish_execution) is False: if disruption_count: disruption_count = disruption_count - 1 for node in nodes_to_be_disrupted: ip = host_config.get(node, None).get('ip', None) user = host_config.get(node, None).get('user', None) password = host_config.get(node, None).get('password', None) infra.display_on_terminal(self, "IP: ", ip, " User: "******" Pwd: ", password) infra.display_on_terminal(self, "Stopping ", process_name) infra.display_on_terminal(self, "Executing ", rhel_stop_command) code, out, error = infra.ssh_and_execute_command( ip, user, password, rhel_stop_command) infra.add_table_rows(self, table_name, [[ ip, process_name, utils.get_timestamp(), HAConstants.WARNING + 'Stopped' + HAConstants.ENDC ]]) infra.display_on_terminal(self, "Sleeping for interval ", str(ha_interval), " seconds") time.sleep(ha_interval) infra.display_on_terminal(self, "Starting ", process_name) infra.display_on_terminal(self, "Executing ", rhel_start_command) code, out, error = infra.ssh_and_execute_command( ip, user, password, rhel_start_command) time.sleep(ha_interval) infra.add_table_rows(self, table_name, [[ ip, process_name, utils.get_timestamp(), HAConstants.OKGREEN + 'Started' + HAConstants.ENDC ]]) # bring it back to stable state infra.display_on_terminal(self, "Bringing the process to stable state") infra.display_on_terminal(self, "Executing ", rhel_start_command) code, out, error = infra.ssh_and_execute_command( ip, user, password, rhel_start_command) infra.display_on_terminal(self, "Finishing Process Disruption")
def container_disruption(self, sync=None, finish_execution=None): self.sync = sync self.finish_execution = finish_execution infra.display_on_terminal(self, "Entering Container Disruption plugin") table_name = "Container Disruption" infra.create_report_table(self, table_name) infra.add_table_headers(self, table_name, ["Host", "Container Process", "Timestamp", "Status of Disruption"]) input_args_dict = self.get_input_arguments() node_name = input_args_dict.keys()[0] input_args = input_args_dict.get(node_name, None) host_config = infra.get_openstack_config() if input_args: print "Inpt " + str(input_args) container_name = input_args.get('container_name', None) role = input_args.get('role', None) disruption_type = input_args.get('disruption', None) infra.display_on_terminal(self, "Container ", container_name, " will be disrupted") nodes_to_be_disrupted = [] for node in host_config: if 'controller' in host_config[node].get('role', None): infra.display_on_terminal(self, node, " will be disrupted ") nodes_to_be_disrupted.append(node) # For now disrupt on only one node break # Deprecate process disruptor and converge on this for both cases later container_stop_command = "systemctl stop " + container_name container_start_command = "systemctl start " + container_name ha_start_delay = self.get_ha_start_delay() if sync: infra.display_on_terminal(self, "Waiting for notification") infra.wait_for_notification(sync) infra.display_on_terminal(self, "Received notification, Starting") # Start the actual disruption after 45 seconds time.sleep(ha_start_delay) ha_interval = self.get_ha_interval() disruption_count = self.get_disruption_count() if disruption_type == 'infinite': #Override the disruption count in executor.yaml disruption_count = 1 while infra.is_execution_completed(self.finish_execution) is False: if disruption_count: disruption_count = disruption_count - 1 for node in nodes_to_be_disrupted: ip = host_config.get(node, None).get('ip', None) user = host_config.get(node, None).get('user', None) password = host_config.get(node, None).get('password', None) infra.display_on_terminal(self, "Stopping ", container_name) infra.display_on_terminal(self, "Executing ", container_stop_command) code, out, error = infra.ssh_and_execute_command(ip, user, password, container_stop_command) infra.add_table_rows(self, table_name, [[ip, container_name, utils.get_timestamp(), HAConstants.WARNING + 'Stopped' + HAConstants.ENDC]]) if disruption_type == 'infinite': infra.display_on_terminal(self, "Infinite disruption chosen bring up container manually") break infra.display_on_terminal(self, "Sleeping for interval ", str(ha_interval), " seconds") time.sleep(ha_interval) infra.display_on_terminal(self, "Starting ", container_name) infra.display_on_terminal(self, "Executing ", container_start_command) code, out, error = infra.ssh_and_execute_command(ip, user, password, container_start_command) time.sleep(ha_interval) infra.add_table_rows(self, table_name, [[ip, container_name, utils.get_timestamp(), HAConstants.OKGREEN + 'Started' + HAConstants.ENDC]]) # bring it back to stable state if disruption_type != 'infinite': infra.display_on_terminal(self, "Bringing the container to stable state") infra.display_on_terminal(self, "Executing ", container_start_command) code, out, error = infra.ssh_and_execute_command(ip, user, password, container_start_command) infra.display_on_terminal(self, "Finishing Container Disruption")
def process_disruption(self, sync=None, finish_execution=None): self.sync = sync self.finish_execution = finish_execution table_name = "Process Disruption" infra.create_report_table(self, table_name) infra.add_table_headers(self, table_name, ["Host", "Process", "TimeStamp", "Status of Disruption"]) infra.display_on_terminal(self, "Entering Process Disruption plugin") input_args_dict = self.get_input_arguments() node_name = input_args_dict.keys()[0] input_args = input_args_dict.get(node_name, None) host_config = infra.get_openstack_config() if input_args: print "Inpt " + str(input_args) process_name = input_args.get('process_name', None) role = input_args.get('role', None) type = input_args.get('type', None) infra.display_on_terminal(self, "Process ", process_name, " will be disrupted") nodes_to_be_disrupted = [] for node in host_config: if 'controller' in host_config[node].get('role', None): infra.display_on_terminal(self, node, " will be disrupted ") nodes_to_be_disrupted.append(node) self.expected_failures.append(node + "::" + process_name) self.set_expected_failures(self.expected_failures) rhel_stop_command = "systemctl stop " + process_name rhel_start_command = "systemctl start " + process_name if sync: infra.display_on_terminal(self, "Waiting for notification") infra.wait_for_notification(sync) infra.display_on_terminal(self, "Received notification, Starting") ha_interval = self.get_ha_interval() disruption_count = self.get_disruption_count() infra.display_on_terminal(self, "Process will be disrupted " , str(disruption_count)) while infra.is_execution_completed(self.finish_execution) is False: if disruption_count: disruption_count = disruption_count - 1 for node in nodes_to_be_disrupted: ip = host_config.get(node, None).get('ip', None) user = host_config.get(node, None).get('user', None) password = host_config.get(node, None).get('password', None) infra.display_on_terminal(self, "IP: ", ip, " User: "******" Pwd: ", password) infra.display_on_terminal(self, "Stopping ", process_name) infra.display_on_terminal(self, "Executing ", rhel_stop_command) code, out, error = infra.ssh_and_execute_command(ip, user, password, rhel_stop_command) infra.add_table_rows(self, table_name, [[ip, process_name, utils.get_timestamp(), HAConstants.WARNING + 'Stopped' + HAConstants.ENDC]]) infra.display_on_terminal(self, "Sleeping for interval ", str(ha_interval), " seconds") time.sleep(ha_interval) infra.display_on_terminal(self, "Starting ", process_name) infra.display_on_terminal(self, "Executing ", rhel_start_command) code, out, error = infra.ssh_and_execute_command(ip, user, password, rhel_start_command) time.sleep(ha_interval) infra.add_table_rows(self, table_name, [[ip, process_name, utils.get_timestamp(), HAConstants.OKGREEN + 'Started' + HAConstants.ENDC]]) # bring it back to stable state infra.display_on_terminal(self, "Bringing the process to stable state") infra.display_on_terminal(self, "Executing ", rhel_start_command) code, out, error = infra.ssh_and_execute_command(ip, user, password, rhel_start_command) infra.display_on_terminal(self, "Finishing Process Disruption")