def restart_machine(self): """ Restart the nodes to fail in the tests :return: Nothing """ node_down_timer_tasks = [] for node in self.server_to_fail: node_failure_timer_task = NodeDownTimerTask(node.ip) node_down_timer_tasks.append(node_failure_timer_task) task = AutoFailoverNodesFailureTask(self.orchestrator, self.server_to_fail, "restart_machine", self.timeout, self.pause_between_failover_action, self.failover_expected, self.timeout_buffer, failure_timers=node_down_timer_tasks) for node_down_timer_task in node_down_timer_tasks: self.node_failure_task_manager.schedule(node_down_timer_task, 2) self.task_manager.schedule(task) try: task.result() except Exception, e: self.fail("Exception: {}".format(e))
def stop_memcached(self): """ Stop the memcached on the nodes to fail in the tests :return: Nothing """ node_down_timer_tasks = [] for node in self.server_to_fail: node_failure_timer_task = NodeDownTimerTask(node.ip, 11211) node_down_timer_tasks.append(node_failure_timer_task) self.timeout_buffer += 3 task = AutoFailoverNodesFailureTask(self.orchestrator, self.server_to_fail, "stop_memcached", self.timeout, self.pause_between_failover_action, self.failover_expected, self.timeout_buffer, failure_timers=node_down_timer_tasks) for node_down_timer_task in node_down_timer_tasks: self.node_failure_task_manager.schedule(node_down_timer_task, 2) self.task_manager.schedule(task) try: task.result() except Exception as e: self.fail("Exception: {}".format(e)) finally: task = AutoFailoverNodesFailureTask(self.orchestrator, self.server_to_fail, "start_memcached", self.timeout, 0, False, 0, check_for_failover=False) self.task_manager.schedule(task) task.result()
def async_restart_machine(self): node_down_timer_tasks = [] for node in self.server_to_fail: node_failure_timer_task = NodeDownTimerTask(node.ip) node_down_timer_tasks.append(node_failure_timer_task) task = AutoFailoverNodesFailureTask( self.orchestrator, self.server_to_fail, "restart_machine", self.timeout, self.pause_between_failover_action, self.failover_expected, self.timeout_buffer, failure_timers=node_down_timer_tasks) for node_down_timer_task in node_down_timer_tasks: self.node_failure_task_manager.schedule(node_down_timer_task, 2) return task
def async_block_indexer_port(self): node_down_timer_tasks = [] for node in self.server_to_fail: node_failure_timer_task = NodeDownTimerTask(node.ip, 11211) node_down_timer_tasks.append(node_failure_timer_task) self.timeout_buffer += 3 task = AutoFailoverNodesFailureTask( self.orchestrator, self.server_to_fail, "block_indexer_port", self.timeout, self.pause_between_failover_action, self.failover_expected, self.timeout_buffer, failure_timers=node_down_timer_tasks) for node_down_timer_task in node_down_timer_tasks: self.node_failure_task_manager.schedule(node_down_timer_task, 2) return task
def fail_disk_via_disk_full(self): node_down_timer_tasks = [] for node in self.server_to_fail: node_failure_timer_task = NodeDownTimerTask(node.ip) node_down_timer_tasks.append(node_failure_timer_task) task = AutoFailoverNodesFailureTask(self.orchestrator, self.server_to_fail, "disk_full", self.timeout, self.pause_between_failover_action, self.failover_expected, self.timeout_buffer, failure_timers=node_down_timer_tasks, disk_timeout=self.disk_timeout, disk_location=self.disk_location, disk_size=self.disk_location_size) for node_down_timer_task in node_down_timer_tasks: self.node_failure_task_manager.schedule(node_down_timer_task, 2) self.task_manager.schedule(task) try: task.result() except Exception as e: self.fail("Exception: {}".format(e))
def restart_machine(self): """ Restart the nodes to fail in the tests :return: Nothing """ node_down_timer_tasks = [] for node in self.server_to_fail: node_failure_timer_task = NodeDownTimerTask(node.ip) node_down_timer_tasks.append(node_failure_timer_task) task = AutoFailoverNodesFailureTask( self.orchestrator, self.server_to_fail, "restart_machine", self.timeout, self.pause_between_failover_action, self.failover_expected, self.timeout_buffer, failure_timers=node_down_timer_tasks) for node_down_timer_task in node_down_timer_tasks: self.node_failure_task_manager.schedule(node_down_timer_task, 2) self.task_manager.schedule(task) try: task.result() except Exception as e: self.fail("Exception: {}".format(e)) finally: self.sleep(120, "Sleeping for 2 min for the machines to restart") for node in self.server_to_fail: for i in range(0, 2): try: shell = RemoteMachineShellConnection(node) break except: self.log.info("Unable to connect to the host. " "Machine has not restarted") self.sleep(60, "Sleep for another minute and try " "again")