def restart_machine(self):
"""
Restart the nodes to fail in the tests
:return: Nothing
"""
node_down_timer_tasks = []
for node in self.server_to_fail:
node_failure_timer_task = NodeDownTimerTask(node.ip)
node_down_timer_tasks.append(node_failure_timer_task)
task = AutoFailoverNodesFailureTask(self.orchestrator,
self.server_to_fail,
"restart_machine", self.timeout,
self.pause_between_failover_action,
self.failover_expected,
self.timeout_buffer,
failure_timers=node_down_timer_tasks)
for node_down_timer_task in node_down_timer_tasks:
self.node_failure_task_manager.schedule(node_down_timer_task, 2)
self.task_manager.schedule(task)
try:
task.result()
except Exception, e:
self.fail("Exception: {}".format(e))
def stop_memcached(self):
"""
Stop the memcached on the nodes to fail in the tests
:return: Nothing
"""
node_down_timer_tasks = []
for node in self.server_to_fail:
node_failure_timer_task = NodeDownTimerTask(node.ip, 11211)
node_down_timer_tasks.append(node_failure_timer_task)
self.timeout_buffer += 3
task = AutoFailoverNodesFailureTask(self.orchestrator,
self.server_to_fail,
"stop_memcached", self.timeout,
self.pause_between_failover_action,
self.failover_expected,
self.timeout_buffer,
failure_timers=node_down_timer_tasks)
for node_down_timer_task in node_down_timer_tasks:
self.node_failure_task_manager.schedule(node_down_timer_task, 2)
self.task_manager.schedule(task)
try:
task.result()
except Exception as e:
self.fail("Exception: {}".format(e))
finally:
task = AutoFailoverNodesFailureTask(self.orchestrator,
self.server_to_fail,
"start_memcached",
self.timeout, 0, False, 0,
check_for_failover=False)
self.task_manager.schedule(task)
task.result()
def async_restart_machine(self):
node_down_timer_tasks = []
for node in self.server_to_fail:
node_failure_timer_task = NodeDownTimerTask(node.ip)
node_down_timer_tasks.append(node_failure_timer_task)
task = AutoFailoverNodesFailureTask(
self.orchestrator,
self.server_to_fail,
"restart_machine",
self.timeout,
self.pause_between_failover_action,
self.failover_expected,
self.timeout_buffer,
failure_timers=node_down_timer_tasks)
for node_down_timer_task in node_down_timer_tasks:
self.node_failure_task_manager.schedule(node_down_timer_task, 2)
return task
def async_block_indexer_port(self):
node_down_timer_tasks = []
for node in self.server_to_fail:
node_failure_timer_task = NodeDownTimerTask(node.ip, 11211)
node_down_timer_tasks.append(node_failure_timer_task)
self.timeout_buffer += 3
task = AutoFailoverNodesFailureTask(
self.orchestrator,
self.server_to_fail,
"block_indexer_port",
self.timeout,
self.pause_between_failover_action,
self.failover_expected,
self.timeout_buffer,
failure_timers=node_down_timer_tasks)
for node_down_timer_task in node_down_timer_tasks:
self.node_failure_task_manager.schedule(node_down_timer_task, 2)
return task
def fail_disk_via_disk_full(self):
node_down_timer_tasks = []
for node in self.server_to_fail:
node_failure_timer_task = NodeDownTimerTask(node.ip)
node_down_timer_tasks.append(node_failure_timer_task)
task = AutoFailoverNodesFailureTask(self.orchestrator,
self.server_to_fail,
"disk_full", self.timeout,
self.pause_between_failover_action,
self.failover_expected,
self.timeout_buffer,
failure_timers=node_down_timer_tasks,
disk_timeout=self.disk_timeout, disk_location=self.disk_location,
disk_size=self.disk_location_size)
for node_down_timer_task in node_down_timer_tasks:
self.node_failure_task_manager.schedule(node_down_timer_task, 2)
self.task_manager.schedule(task)
try:
task.result()
except Exception as e:
self.fail("Exception: {}".format(e))
def restart_machine(self):
"""
Restart the nodes to fail in the tests
:return: Nothing
"""
node_down_timer_tasks = []
for node in self.server_to_fail:
node_failure_timer_task = NodeDownTimerTask(node.ip)
node_down_timer_tasks.append(node_failure_timer_task)
task = AutoFailoverNodesFailureTask(
self.orchestrator,
self.server_to_fail,
"restart_machine",
self.timeout,
self.pause_between_failover_action,
self.failover_expected,
self.timeout_buffer,
failure_timers=node_down_timer_tasks)
for node_down_timer_task in node_down_timer_tasks:
self.node_failure_task_manager.schedule(node_down_timer_task, 2)
self.task_manager.schedule(task)
try:
task.result()
except Exception as e:
self.fail("Exception: {}".format(e))
finally:
self.sleep(120, "Sleeping for 2 min for the machines to restart")
for node in self.server_to_fail:
for i in range(0, 2):
try:
shell = RemoteMachineShellConnection(node)
break
except:
self.log.info("Unable to connect to the host. "
"Machine has not restarted")
self.sleep(60, "Sleep for another minute and try "
"again")