def create(self, action=None):
self.log.info("Simulation disk scenario '{0}'".format(action))
if action == DiskError.FAILOVER_DISK:
task = AutoFailoverNodesFailureTask(
self.task_manager,
self.orchestrator,
self.server_to_fail,
"disk_failure",
self.timeout,
self.pause_between_failover_action,
self.failover_expected,
self.timeout_buffer,
disk_timeout=self.disk_timeout,
disk_location=self.disk_location,
disk_size=self.disk_location_size)
self.task_manager.add_new_task(task)
self.task_manager.get_task_result(task)
elif action == DiskError.DISK_FULL:
task = AutoFailoverNodesFailureTask(
self.task_manager,
self.orchestrator,
self.server_to_fail,
"disk_full",
self.timeout,
self.pause_between_failover_action,
self.failover_expected,
self.timeout_buffer,
disk_timeout=self.disk_timeout,
disk_location=self.disk_location,
disk_size=self.disk_location_size)
self.task_manager.add_new_task(task)
self.task_manager.get_task_result(task)
else:
self.log.warning("Unsupported disk action '{0}'".format(action))
def stop_memcached(self):
"""
Stop the memcached on the nodes to fail in the tests
:return: Nothing
"""
node_down_timer_tasks = []
for node in self.server_to_fail:
node_failure_timer_task = NodeDownTimerTask(node.ip, 11211)
node_down_timer_tasks.append(node_failure_timer_task)
self.timeout_buffer += 3
task = AutoFailoverNodesFailureTask(
self.task_manager,
self.orchestrator,
self.server_to_fail,
"stop_memcached",
self.timeout,
self.pause_between_failover_action,
self.failover_expected,
self.timeout_buffer,
failure_timers=node_down_timer_tasks)
self.task_manager.add_new_task(task)
try:
self.task_manager.get_task_result(task)
except Exception, e:
self.fail("Exception: {}".format(e))
def restart_couchbase_server(self):
"""
Restart couchbase server on the nodes to fail in the tests
:return: Nothing
"""
node_down_timer_tasks = []
for node in self.server_to_fail:
node_failure_timer_task = NodeDownTimerTask(node.ip, node.port)
node_down_timer_tasks.append(node_failure_timer_task)
task = AutoFailoverNodesFailureTask(
self.task_manager,
self.orchestrator,
self.server_to_fail,
"restart_couchbase",
self.timeout,
self.pause_between_failover_action,
self.failover_expected,
self.timeout_buffer,
failure_timers=node_down_timer_tasks)
self.task_manager.add_new_task(task)
self.sleep(30, "Waiting for couchbase-server to come up")
try:
self.task_manager.get_task_result(task)
except Exception, e:
self.fail("Exception: {}".format(e))
def revert(self, action=None):
self.log.info("Reverting disk scenario '{0}'".format(action))
if action == DiskError.FAILOVER_DISK:
action = "recover_disk_failure"
elif action == DiskError.DISK_FULL:
action = "recover_disk_full_failure"
else:
self.log.warning("Unsupported disk action '{0}'".format(action))
return
task = AutoFailoverNodesFailureTask(self.task_manager,
self.orchestrator,
self.server_to_fail,
action,
self.timeout,
self.pause_between_failover_action,
expect_auto_failover=False,
timeout_buffer=self.timeout_buffer,
check_for_failover=False,
disk_timeout=self.disk_timeout,
disk_location=self.disk_location,
disk_size=self.disk_location_size)
self.task_manager.add_new_task(task)
self.task_manager.get_task_result(task)
def start_couchbase_server(self):
"""
Start the couchbase server on the nodes to fail in the tests
:return: Nothing
"""
task = AutoFailoverNodesFailureTask(self.task_manager,
self.orchestrator,
self.server_to_fail,
"start_couchbase", self.timeout, 0,
False, self.timeout_buffer, False)
self.task_manager.add_new_task(task)
try:
self.task_manager.get_task_result(task)
except Exception, e:
self.fail("Exception: {}".format(e))
def disable_firewall(self):
"""
Disable firewall on the nodes to fail in the tests
:return: Nothing
"""
self.time_start = time.time()
task = AutoFailoverNodesFailureTask(self.task_manager,
self.orchestrator,
self.server_to_fail,
"disable_firewall", self.timeout,
self.pause_between_failover_action,
False, self.timeout_buffer, False)
self.task_manager.add_new_task(task)
try:
self.task_manager.get_task_result(task)
except Exception, e:
self.fail("Exception: {}".format(e))
def bring_back_failed_nodes_up(self):
if self.failover_action == "disk_failure":
task = AutoFailoverNodesFailureTask(
self.task_manager,
self.orchestrator,
self.server_to_fail,
"recover_disk_failure",
self.timeout,
self.pause_between_failover_action,
expect_auto_failover=False,
timeout_buffer=self.timeout_buffer,
check_for_failover=False,
disk_timeout=self.disk_timeout,
disk_location=self.disk_location,
disk_size=self.disk_location_size)
self.task_manager.add_new_task(task)
try:
self.task_manager.get_task_result(task)
except Exception, e:
self.fail("Exception: {}".format(e))
def split_network(self):
"""
Split the network in the cluster. Stop network traffic from few
nodes while allowing the traffic from rest of the cluster.
:return: Nothing
"""
self.time_start = time.time()
if self.server_to_fail.__len__() < 2:
self.fail("Need atleast 2 servers to fail")
task = AutoFailoverNodesFailureTask(self.task_manager,
self.orchestrator,
self.server_to_fail,
"network_split", self.timeout,
self.pause_between_failover_action,
False, self.timeout_buffer)
self.task_manager.add_new_task(task)
try:
self.task_manager.get_task_result(task)
except Exception, e:
self.fail("Exception: {}".format(e))
def fail_disk_via_disk_full(self):
node_down_timer_tasks = []
for node in self.server_to_fail:
node_failure_timer_task = NodeDownTimerTask(node.ip)
node_down_timer_tasks.append(node_failure_timer_task)
task = AutoFailoverNodesFailureTask(
self.task_manager,
self.orchestrator,
self.server_to_fail,
"disk_full",
self.timeout,
self.pause_between_failover_action,
self.failover_expected,
self.timeout_buffer,
failure_timers=node_down_timer_tasks,
disk_timeout=self.disk_timeout,
disk_location=self.disk_location,
disk_size=self.disk_location_size)
self.task_manager.add_new_task(task)
try:
self.task_manager.get_task_result(task)
except Exception, e:
self.fail("Exception: {}".format(e))
def enable_firewall(self):
"""
Enable firewall on the nodes to fail in the tests.
:return: Nothing
"""
node_down_timer_tasks = []
for node in self.server_to_fail:
node_failure_timer_task = NodeDownTimerTask(node.ip)
node_down_timer_tasks.append(node_failure_timer_task)
task = AutoFailoverNodesFailureTask(
self.task_manager,
self.orchestrator,
self.server_to_fail,
"enable_firewall",
self.timeout,
self.pause_between_failover_action,
self.failover_expected,
self.timeout_buffer,
failure_timers=node_down_timer_tasks)
self.task_manager.add_new_task(task)
try:
self.task_manager.get_task_result(task)
except Exception, e:
self.fail("Exception: {}".format(e))
self.timeout,
self.pause_between_failover_action,
self.failover_expected,
self.timeout_buffer,
failure_timers=node_down_timer_tasks)
self.task_manager.add_new_task(task)
try:
self.task_manager.get_task_result(task)
except Exception, e:
self.fail("Exception: {}".format(e))
finally:
task = AutoFailoverNodesFailureTask(self.task_manager,
self.orchestrator,
self.server_to_fail,
"start_memcached",
self.timeout,
0,
False,
0,
check_for_failover=False)
self.task_manager.add_new_task(task)
self.task_manager.get_task_result(task)
def split_network(self):
"""
Split the network in the cluster. Stop network traffic from few
nodes while allowing the traffic from rest of the cluster.
:return: Nothing
"""
self.time_start = time.time()
if self.server_to_fail.__len__() < 2:
