def __setup_app(self, ud={}):
self.app = TestApp(ud=ud)
self.inst = MockBotoInstance()
self.instance = Instance(self.app, inst=self.inst)
self.app.manager.worker_instances = [self.instance]
class MasterInstanceTestCase(TestCase):
def setUp(self):
self.__setup_app()
def __setup_app(self, ud={}):
self.app = TestApp(ud=ud)
self.inst = MockBotoInstance()
self.instance = Instance(self.app, inst=self.inst)
self.app.manager.worker_instances = [self.instance]
def test_id(self):
assert self.instance.id == DEFAULT_MOCK_BOTO_INSTANCE_ID
def test_get_cloud_instance_object(self):
instance = self.instance
# Without deep=True, just returned cached boto inst object.
assert instance.get_cloud_instance_object() is self.inst
# With deep=True should fetch new instance.
fresh_instance = self.__seed_fresh_instance()
assert instance.get_cloud_instance_object(deep=True) is fresh_instance
def test_get_m_state(self):
assert self.instance.m_state == None
self.__seed_fresh_instance(state=instance_states.RUNNING)
assert self.instance.get_m_state() == instance_states.RUNNING
assert self.instance.m_state == instance_states.RUNNING
def test_reboot(self):
"""
Check reboot was called on boto instance, time_rebooted is
updated, and reboot_count is incremeneted.
"""
assert self.instance.time_rebooted is TIME_IN_PAST
assert self.instance.reboot_count == 0
self.instance.reboot()
assert self.inst.was_rebooted
assert self.instance.time_rebooted is not TIME_IN_PAST
assert self.instance.reboot_count == 1
# Check successive calls continue to increment reboot_count.
self.instance.reboot()
assert self.instance.reboot_count == 2
def test_terminate_success(self):
self.__expect_terminatation(success=True)
assert self.instance.terminate_attempt_count == 0
assert self.instance.inst is not None
assert self.instance in self.app.manager.worker_instances
thread = self.instance.terminate()
thread.join()
assert self.instance.inst is None
assert self.instance.terminate_attempt_count == 1
assert not self.instance in self.app.manager.worker_instances
def test_terminate_failure(self):
self.__expect_terminatation(success=False)
assert self.instance.terminate_attempt_count == 0
self.__seed_fresh_instance() # Needed for log statement in failure
thread = self.instance.terminate()
thread.join()
# inst is only set to None after success
assert self.instance.inst is not None
assert self.instance.terminate_attempt_count == 1
assert self.instance in self.app.manager.worker_instances
def test_maintain_reboot_stuck(self):
""" Test method verifies instance is rebooted after stuck
in PENDING state for 1000 seconds."""
with instrument_time() as time:
self.__assert_maintain_does_not_reboot(with_state=instance_states.PENDING)
# Not rebooted after 100 seconds
time.set_offset(seconds=100)
self.__assert_maintain_does_not_reboot(with_state=instance_states.PENDING)
# Does reboot after 600 seconds
time.set_offset(seconds=600)
self.__assert_maintain_reboots(with_state=instance_states.PENDING)
def test_maintain_retry_reboot(self):
with instrument_time() as time:
self.__assert_maintain_does_not_reboot(with_state=instance_states.PENDING)
# Maintain at 500 seconds determines it is stuck, attempts reboot
time.set_offset(seconds=500)
self.__assert_maintain_reboots(with_state=instance_states.PENDING)
# Maintain at 600 is still stuck, but waiting for reboot.
time.set_offset(seconds=700)
self.__assert_maintain_does_not_reboot(with_state=instance_states.PENDING)
# Maintain at 900 seconds, still stuck retries reboot
time.set_offset(seconds=900)
self.__assert_maintain_reboots(with_state=instance_states.PENDING)
def test_maintain_extend_reboot_timeout(self):
self.__setup_app(ud={"instance_reboot_timeout": 500})
with instrument_time() as time:
self.__assert_maintain_does_not_reboot(with_state=instance_states.PENDING)
# Maintain at 500 seconds determines it is stuck, attempts reboot
time.set_offset(seconds=500)
self.__assert_maintain_reboots(with_state=instance_states.PENDING)
# Maintain at 600 is still stuck, but waiting for reboot.
time.set_offset(seconds=700)
self.__assert_maintain_does_not_reboot(with_state=instance_states.PENDING)
# Maintain at 900 seconds, would normally reboot but timeout is
# extended so it won't.
time.set_offset(seconds=900)
self.__assert_maintain_does_not_reboot(with_state=instance_states.PENDING)
# Will eventually reboot again though...
time.set_offset(seconds=1200)
self.__assert_maintain_reboots(with_state=instance_states.PENDING)
def test_maintain_state_change(self):
with instrument_time() as time:
self.__assert_maintain_does_not_reboot(with_state=instance_states.PENDING)
# 350 is past reboot timeout (300), but wait for 400 seconds for state change
# so no reboot.
time.set_offset(seconds=350)
self.__assert_maintain_does_not_reboot(with_state=instance_states.PENDING)
def test_maintain_extend_state_change_wait(self):
self.__setup_app(ud={"instance_state_change_wait": 700})
with instrument_time() as time:
self.__assert_maintain_does_not_reboot(with_state=instance_states.PENDING)
# Does not reboot after 600 seconds, waiting for state change.
time.set_offset(seconds=600)
self.__assert_maintain_does_not_reboot(with_state=instance_states.PENDING)
# Does eventually reboot though
time.set_offset(seconds=800)
self.__assert_maintain_reboots(with_state=instance_states.PENDING)
def test_maintain_reboots_on_error(self):
self.__assert_maintain_reboots(with_state=instance_states.ERROR)
def test_maintain_reboots_after_comm_loss(self):
with instrument_time() as time:
self.instance.handle_message("TEST")
self.__assert_maintain_does_not_reboot(with_state=instance_states.RUNNING)
time.set_offset(seconds=500)
self.__assert_maintain_reboots(with_state=instance_states.RUNNING)
def test_extend_comm_timeout(self):
# Same test as above, but extend the comm timeout to verify it prevents
# instance from being rebooted.
self.__setup_app(ud={"instance_comm_timeout": 700})
with instrument_time() as time:
self.instance.handle_message("TEST")
self.__assert_maintain_does_not_reboot(with_state=instance_states.RUNNING)
time.set_offset(seconds=500)
self.__assert_maintain_does_not_reboot(with_state=instance_states.RUNNING)
def test_maintain_no_reboot_if_comm_active(self):
with instrument_time() as time:
self.instance.handle_message("TEST")
self.__assert_maintain_does_not_reboot(with_state=instance_states.RUNNING)
time.set_offset(seconds=350)
self.instance.handle_message("TEST")
time.set_offset(seconds=500)
self.__assert_maintain_does_not_reboot(with_state=instance_states.RUNNING)
def test_terminates_after_enough_reboots(self):
for _ in range(4):
self.__assert_maintain_reboots(with_state=instance_states.ERROR)
self.__expect_terminatation(success=False)
self.__assert_maintain_does_not_reboot(with_state=instance_states.ERROR)
def test_change_reboot_attempts(self):
self.__setup_app(ud={"instance_reboot_attempts": 2})
for _ in range(2):
self.__assert_maintain_reboots(with_state=instance_states.ERROR)
self.__expect_terminatation(success=False)
self.__assert_maintain_does_not_reboot(with_state=instance_states.ERROR)
# Test is flaky, need to join thread somehow.
def test_terminate_attempts(self):
for _ in range(4):
self.__assert_maintain_reboots(with_state=instance_states.ERROR)
for _ in range(4):
self.__expect_terminatation(success=False)
self.__assert_maintain_does_not_reboot(with_state=instance_states.ERROR)
assert self.instance in self.app.manager.worker_instances
# Ultimately gives on terminates and just reboots.
self.__assert_maintain_does_not_reboot(with_state=instance_states.ERROR)
assert self.instance not in self.app.manager.worker_instances
def test_modify_terminate_attempts(self):
self.__setup_app(ud={"instance_terminate_attempts": 2})
for _ in range(4):
self.__assert_maintain_reboots(with_state=instance_states.ERROR)
for _ in range(2):
self.__expect_terminatation(success=False)
self.__assert_maintain_does_not_reboot(with_state=instance_states.ERROR)
assert self.instance in self.app.manager.worker_instances
# Ultimately gives on terminates and just reboots.
self.__assert_maintain_does_not_reboot(with_state=instance_states.ERROR)
assert self.instance not in self.app.manager.worker_instances
def __assert_maintain_reboots(self, with_state):
inst = self.__maintain_with_instance(state=with_state)
assert inst.was_rebooted
def __assert_maintain_does_not_reboot(self, with_state):
inst = self.__maintain_with_instance(state=with_state)
assert not inst.was_rebooted
def __maintain_with_instance(self, **instance_kwds):
inst = self.__seed_fresh_instance(**instance_kwds)
self.instance.maintain()
return inst
def __expect_terminatation(self, success):
self.app.cloud_interface.expect_terminatation( \
DEFAULT_MOCK_BOTO_INSTANCE_ID, spot_request_id=None, success=success)
def __seed_fresh_instance(self, state=None):
fresh_instance = MockBotoInstance()
self.app.cloud_interface.set_mock_instances([fresh_instance])
if state:
fresh_instance.state = state
return fresh_instance