def test_maintain_reboots_after_comm_loss(self):
with instrument_time() as time:
self.instance.handle_message("TEST")
self.__assert_maintain_does_not_reboot(with_state=instance_states.RUNNING)
time.set_offset(seconds=500)
self.__assert_maintain_reboots(with_state=instance_states.RUNNING)
def test_maintain_reboots_after_comm_loss(self):
with instrument_time() as time:
self.instance.handle_message("TEST")
self.__assert_maintain_does_not_reboot(
with_state=instance_states.RUNNING)
time.set_offset(seconds=500)
self.__assert_maintain_reboots(with_state=instance_states.RUNNING)
def test_maintain_state_change(self):
with instrument_time() as time:
self.__assert_maintain_does_not_reboot(with_state=instance_states.PENDING)
# 350 is past reboot timeout (300), but wait for 400 seconds for state change
# so no reboot.
time.set_offset(seconds=350)
self.__assert_maintain_does_not_reboot(with_state=instance_states.PENDING)
def test_extend_comm_timeout(self):
# Same test as above, but extend the comm timeout to verify it prevents
# instance from being rebooted.
self.__setup_app(ud={"instance_comm_timeout": 700})
with instrument_time() as time:
self.instance.handle_message("TEST")
self.__assert_maintain_does_not_reboot(with_state=instance_states.RUNNING)
time.set_offset(seconds=500)
self.__assert_maintain_does_not_reboot(with_state=instance_states.RUNNING)
def test_maintain_state_change(self):
with instrument_time() as time:
self.__assert_maintain_does_not_reboot(
with_state=instance_states.PENDING)
# 350 is past reboot timeout (300), but wait for 400 seconds for state change
# so no reboot.
time.set_offset(seconds=350)
self.__assert_maintain_does_not_reboot(
with_state=instance_states.PENDING)
def test_maintain_extend_state_change_wait(self):
self.__setup_app(ud={"instance_state_change_wait": 700})
with instrument_time() as time:
self.__assert_maintain_does_not_reboot(with_state=instance_states.PENDING)
# Does not reboot after 600 seconds, waiting for state change.
time.set_offset(seconds=600)
self.__assert_maintain_does_not_reboot(with_state=instance_states.PENDING)
# Does eventually reboot though
time.set_offset(seconds=800)
self.__assert_maintain_reboots(with_state=instance_states.PENDING)
def test_extend_comm_timeout(self):
# Same test as above, but extend the comm timeout to verify it prevents
# instance from being rebooted.
self.__setup_app(ud={"instance_comm_timeout": 700})
with instrument_time() as time:
self.instance.handle_message("TEST")
self.__assert_maintain_does_not_reboot(
with_state=instance_states.RUNNING)
time.set_offset(seconds=500)
self.__assert_maintain_does_not_reboot(
with_state=instance_states.RUNNING)
def test_maintain_reboot_stuck(self):
""" Test method verifies instance is rebooted after stuck
in PENDING state for 1000 seconds."""
with instrument_time() as time:
self.__assert_maintain_does_not_reboot(with_state=instance_states.PENDING)
# Not rebooted after 100 seconds
time.set_offset(seconds=100)
self.__assert_maintain_does_not_reboot(with_state=instance_states.PENDING)
# Does reboot after 600 seconds
time.set_offset(seconds=600)
self.__assert_maintain_reboots(with_state=instance_states.PENDING)
def test_maintain_extend_state_change_wait(self):
self.__setup_app(ud={"instance_state_change_wait": 700})
with instrument_time() as time:
self.__assert_maintain_does_not_reboot(
with_state=instance_states.PENDING)
# Does not reboot after 600 seconds, waiting for state change.
time.set_offset(seconds=600)
self.__assert_maintain_does_not_reboot(
with_state=instance_states.PENDING)
# Does eventually reboot though
time.set_offset(seconds=800)
self.__assert_maintain_reboots(with_state=instance_states.PENDING)
def test_maintain_retry_reboot(self):
with instrument_time() as time:
self.__assert_maintain_does_not_reboot(with_state=instance_states.PENDING)
# Maintain at 500 seconds determines it is stuck, attempts reboot
time.set_offset(seconds=500)
self.__assert_maintain_reboots(with_state=instance_states.PENDING)
# Maintain at 600 is still stuck, but waiting for reboot.
time.set_offset(seconds=700)
self.__assert_maintain_does_not_reboot(with_state=instance_states.PENDING)
# Maintain at 900 seconds, still stuck retries reboot
time.set_offset(seconds=900)
self.__assert_maintain_reboots(with_state=instance_states.PENDING)
def test_maintain_reboot_stuck(self):
""" Test method verifies instance is rebooted after stuck
in PENDING state for 1000 seconds."""
with instrument_time() as time:
self.__assert_maintain_does_not_reboot(
with_state=instance_states.PENDING)
# Not rebooted after 100 seconds
time.set_offset(seconds=100)
self.__assert_maintain_does_not_reboot(
with_state=instance_states.PENDING)
# Does reboot after 600 seconds
time.set_offset(seconds=600)
self.__assert_maintain_reboots(with_state=instance_states.PENDING)
def test_maintain_retry_reboot(self):
with instrument_time() as time:
self.__assert_maintain_does_not_reboot(
with_state=instance_states.PENDING)
# Maintain at 500 seconds determines it is stuck, attempts reboot
time.set_offset(seconds=500)
self.__assert_maintain_reboots(with_state=instance_states.PENDING)
# Maintain at 600 is still stuck, but waiting for reboot.
time.set_offset(seconds=700)
self.__assert_maintain_does_not_reboot(
with_state=instance_states.PENDING)
# Maintain at 900 seconds, still stuck retries reboot
time.set_offset(seconds=900)
self.__assert_maintain_reboots(with_state=instance_states.PENDING)
def test_maintain_extend_reboot_timeout(self):
self.__setup_app(ud={"instance_reboot_timeout": 500})
with instrument_time() as time:
self.__assert_maintain_does_not_reboot(with_state=instance_states.PENDING)
# Maintain at 500 seconds determines it is stuck, attempts reboot
time.set_offset(seconds=500)
self.__assert_maintain_reboots(with_state=instance_states.PENDING)
# Maintain at 600 is still stuck, but waiting for reboot.
time.set_offset(seconds=700)
self.__assert_maintain_does_not_reboot(with_state=instance_states.PENDING)
# Maintain at 900 seconds, would normally reboot but timeout is
# extended so it won't.
time.set_offset(seconds=900)
self.__assert_maintain_does_not_reboot(with_state=instance_states.PENDING)
# Will eventually reboot again though...
time.set_offset(seconds=1200)
self.__assert_maintain_reboots(with_state=instance_states.PENDING)
def test_maintain_extend_reboot_timeout(self):
self.__setup_app(ud={"instance_reboot_timeout": 500})
with instrument_time() as time:
self.__assert_maintain_does_not_reboot(
with_state=instance_states.PENDING)
# Maintain at 500 seconds determines it is stuck, attempts reboot
time.set_offset(seconds=500)
self.__assert_maintain_reboots(with_state=instance_states.PENDING)
# Maintain at 600 is still stuck, but waiting for reboot.
time.set_offset(seconds=700)
self.__assert_maintain_does_not_reboot(
with_state=instance_states.PENDING)
# Maintain at 900 seconds, would normally reboot but timeout is
# extended so it won't.
time.set_offset(seconds=900)
self.__assert_maintain_does_not_reboot(
with_state=instance_states.PENDING)
# Will eventually reboot again though...
time.set_offset(seconds=1200)
self.__assert_maintain_reboots(with_state=instance_states.PENDING)