def setUp(self):
self._role = 'mesos'
self._env = 'test'
self._name = 'jimbob'
self._clock = FakeClock()
self._scheduler = mox.MockObject(scheduler_client)
job_key = JobKey(name=self._name, environment=self._env, role=self._role)
self._health_check = mox.MockObject(HealthCheck)
self._watcher = InstanceWatcher(self._scheduler,
job_key,
self.RESTART_THRESHOLD,
self.WATCH_SECS,
health_check_interval_seconds=3,
clock=self._clock)
class InstanceWatcherTest(unittest.TestCase):
WATCH_INSTANCES = range(3)
RESTART_THRESHOLD = WATCH_SECS = 50
EXPECTED_CYCLES = find_expected_cycles(WATCH_SECS, 3.0)
def setUp(self):
self._role = 'mesos'
self._env = 'test'
self._name = 'jimbob'
self._clock = FakeClock()
self._scheduler = mox.MockObject(scheduler_client)
job_key = JobKey(name=self._name, environment=self._env, role=self._role)
self._health_check = mox.MockObject(HealthCheck)
self._watcher = InstanceWatcher(self._scheduler,
job_key,
self.RESTART_THRESHOLD,
self.WATCH_SECS,
health_check_interval_seconds=3,
clock=self._clock)
def get_tasks_status_query(self, instance_ids):
query = TaskQuery()
query.owner = Identity(role=self._role)
query.environment = self._env
query.jobName = self._name
query.statuses = set([ScheduleStatus.RUNNING])
query.instanceIds = set(instance_ids)
return query
def create_task(self, instance_id):
return ScheduledTask(
assignedTask=AssignedTask(instanceId=instance_id, task=TaskConfig()))
def expect_get_statuses(self, instance_ids=WATCH_INSTANCES, num_calls=EXPECTED_CYCLES):
tasks = [self.create_task(instance_id) for instance_id in instance_ids]
response = Response(responseCode=ResponseCode.OK, message='test')
response.result = Result()
response.result.scheduleStatusResult = ScheduleStatusResult(tasks=tasks)
query = self.get_tasks_status_query(instance_ids)
for x in range(int(num_calls)):
self._scheduler.getTasksStatus(query).AndReturn(response)
def expect_io_error_in_get_statuses(self, instance_ids=WATCH_INSTANCES, num_calls=EXPECTED_CYCLES):
tasks = [self.create_task(instance_id) for instance_id in instance_ids]
response = Response(responseCode=ResponseCode.OK, message='test')
response.result = Result()
response.result.scheduleStatusResult = ScheduleStatusResult(tasks=tasks)
query = self.get_tasks_status_query(instance_ids)
for x in range(int(num_calls)):
self._scheduler.getTasksStatus(query).AndRaise(IOError('oops'))
def mock_health_check(self, task, status, retry):
self._health_check.health(task).InAnyOrder().AndReturn((status, retry))
def expect_health_check(self, instance, status, retry=True, num_calls=EXPECTED_CYCLES):
for x in range(int(num_calls)):
self.mock_health_check(self.create_task(instance), status, retry)
def assert_watch_result(self, expected_failed_instances, instances_to_watch=WATCH_INSTANCES):
instances_returned = self._watcher.watch(instances_to_watch, self._health_check)
assert set(expected_failed_instances) == instances_returned, (
'Expected instances (%s) : Returned instances (%s)' % (expected_failed_instances, instances_returned))
def replay_mocks(self):
mox.Replay(self._scheduler)
mox.Replay(self._health_check)
def verify_mocks(self):
mox.Verify(self._scheduler)
mox.Verify(self._health_check)
def test_successful_watch(self):
"""All instances are healthy immediately"""
self.expect_get_statuses()
self.expect_health_check(0, True)
self.expect_health_check(1, True)
self.expect_health_check(2, True)
self.replay_mocks()
self.assert_watch_result([])
self.verify_mocks()
def test_single_instance_failure(self):
"""One failed instance in a batch of instances"""
self.expect_get_statuses()
self.expect_health_check(0, False)
self.expect_health_check(1, True)
self.expect_health_check(2, True)
self.replay_mocks()
self.assert_watch_result([0])
self.verify_mocks()
def test_io_failure(self):
"""Check that IO errors (socket errors) communicating with the scheduler get handled
correctly"""
self.expect_io_error_in_get_statuses()
self.replay_mocks()
self.assert_watch_result([0, 1, 2])
self.verify_mocks()
def test_all_instance_failure(self):
"""All failed instance in a batch of instances"""
self.expect_get_statuses()
self.expect_health_check(0, False)
self.expect_health_check(1, False)
self.expect_health_check(2, False)
self.replay_mocks()
self.assert_watch_result([0, 1, 2])
self.verify_mocks()
def test_restart_threshold_fail_fast(self):
"""Instances are reported unhealthy with retry set to False"""
self.expect_get_statuses(num_calls=1)
self.expect_health_check(0, False, retry=False, num_calls=1)
self.expect_health_check(1, False, retry=False, num_calls=1)
self.expect_health_check(2, False, retry=False, num_calls=1)
self.replay_mocks()
self.assert_watch_result([0, 1, 2])
self.verify_mocks()
def test_restart_threshold(self):
"""Instances are reported healthy at the end of the restart_threshold"""
self.expect_get_statuses(num_calls=self.EXPECTED_CYCLES - 1)
self.expect_health_check(0, False, num_calls=self.EXPECTED_CYCLES - 1)
self.expect_health_check(1, False, num_calls=self.EXPECTED_CYCLES - 1)
self.expect_health_check(2, False, num_calls=self.EXPECTED_CYCLES - 1)
self.expect_get_statuses()
self.expect_health_check(0, True)
self.expect_health_check(1, True)
self.expect_health_check(2, True)
self.replay_mocks()
self.assert_watch_result([])
self.verify_mocks()
def test_watch_period_failure(self):
"""Instances are reported unhealthy before watch_secs expires"""
self.expect_get_statuses()
self.expect_health_check(0, True, num_calls=self.EXPECTED_CYCLES - 1)
self.expect_health_check(1, True, num_calls=self.EXPECTED_CYCLES - 1)
self.expect_health_check(2, True, num_calls=self.EXPECTED_CYCLES - 1)
self.expect_health_check(0, False, num_calls=1)
self.expect_health_check(1, False, num_calls=1)
self.expect_health_check(2, False, num_calls=1)
self.replay_mocks()
self.assert_watch_result([0, 1, 2])
self.verify_mocks()
def test_single_watch_period_failure(self):
"""One instance is reported unhealthy before watch_secs expires"""
self.expect_get_statuses()
self.expect_health_check(0, True)
self.expect_health_check(1, True)
self.expect_health_check(2, True, num_calls=self.EXPECTED_CYCLES - 1)
self.expect_health_check(2, False, num_calls=1)
self.replay_mocks()
self.assert_watch_result([2])
self.verify_mocks()
