예제 #1
0
 def setUp(self):
   self._http_signaler = mox.MockObject(HttpSignaler)
   self._status_health_check = StatusHealthCheck()
   self._http_health_check = HttpHealthCheck(self._http_signaler)
   self._smart_health_check = InstanceWatcherHealthCheck(self._http_signaler)
   self._health_check_a = mox.MockObject(HealthCheck)
   self._health_check_b = mox.MockObject(HealthCheck)
예제 #2
0
class HealthCheckTest(unittest.TestCase):
  HOST = 'host-a'
  HEALTH_PORT = 33333

  def setUp(self):
    self._http_signaler = mox.MockObject(HttpSignaler)
    self._status_health_check = StatusHealthCheck()
    self._http_health_check = HttpHealthCheck(self._http_signaler)
    self._smart_health_check = InstanceWatcherHealthCheck(self._http_signaler)
    self._health_check_a = mox.MockObject(HealthCheck)
    self._health_check_b = mox.MockObject(HealthCheck)

  def create_task(self, instance_id, task_id, status=RUNNING, host=HOST, port=HEALTH_PORT):
    ports = {}
    if port:
      ports['health'] = port
    assigned_task = AssignedTask(taskId=task_id,
                                 instanceId=instance_id,
                                 task=TaskConfig(),
                                 slaveHost=host,
                                 assignedPorts=ports)
    return ScheduledTask(assignedTask=assigned_task, status=status)

  def expect_http_signaler_creation(self, host=HOST, port=HEALTH_PORT):
    self._http_signaler(port, host).AndReturn(self._http_signaler)

  def expect_health_check(self, status=True):
    self._http_signaler.health().AndReturn((status, 'reason'))

  def replay(self):
    mox.Replay(self._http_signaler)
    mox.Replay(self._health_check_a)
    mox.Replay(self._health_check_b)

  def verify(self):
    mox.Verify(self._http_signaler)
    mox.Verify(self._health_check_a)
    mox.Verify(self._health_check_b)

  def test_simple_status_health_check(self):
    """Verify that running instances are reported healthy"""
    task_a = self.create_task(0, 'a')
    task_b = self.create_task(1, 'b')
    assert self._status_health_check.health(task_a) == Retriable.alive()
    assert self._status_health_check.health(task_b) == Retriable.alive()

  def test_failed_status_health_check(self):
    """Verify that the health check fails for tasks in a state other than RUNNING"""
    pending_task = self.create_task(0, 'a', status=PENDING)
    failed_task = self.create_task(1, 'b', status=FAILED)
    assert self._status_health_check.health(pending_task) == Retriable.dead()
    assert self._status_health_check.health(failed_task) == Retriable.dead()

  def test_changed_task_id(self):
    """Verifes that an instance with a different task id causes the health check to fail"""
    task_a = self.create_task(0, 'a')
    task_b = self.create_task(0, 'b')
    assert self._status_health_check.health(task_a) == Retriable.alive()
    assert self._status_health_check.health(task_b) == NotRetriable.dead()

  def test_http_health_check(self):
    """Verify successful and failed http health checks for a task"""
    task_a = self.create_task(0, 'a')
    self.expect_http_signaler_creation()
    self.expect_health_check()
    self.expect_health_check(status=False)
    self.replay()
    assert self._http_health_check.health(task_a) == Retriable.alive()
    assert self._http_health_check.health(task_a) == Retriable.dead()
    self.verify()

  def test_unmatched_host_port(self):
    """Test if an instance with a modified a (host, port) triggers a new http health checker creation"""
    instance_id = 0
    task_a = self.create_task(instance_id, 'a')
    self.expect_http_signaler_creation()
    self.expect_health_check()
    task_b = self.create_task(instance_id, 'b', host='host-b', port=44444)
    self.expect_http_signaler_creation(host='host-b', port=44444)
    self.expect_health_check()
    self.replay()
    assert self._http_health_check.health(task_a) == Retriable.alive()
    assert self._http_health_check.health(task_b) == Retriable.alive()
    self.verify()

  def test_simple_chained_health_check(self):
    """Verify successful health check"""
    task = self.create_task(0, 'a')
    self._health_check_a.health(task).AndReturn(Retriable.alive())
    self._health_check_b.health(task).AndReturn(Retriable.alive())
    health_check = ChainedHealthCheck(self._health_check_a, self._health_check_b)
    self.replay()
    assert health_check.health(task) == Retriable.alive()
    self.verify()

  def test_failed_primary_health_check(self):
    """Verifies that a secondary health check is not invoked when a primary check fails"""
    task = self.create_task(0, 'a')
    self._health_check_a.health(task).AndReturn(NotRetriable.dead())
    self._health_check_a.health(task).AndReturn(Retriable.dead())
    health_check = ChainedHealthCheck(self._health_check_a, self._health_check_b)
    self.replay()
    assert health_check.health(task) == NotRetriable.dead()
    assert health_check.health(task) == Retriable.dead()
    self.verify()

  def test_failed_secondary_health_check(self):
    """Verifies that a secondary health check is not invoked when a primary check fails"""
    task = self.create_task(0, 'a')
    self._health_check_a.health(task).AndReturn(Retriable.alive())
    self._health_check_b.health(task).AndReturn(Retriable.dead())
    health_check = ChainedHealthCheck(self._health_check_a, self._health_check_b)
    self.replay()
    assert health_check.health(task) == Retriable.dead()
    self.verify()

  def test_health_statuses(self):
    """Verfies that the health status tuple (health, retry_status) are as expected"""
    assert Retriable.alive() == (True, True)
    assert Retriable.dead() == (False, True)
    assert NotRetriable.alive() == (True, False)
    assert NotRetriable.dead() == (False, False)

  def test_instancewatcher_health_check(self):
    """Verifies that if the task has no health port, only status check is performed"""
    task = self.create_task(0, 'a', port=None)
    self.replay()
    assert self._smart_health_check.health(task) == Retriable.alive()
    self.verify()

  def test_instancewatcher_http_health_check(self):
    """Verifies that http health check is performed if the task has a health port"""
    task = self.create_task(0, 'a')
    self.expect_http_signaler_creation()
    self.expect_health_check(status=False)
    self.replay()
    assert self._smart_health_check.health(task) == Retriable.dead()
    self.verify()

  def test_instancewatcher_http_health_check_one_http_signaler(self):
    """Verifies that upon multiple http health checks only one HttpHealthChecker is created"""
    task = self.create_task(0, 'a')
    self.expect_http_signaler_creation()
    self.expect_health_check()
    self.expect_health_check()
    self.replay()
    assert self._smart_health_check.health(task) == Retriable.alive()
    assert self._smart_health_check.health(task) == Retriable.alive()
    self.verify()