示例#1
0
    def setUp(self):
        self.db = MagicMock(spec=FlotillaSchedulerDynamo)
        self.elb = MagicMock()
        self.service = {
            'service_name': SERVICE,
            REV: 1,
            'cf_outputs': {
                'Elb': 'test-elb',
                'InstanceSg': 'sg-123456'
            }
        }
        self.db.get_service.return_value = self.service

        self.doctor = ServiceDoctor(self.db, self.elb)
示例#2
0
def wait_for_deployment(dynamo_cache, regions, service_name, rev_hash, timeout):
    logger.info('Waiting for %s in %s regions...', rev_hash, len(regions))

    # There can be only one!
    doctor_cache = {}
    for region in regions:
        tables = dynamo_cache[region]
        tables.setup(['status'])
        db = FlotillaSchedulerDynamo(None, None, tables.services, None,
                                     tables.status)
        elb = boto3.client('elb', region)
        doctor = ServiceDoctor(db, elb)
        doctor_cache[region] = doctor

    start_time = time()
    while True:
        all_healthy = True
        try:
            for region, doctor in doctor_cache.items():
                healthy = doctor.is_healthy_revision(service_name, rev_hash)
                if not healthy:
                    all_healthy = False
                    logger.info('Waiting for %s in %s...', rev_hash, region)
                    continue
                logger.info('Region %s has a health %s instance!', region,
                            rev_hash)
                doctor.db.make_only_revision(service_name, rev_hash)
        except ValueError:
            break

        if all_healthy:
            logger.info('All regions have a health %s instance!', rev_hash)
            return True
        if time() - start_time > timeout:
            break
        sleep(5)

    wait_time = time() - start_time
    logger.info('Revision %s not stable after %s seconds.', rev_hash, wait_time)
    for region, doctor in doctor_cache.items():
        service_item = doctor.db.get_service(service_name)
        if rev_hash in service_item and service_item[rev_hash] > 0:
            service_item[rev_hash] *= -1
            doctor.db.set_services([service_item])
示例#3
0
    def setUp(self):
        self.db = MagicMock(spec=FlotillaSchedulerDynamo)
        self.elb = MagicMock()
        self.service = {
            'service_name': SERVICE,
            REV: 1,
            'cf_outputs': {
                'Elb': 'test-elb',
                'InstanceSg': 'sg-123456'
            }
        }
        self.db.get_service.return_value = self.service

        self.doctor = ServiceDoctor(self.db, self.elb)
示例#4
0
class TestServiceDoctor(unittest.TestCase):
    def setUp(self):
        self.db = MagicMock(spec=FlotillaSchedulerDynamo)
        self.elb = MagicMock()
        self.service = {
            'service_name': SERVICE,
            REV: 1,
            'cf_outputs': {
                'Elb': 'test-elb',
                'InstanceSg': 'sg-123456'
            }
        }
        self.db.get_service.return_value = self.service

        self.doctor = ServiceDoctor(self.db, self.elb)

    def test_failed_revision_missing(self):
        self.db.get_service.return_value = None
        self.doctor.failed_revision(SERVICE, REV, INSTANCE)
        self.db.set_services.assert_not_called()

    def test_failed_revision_bad_rev(self):
        self.doctor.failed_revision(SERVICE, 'abcdef', INSTANCE)
        self.db.set_services.assert_not_called()

    def test_failed_revision_not_running(self):
        # No instances are running this rev
        self.doctor.failed_revision(SERVICE, REV, INSTANCE)
        self.elb.describe_instance_health.assert_not_called()
        self.assertEqual(self.service[REV], -1)
        self.db.set_services.assert_called_with(ANY)

    def test_failed_revision_not_healthy(self):
        # Instances running, but ELB doesn't have them registered
        self._mock_service_status()
        self.doctor.failed_revision(SERVICE, REV, INSTANCE)
        self.assertEqual(self.service[REV], -1)
        self.db.set_services.assert_called_with(ANY)

    def test_failed_revision_healthy(self):
        self._mock_service_status()
        self.elb.describe_instance_health.return_value = {
            'InstanceStates': [
                {'InstanceId': INSTANCE,
                 'State': 'InService'}
            ]
        }
        self.doctor.failed_revision(SERVICE, REV, INSTANCE)
        self.assertEqual(self.service[REV], 1)
        self.db.set_services.assert_not_called()

    def test_healthy_instances_no_outputs(self):
        del self.service['cf_outputs']
        healthy = self.doctor._healthy_instances(self.service, [INSTANCE])
        self.assertEquals(healthy, set())

    def test_healthy_instances_no_elb(self):
        del self.service['cf_outputs']['Elb']
        healthy = self.doctor._healthy_instances(self.service, [INSTANCE])
        self.assertEquals(healthy, set())

    def test_is_healthy_revision_not_found(self):
        self.doctor._get_service_with_rev = MagicMock(return_value=None)
        healthy = self.doctor.is_healthy_revision(SERVICE, REV)
        self.assertEqual(healthy, False)

    def test_is_healthy_revision_not_healthy(self):
        self._mock_service_status()
        self.doctor._healthy_instances_with_rev = MagicMock(return_value=False)
        healthy = self.doctor.is_healthy_revision(SERVICE, REV)
        self.assertEqual(healthy, False)

    def test_is_healthy_revision_hard_fail(self):
        self.service[REV] = -1
        self.assertRaises(ValueError, self.doctor.is_healthy_revision, SERVICE,
                          REV)

    def _mock_service_status(self):
        self.db.get_service_status.return_value = {
            'i-654321': {
                'service-foo': {
                    'sub_state': 'running'
                }
            }
        }.items()
示例#5
0
class TestServiceDoctor(unittest.TestCase):
    def setUp(self):
        self.db = MagicMock(spec=FlotillaSchedulerDynamo)
        self.elb = MagicMock()
        self.service = {
            'service_name': SERVICE,
            REV: 1,
            'cf_outputs': {
                'Elb': 'test-elb',
                'InstanceSg': 'sg-123456'
            }
        }
        self.db.get_service.return_value = self.service

        self.doctor = ServiceDoctor(self.db, self.elb)

    def test_failed_revision_missing(self):
        self.db.get_service.return_value = None
        self.doctor.failed_revision(SERVICE, REV, INSTANCE)
        self.db.set_services.assert_not_called()

    def test_failed_revision_bad_rev(self):
        self.doctor.failed_revision(SERVICE, 'abcdef', INSTANCE)
        self.db.set_services.assert_not_called()

    def test_failed_revision_not_running(self):
        # No instances are running this rev
        self.doctor.failed_revision(SERVICE, REV, INSTANCE)
        self.elb.describe_instance_health.assert_not_called()
        self.assertEqual(self.service[REV], -1)
        self.db.set_services.assert_called_with(ANY)

    def test_failed_revision_not_healthy(self):
        # Instances running, but ELB doesn't have them registered
        self._mock_service_status()
        self.doctor.failed_revision(SERVICE, REV, INSTANCE)
        self.assertEqual(self.service[REV], -1)
        self.db.set_services.assert_called_with(ANY)

    def test_failed_revision_healthy(self):
        self._mock_service_status()
        self.elb.describe_instance_health.return_value = {
            'InstanceStates': [{
                'InstanceId': INSTANCE,
                'State': 'InService'
            }]
        }
        self.doctor.failed_revision(SERVICE, REV, INSTANCE)
        self.assertEqual(self.service[REV], 1)
        self.db.set_services.assert_not_called()

    def test_healthy_instances_no_outputs(self):
        del self.service['cf_outputs']
        healthy = self.doctor._healthy_instances(self.service, [INSTANCE])
        self.assertEquals(healthy, set())

    def test_healthy_instances_no_elb(self):
        del self.service['cf_outputs']['Elb']
        healthy = self.doctor._healthy_instances(self.service, [INSTANCE])
        self.assertEquals(healthy, set())

    def test_is_healthy_revision_not_found(self):
        self.doctor._get_service_with_rev = MagicMock(return_value=None)
        healthy = self.doctor.is_healthy_revision(SERVICE, REV)
        self.assertEqual(healthy, False)

    def test_is_healthy_revision_not_healthy(self):
        self._mock_service_status()
        self.doctor._healthy_instances_with_rev = MagicMock(return_value=False)
        healthy = self.doctor.is_healthy_revision(SERVICE, REV)
        self.assertEqual(healthy, False)

    def test_is_healthy_revision_hard_fail(self):
        self.service[REV] = -1
        self.assertRaises(ValueError, self.doctor.is_healthy_revision, SERVICE,
                          REV)

    def _mock_service_status(self):
        self.db.get_service_status.return_value = {
            'i-654321': {
                'service-foo': {
                    'sub_state': 'running'
                }
            }
        }.items()