def test_multi_cloud_n_decrease_multi_site(self): control = MockControl() state = MockState() n = 14 cloud1 = make_cloud_conf('hotel', 8, 1) cloud2 = make_cloud_conf('sierra', 4, 2) cloud3 = make_cloud_conf('foxtrot', 2, 3) conf = make_conf([cloud1, cloud2, cloud3], n, 'testdt', 'm1.small') de = PhantomMultiSiteOverflowEngine() de.initialize(control, state, conf) de.decide(control, control.get_state()) self.assertEqual(control._launch_calls, n) n = 10 newconf = make_conf([cloud1, cloud2, cloud3], n, 'testdt', 'm1.small') de.reconfigure(control, newconf) de.decide(control, control.get_state()) self.assertEqual(control._launch_calls - control._destroy_calls, n) healthy_instances = control.get_instances(states=HEALTHY_STATES, site='foxtrot') self.assertEqual(len(healthy_instances), 0) healthy_instances = control.get_instances(states=HEALTHY_STATES, site='sierra') self.assertEqual(len(healthy_instances), 2) healthy_instances = control.get_instances(states=HEALTHY_STATES, site='hotel') self.assertEqual(len(healthy_instances), 8)
def test_basic_capacity(self): control = MockControl() state = MockState() desired_n = 10 capacity = 5 hotel_cloud = make_cloud_conf('hotel', desired_n, 1) conf = make_conf([hotel_cloud, ], desired_n, 'testdt', 'm1.small') de = PhantomMultiSiteOverflowEngine() de.initialize(control, state, conf) de.decide(control, control.get_state()) healthy_instances = control.get_instances(states=HEALTHY_STATES) self.assertEqual(len(healthy_instances), desired_n) for i in range(100): # may need to add a sleep once the time metric is added de.decide(control, control.get_state()) # mark them all failed, go back and mark up to capacity as running # this is how we fake a capacity of n for i in control.instances: i.state = InstanceState.FAILED for i in range(0, capacity): control.instances[i].state = InstanceState.RUNNING healthy_instances = control.get_instances(states=HEALTHY_STATES) self.assertEqual(len(healthy_instances), capacity) # at this point the DE should have figured out that we are at capacity so it should # not decide to try to add more de.decide(control, control.get_state()) healthy_instances = control.get_instances(states=HEALTHY_STATES) self.assertEqual(len(healthy_instances), capacity) # now reduce the desired n down to the capacity. nothing should be killed conf = make_conf([hotel_cloud, ], capacity, 'testdt', 'm1.small') de.reconfigure(control, conf) de.decide(control, control.get_state()) healthy_instances = control.get_instances(states=HEALTHY_STATES) self.assertEqual(len(healthy_instances), capacity) self.assertEqual(control._destroy_calls, 0)
def test_reconf_two_clouds_n_stays_the_same_but_node_dies(self): control = MockControl() state = MockState() hotel_n = 4 sierra_n = 2 overall_n = hotel_n + sierra_n hotel_cloud = make_cloud_conf('hotel', hotel_n, 1) sierra_cloud = make_cloud_conf('sierra', sierra_n, 2) conf = make_conf([hotel_cloud, sierra_cloud, ], overall_n, 'testdt', 'm1.small') de = PhantomMultiSiteOverflowEngine() de.initialize(control, state, conf) de.decide(control, control.get_state()) self.assertEqual(control._launch_calls, overall_n) self.assertEqual(control.site_launch_calls['hotel'], hotel_n) self.assertEqual(control.site_launch_calls['sierra'], sierra_n) hotel_cloud = make_cloud_conf('hotel', hotel_n + 5, 1) sierra_cloud = make_cloud_conf('sierra', sierra_n, 2) newconf = make_conf([hotel_cloud, sierra_cloud, ], overall_n, 'testdt', 'm1.small') de.reconfigure(control, newconf) de.decide(control, control.get_state()) # since we only rebalance optimistically nothing should change self.assertEqual(control._launch_calls, overall_n) self.assertEqual(control.site_launch_calls['hotel'], hotel_n) self.assertEqual(control.site_launch_calls['sierra'], sierra_n) for i in control.instances: if i.site == 'sierra': i.state = InstanceState.FAILED break # check to verify that we optimistically rebalanced de.decide(control, control.get_state()) healthy_instances = control.get_instances(states=HEALTHY_STATES) sierra_instances = control.get_instances(site='sierra', states=HEALTHY_STATES) hotel_instances = control.get_instances(site='hotel', states=HEALTHY_STATES) self.assertEqual(len(healthy_instances), overall_n) self.assertEqual(len(hotel_instances), hotel_n + 1) self.assertEqual(len(sierra_instances), sierra_n - 1)
def test_capacity_overflow(self): control = MockControl() state = MockState() desired_n = 10 capacity = 5 hotel_cloud = make_cloud_conf('hotel', desired_n * 2, 1) # set hotel to have plenty of room sierra_cloud = make_cloud_conf('sierra', desired_n, 2) # set hotel to have enough for the overflow conf = make_conf([hotel_cloud, sierra_cloud, ], desired_n, 'testdt', 'm1.small') de = PhantomMultiSiteOverflowEngine() de.initialize(control, state, conf) de.decide(control, control.get_state()) healthy_instances = control.get_instances(states=HEALTHY_STATES) self.assertEqual(len(healthy_instances), desired_n) for i in range(100): de.decide(control, control.get_state()) skip_count = 0 for i in control.instances: if i.site == 'hotel': if skip_count >= capacity: i.state = InstanceState.FAILED skip_count = skip_count + 1 healthy_instances = control.get_instances(states=HEALTHY_STATES, site='hotel') self.assertEqual(len(healthy_instances), capacity) # at this point the DE should have figured out that we are at capacity so it should # not decide to try to add more de.decide(control, control.get_state()) healthy_instances = control.get_instances(states=HEALTHY_STATES, site='hotel') self.assertEqual(len(healthy_instances), capacity) sierra_healthy_instances = control.get_instances(states=HEALTHY_STATES, site='sierra') self.assertEqual(len(sierra_healthy_instances), desired_n - capacity) healthy_instances = control.get_instances(states=HEALTHY_STATES) self.assertEqual(len(healthy_instances), desired_n)