def test_multi_cloud_n_decrease_multi_site(self):
control = MockControl()
state = MockState()
n = 14
cloud1 = make_cloud_conf('hotel', 8, 1)
cloud2 = make_cloud_conf('sierra', 4, 2)
cloud3 = make_cloud_conf('foxtrot', 2, 3)
conf = make_conf([cloud1, cloud2, cloud3], n, 'testdt', 'm1.small')
de = PhantomMultiSiteOverflowEngine()
de.initialize(control, state, conf)
de.decide(control, control.get_state())
self.assertEqual(control._launch_calls, n)
n = 10
newconf = make_conf([cloud1, cloud2, cloud3], n, 'testdt', 'm1.small')
de.reconfigure(control, newconf)
de.decide(control, control.get_state())
self.assertEqual(control._launch_calls - control._destroy_calls, n)
healthy_instances = control.get_instances(states=HEALTHY_STATES, site='foxtrot')
self.assertEqual(len(healthy_instances), 0)
healthy_instances = control.get_instances(states=HEALTHY_STATES, site='sierra')
self.assertEqual(len(healthy_instances), 2)
healthy_instances = control.get_instances(states=HEALTHY_STATES, site='hotel')
self.assertEqual(len(healthy_instances), 8)
def test_basic_capacity(self):
control = MockControl()
state = MockState()
desired_n = 10
capacity = 5
hotel_cloud = make_cloud_conf('hotel', desired_n, 1)
conf = make_conf([hotel_cloud, ], desired_n, 'testdt', 'm1.small')
de = PhantomMultiSiteOverflowEngine()
de.initialize(control, state, conf)
de.decide(control, control.get_state())
healthy_instances = control.get_instances(states=HEALTHY_STATES)
self.assertEqual(len(healthy_instances), desired_n)
for i in range(100):
# may need to add a sleep once the time metric is added
de.decide(control, control.get_state())
# mark them all failed, go back and mark up to capacity as running
# this is how we fake a capacity of n
for i in control.instances:
i.state = InstanceState.FAILED
for i in range(0, capacity):
control.instances[i].state = InstanceState.RUNNING
healthy_instances = control.get_instances(states=HEALTHY_STATES)
self.assertEqual(len(healthy_instances), capacity)
# at this point the DE should have figured out that we are at capacity so it should
# not decide to try to add more
de.decide(control, control.get_state())
healthy_instances = control.get_instances(states=HEALTHY_STATES)
self.assertEqual(len(healthy_instances), capacity)
# now reduce the desired n down to the capacity. nothing should be killed
conf = make_conf([hotel_cloud, ], capacity, 'testdt', 'm1.small')
de.reconfigure(control, conf)
de.decide(control, control.get_state())
healthy_instances = control.get_instances(states=HEALTHY_STATES)
self.assertEqual(len(healthy_instances), capacity)
self.assertEqual(control._destroy_calls, 0)
def test_reconf_two_clouds_n_stays_the_same_but_node_dies(self):
control = MockControl()
state = MockState()
hotel_n = 4
sierra_n = 2
overall_n = hotel_n + sierra_n
hotel_cloud = make_cloud_conf('hotel', hotel_n, 1)
sierra_cloud = make_cloud_conf('sierra', sierra_n, 2)
conf = make_conf([hotel_cloud, sierra_cloud, ], overall_n, 'testdt', 'm1.small')
de = PhantomMultiSiteOverflowEngine()
de.initialize(control, state, conf)
de.decide(control, control.get_state())
self.assertEqual(control._launch_calls, overall_n)
self.assertEqual(control.site_launch_calls['hotel'], hotel_n)
self.assertEqual(control.site_launch_calls['sierra'], sierra_n)
hotel_cloud = make_cloud_conf('hotel', hotel_n + 5, 1)
sierra_cloud = make_cloud_conf('sierra', sierra_n, 2)
newconf = make_conf([hotel_cloud, sierra_cloud, ], overall_n, 'testdt', 'm1.small')
de.reconfigure(control, newconf)
de.decide(control, control.get_state())
# since we only rebalance optimistically nothing should change
self.assertEqual(control._launch_calls, overall_n)
self.assertEqual(control.site_launch_calls['hotel'], hotel_n)
self.assertEqual(control.site_launch_calls['sierra'], sierra_n)
for i in control.instances:
if i.site == 'sierra':
i.state = InstanceState.FAILED
break
# check to verify that we optimistically rebalanced
de.decide(control, control.get_state())
healthy_instances = control.get_instances(states=HEALTHY_STATES)
sierra_instances = control.get_instances(site='sierra', states=HEALTHY_STATES)
hotel_instances = control.get_instances(site='hotel', states=HEALTHY_STATES)
self.assertEqual(len(healthy_instances), overall_n)
self.assertEqual(len(hotel_instances), hotel_n + 1)
self.assertEqual(len(sierra_instances), sierra_n - 1)
def test_capacity_overflow(self):
control = MockControl()
state = MockState()
desired_n = 10
capacity = 5
hotel_cloud = make_cloud_conf('hotel', desired_n * 2, 1) # set hotel to have plenty of room
sierra_cloud = make_cloud_conf('sierra', desired_n, 2) # set hotel to have enough for the overflow
conf = make_conf([hotel_cloud, sierra_cloud, ], desired_n, 'testdt', 'm1.small')
de = PhantomMultiSiteOverflowEngine()
de.initialize(control, state, conf)
de.decide(control, control.get_state())
healthy_instances = control.get_instances(states=HEALTHY_STATES)
self.assertEqual(len(healthy_instances), desired_n)
for i in range(100):
de.decide(control, control.get_state())
skip_count = 0
for i in control.instances:
if i.site == 'hotel':
if skip_count >= capacity:
i.state = InstanceState.FAILED
skip_count = skip_count + 1
healthy_instances = control.get_instances(states=HEALTHY_STATES, site='hotel')
self.assertEqual(len(healthy_instances), capacity)
# at this point the DE should have figured out that we are at capacity so it should
# not decide to try to add more
de.decide(control, control.get_state())
healthy_instances = control.get_instances(states=HEALTHY_STATES, site='hotel')
self.assertEqual(len(healthy_instances), capacity)
sierra_healthy_instances = control.get_instances(states=HEALTHY_STATES, site='sierra')
self.assertEqual(len(sierra_healthy_instances), desired_n - capacity)
healthy_instances = control.get_instances(states=HEALTHY_STATES)
self.assertEqual(len(healthy_instances), desired_n)