def test_reconf_two_clouds_n_stays_the_same(self):
control = MockControl()
state = MockState()
hotel_n = 4
sierra_n = 2
overall_n = hotel_n + sierra_n
hotel_cloud = make_cloud_conf('hotel', hotel_n, 1)
sierra_cloud = make_cloud_conf('sierra', sierra_n, 2)
conf = make_conf([hotel_cloud, sierra_cloud, ], overall_n, 'testdt', 'm1.small')
de = PhantomMultiSiteOverflowEngine()
de.initialize(control, state, conf)
de.decide(control, control.get_state())
self.assertEqual(control._launch_calls, overall_n)
self.assertEqual(control.site_launch_calls['hotel'], hotel_n)
self.assertEqual(control.site_launch_calls['sierra'], sierra_n)
hotel_cloud = make_cloud_conf('hotel', hotel_n + 5, 1)
sierra_cloud = make_cloud_conf('sierra', sierra_n, 2)
newconf = make_conf([hotel_cloud, sierra_cloud, ], overall_n, 'testdt', 'm1.small')
de.reconfigure(control, newconf)
de.decide(control, control.get_state())
# since we only rebalance optimistically nothing should change
self.assertEqual(control._launch_calls, overall_n)
self.assertEqual(control.site_launch_calls['hotel'], hotel_n)
self.assertEqual(control.site_launch_calls['sierra'], sierra_n)
def test_multi_cloud_n_decrease_multi_site(self):
control = MockControl()
state = MockState()
n = 14
cloud1 = make_cloud_conf('hotel', 8, 1)
cloud2 = make_cloud_conf('sierra', 4, 2)
cloud3 = make_cloud_conf('foxtrot', 2, 3)
conf = make_conf([cloud1, cloud2, cloud3], n, 'testdt', 'm1.small')
de = PhantomMultiSiteOverflowEngine()
de.initialize(control, state, conf)
de.decide(control, control.get_state())
self.assertEqual(control._launch_calls, n)
n = 10
newconf = make_conf([cloud1, cloud2, cloud3], n, 'testdt', 'm1.small')
de.reconfigure(control, newconf)
de.decide(control, control.get_state())
self.assertEqual(control._launch_calls - control._destroy_calls, n)
healthy_instances = control.get_instances(states=HEALTHY_STATES, site='foxtrot')
self.assertEqual(len(healthy_instances), 0)
healthy_instances = control.get_instances(states=HEALTHY_STATES, site='sierra')
self.assertEqual(len(healthy_instances), 2)
healthy_instances = control.get_instances(states=HEALTHY_STATES, site='hotel')
self.assertEqual(len(healthy_instances), 8)
def test_basic_capacity(self):
control = MockControl()
state = MockState()
desired_n = 10
capacity = 5
hotel_cloud = make_cloud_conf('hotel', desired_n, 1)
conf = make_conf([hotel_cloud, ], desired_n, 'testdt', 'm1.small')
de = PhantomMultiSiteOverflowEngine()
de.initialize(control, state, conf)
de.decide(control, control.get_state())
healthy_instances = control.get_instances(states=HEALTHY_STATES)
self.assertEqual(len(healthy_instances), desired_n)
for i in range(100):
# may need to add a sleep once the time metric is added
de.decide(control, control.get_state())
# mark them all failed, go back and mark up to capacity as running
# this is how we fake a capacity of n
for i in control.instances:
i.state = InstanceState.FAILED
for i in range(0, capacity):
control.instances[i].state = InstanceState.RUNNING
healthy_instances = control.get_instances(states=HEALTHY_STATES)
self.assertEqual(len(healthy_instances), capacity)
# at this point the DE should have figured out that we are at capacity so it should
# not decide to try to add more
de.decide(control, control.get_state())
healthy_instances = control.get_instances(states=HEALTHY_STATES)
self.assertEqual(len(healthy_instances), capacity)
# now reduce the desired n down to the capacity. nothing should be killed
conf = make_conf([hotel_cloud, ], capacity, 'testdt', 'm1.small')
de.reconfigure(control, conf)
de.decide(control, control.get_state())
healthy_instances = control.get_instances(states=HEALTHY_STATES)
self.assertEqual(len(healthy_instances), capacity)
self.assertEqual(control._destroy_calls, 0)
def test_reconf_two_clouds_n_stays_the_same_but_node_dies(self):
control = MockControl()
state = MockState()
hotel_n = 4
sierra_n = 2
overall_n = hotel_n + sierra_n
hotel_cloud = make_cloud_conf('hotel', hotel_n, 1)
sierra_cloud = make_cloud_conf('sierra', sierra_n, 2)
conf = make_conf([hotel_cloud, sierra_cloud, ], overall_n, 'testdt', 'm1.small')
de = PhantomMultiSiteOverflowEngine()
de.initialize(control, state, conf)
de.decide(control, control.get_state())
self.assertEqual(control._launch_calls, overall_n)
self.assertEqual(control.site_launch_calls['hotel'], hotel_n)
self.assertEqual(control.site_launch_calls['sierra'], sierra_n)
hotel_cloud = make_cloud_conf('hotel', hotel_n + 5, 1)
sierra_cloud = make_cloud_conf('sierra', sierra_n, 2)
newconf = make_conf([hotel_cloud, sierra_cloud, ], overall_n, 'testdt', 'm1.small')
de.reconfigure(control, newconf)
de.decide(control, control.get_state())
# since we only rebalance optimistically nothing should change
self.assertEqual(control._launch_calls, overall_n)
self.assertEqual(control.site_launch_calls['hotel'], hotel_n)
self.assertEqual(control.site_launch_calls['sierra'], sierra_n)
for i in control.instances:
if i.site == 'sierra':
i.state = InstanceState.FAILED
break
# check to verify that we optimistically rebalanced
de.decide(control, control.get_state())
healthy_instances = control.get_instances(states=HEALTHY_STATES)
sierra_instances = control.get_instances(site='sierra', states=HEALTHY_STATES)
hotel_instances = control.get_instances(site='hotel', states=HEALTHY_STATES)
self.assertEqual(len(healthy_instances), overall_n)
self.assertEqual(len(hotel_instances), hotel_n + 1)
self.assertEqual(len(sierra_instances), sierra_n - 1)
def test_reconf_two_clouds_n_increase(self):
control = MockControl()
state = MockState()
overall_n = 3
hotel_n = 4
sierra_n = 2
hotel_cloud = make_cloud_conf('hotel', hotel_n, 1)
sierra_cloud = make_cloud_conf('sierra', sierra_n, 2)
conf = make_conf([hotel_cloud, sierra_cloud, ], overall_n, 'testdt', 'm1.small')
de = PhantomMultiSiteOverflowEngine()
de.initialize(control, state, conf)
de.decide(control, control.get_state())
self.assertEqual(control._launch_calls, overall_n)
self.assertEqual(control.site_launch_calls['hotel'], 3)
self.assertFalse('sierra' in control.site_launch_calls)
overall_n = 5
hotel_cloud = make_cloud_conf('hotel', hotel_n, 1)
sierra_cloud = make_cloud_conf('sierra', sierra_n, 2)
newconf = make_conf([hotel_cloud, sierra_cloud, ], overall_n, 'testdt', 'm1.small')
de.reconfigure(control, newconf)
de.decide(control, control.get_state())
self.assertEqual(control._launch_calls, overall_n)
self.assertEqual(control.site_launch_calls['hotel'], hotel_n)
self.assertEqual(control.site_launch_calls['sierra'], 1)
overall_n = hotel_n + sierra_n + 10
hotel_cloud = make_cloud_conf('hotel', hotel_n, 1)
sierra_cloud = make_cloud_conf('sierra', sierra_n, 2)
newconf = make_conf([hotel_cloud, sierra_cloud, ], overall_n, 'testdt', 'm1.small')
de.reconfigure(control, newconf)
de.decide(control, control.get_state())
self.assertEqual(control._launch_calls, hotel_n + sierra_n)
self.assertEqual(control.site_launch_calls['hotel'], hotel_n)
self.assertEqual(control.site_launch_calls['sierra'], sierra_n)
def test_basic_reconf_one_cloud_n_and_overall_n_decrease(self):
control = MockControl()
state = MockState()
n = 4
cloud = make_cloud_conf('hotel', n, 1)
conf = make_conf([cloud, ], n, 'testdt', 'm1.small')
de = PhantomMultiSiteOverflowEngine()
de.initialize(control, state, conf)
de.decide(control, control.get_state())
self.assertEqual(control._launch_calls, n)
n = 2
cloud = make_cloud_conf('hotel', n, 1)
newconf = make_conf([cloud, ], n, 'testdt', 'm1.small')
de.reconfigure(control, newconf)
de.decide(control, control.get_state())
self.assertEqual(control._launch_calls - control._destroy_calls, n)
def test_conf_errors(self):
control = MockControl()
state = MockState()
conf = None
de = PhantomMultiSiteOverflowEngine()
try:
de.initialize(control, state, conf)
self.fail("An exception should have been thrown")
except ValueError:
pass
conf = {}
de = PhantomMultiSiteOverflowEngine()
try:
de.initialize(control, state, conf)
self.fail("An exception should have been thrown")
except ValueError:
pass
needed_keys = [('minimum_vms', 1), ('dtname', 'hello'), ('instance_type', 'm1.huge')]
for k in needed_keys:
conf[k[0]] = k[1]
de = PhantomMultiSiteOverflowEngine()
try:
de.initialize(control, state, conf)
self.fail("An exception should have been thrown")
except ValueError:
pass
conf['clouds'] = make_cloud_conf('hotel', 10, 3)
de = PhantomMultiSiteOverflowEngine()
try:
de.initialize(control, state, conf)
self.fail("An exception should have been thrown")
except ValueError:
pass
except Exception:
pass
cloud1 = make_cloud_conf('hotel', 10, 1)
cloud2 = make_cloud_conf('sierra', 10, 3)
conf['clouds'] = [cloud1, cloud2]
de = PhantomMultiSiteOverflowEngine()
try:
de.initialize(control, state, conf)
self.fail("An exception should have been thrown")
except ValueError:
pass
except Exception:
pass
cloud1 = make_cloud_conf('hotel', 10, 1)
cloud2 = make_cloud_conf('sierra', 10, 2)
conf['clouds'] = [cloud1, cloud2]
de.initialize(control, state, conf)
cloud1 = make_cloud_conf('hotel', 10, 1)
cloud2 = make_cloud_conf('sierra', 10, 5)
conf['clouds'] = [cloud1, cloud2]
try:
de.reconfigure(control, conf)
self.fail("An exception should have been thrown")
except ValueError:
pass
except Exception:
pass
cloud2 = make_cloud_conf('foxtrot', 10, 1)
conf['clouds'] = [cloud2, ]
try:
de.reconfigure(control, conf)
self.fail("An exception should have been thrown")
except ValueError:
pass
except Exception:
pass
cloud2 = make_cloud_conf('foxtrot', 10, 3)
conf['clouds'] = [cloud2, ]
de.reconfigure(control, conf)
try:
de.reconfigure(control, {})
self.fail("An exception should have been thrown")
except ValueError:
pass
except Exception:
pass
de.dying()
