def test_uniques(self):
control = MockControl()
state = control.get_state()
minimum_n = 2
maximum_n = 3
metric = 'fake'
sample_function = Statistics.AVERAGE
scale_up_threshold = 2.0
scale_down_threshold = 0.5
scale_up_n_vms = 1
scale_down_n_vms = 1
config = self._get_config(minimum_n, maximum_n, metric, sample_function,
scale_up_threshold, scale_up_n_vms, scale_down_threshold,
scale_down_n_vms, "dt1", unique_key="somekey", unique_values=[1, 2, 3])
de = SensorEngine()
de.initialize(control, state, config)
de.decide(control, state)
self.assertEqual(control._launch_calls, 2)
self.assertEqual(control.instances[0].extravars, {"somekey": 1})
self.assertEqual(control.instances[1].extravars, {"somekey": 2})
# kill first. replacement should get same unique
control.instances[0].state = InstanceState.FAILED
state = control.get_state()
de.decide(control, state)
self.assertEqual(control._launch_calls, 3)
self.assertEqual(control.instances[2].extravars, {"somekey": 1})
self.assertEqual(control.instances[1].extravars, {"somekey": 2})
def test_reconf_two_clouds_n_stays_the_same(self):
control = MockControl()
state = MockState()
hotel_n = 4
sierra_n = 2
overall_n = hotel_n + sierra_n
hotel_cloud = make_cloud_conf('hotel', hotel_n, 1)
sierra_cloud = make_cloud_conf('sierra', sierra_n, 2)
conf = make_conf([hotel_cloud, sierra_cloud, ], overall_n, 'testdt', 'm1.small')
de = PhantomMultiSiteOverflowEngine()
de.initialize(control, state, conf)
de.decide(control, control.get_state())
self.assertEqual(control._launch_calls, overall_n)
self.assertEqual(control.site_launch_calls['hotel'], hotel_n)
self.assertEqual(control.site_launch_calls['sierra'], sierra_n)
hotel_cloud = make_cloud_conf('hotel', hotel_n + 5, 1)
sierra_cloud = make_cloud_conf('sierra', sierra_n, 2)
newconf = make_conf([hotel_cloud, sierra_cloud, ], overall_n, 'testdt', 'm1.small')
de.reconfigure(control, newconf)
de.decide(control, control.get_state())
# since we only rebalance optimistically nothing should change
self.assertEqual(control._launch_calls, overall_n)
self.assertEqual(control.site_launch_calls['hotel'], hotel_n)
self.assertEqual(control.site_launch_calls['sierra'], sierra_n)
def test_basic_move_to_pending(self):
control = MockControl()
state = MockState()
n = 2
cloud_names = [('hotel', n), ]
clouds = []
rank = 1
for (cn, count) in cloud_names:
cloud = make_cloud_conf(cn, count, rank)
rank = rank + 1
clouds.append(cloud)
conf = make_conf(clouds, n, 'testdt', 'm1.small')
de = PhantomMultiSiteOverflowEngine()
de.initialize(control, state, conf)
de.decide(control, control.get_state())
self.assertEqual(control._launch_calls, n)
for i in control.instances:
i.state = InstanceState.PENDING
de.decide(control, control.get_state())
self.assertEqual(control._launch_calls, n)
def test_uniques_empty_values_2(self):
control = MockControl()
state = control.get_state()
de = NeedyEngine()
de.initialize(control, state, self._get_config(1, "dt1",
unique_key="somekey", unique_values=" "))
de.decide(control, state)
self.assertEqual(control._launch_calls, 1)
self.assertEqual(control.instances[0].extravars, None)
def test_basic(self):
control = MockControl()
state = control.get_state()
de = NeedyEngine()
de.initialize(control, state, self._get_config(1, "dt1"))
de.decide(control, state)
self.assertEqual(control._launch_calls, 1)
state = control.get_state()
de.reconfigure(control, dict(preserve_n=2))
de.decide(control, state)
self.assertEqual(control._launch_calls, 2)
def test_basic_capacity(self):
control = MockControl()
state = MockState()
desired_n = 10
capacity = 5
hotel_cloud = make_cloud_conf('hotel', desired_n, 1)
conf = make_conf([hotel_cloud, ], desired_n, 'testdt', 'm1.small')
de = PhantomMultiSiteOverflowEngine()
de.initialize(control, state, conf)
de.decide(control, control.get_state())
healthy_instances = control.get_instances(states=HEALTHY_STATES)
self.assertEqual(len(healthy_instances), desired_n)
for i in range(100):
# may need to add a sleep once the time metric is added
de.decide(control, control.get_state())
# mark them all failed, go back and mark up to capacity as running
# this is how we fake a capacity of n
for i in control.instances:
i.state = InstanceState.FAILED
for i in range(0, capacity):
control.instances[i].state = InstanceState.RUNNING
healthy_instances = control.get_instances(states=HEALTHY_STATES)
self.assertEqual(len(healthy_instances), capacity)
# at this point the DE should have figured out that we are at capacity so it should
# not decide to try to add more
de.decide(control, control.get_state())
healthy_instances = control.get_instances(states=HEALTHY_STATES)
self.assertEqual(len(healthy_instances), capacity)
# now reduce the desired n down to the capacity. nothing should be killed
conf = make_conf([hotel_cloud, ], capacity, 'testdt', 'm1.small')
de.reconfigure(control, conf)
de.decide(control, control.get_state())
healthy_instances = control.get_instances(states=HEALTHY_STATES)
self.assertEqual(len(healthy_instances), capacity)
self.assertEqual(control._destroy_calls, 0)
def test_reconf_two_clouds_n_increase(self):
control = MockControl()
state = MockState()
overall_n = 3
hotel_n = 4
sierra_n = 2
hotel_cloud = make_cloud_conf('hotel', hotel_n, 1)
sierra_cloud = make_cloud_conf('sierra', sierra_n, 2)
conf = make_conf([hotel_cloud, sierra_cloud, ], overall_n, 'testdt', 'm1.small')
de = PhantomMultiSiteOverflowEngine()
de.initialize(control, state, conf)
de.decide(control, control.get_state())
self.assertEqual(control._launch_calls, overall_n)
self.assertEqual(control.site_launch_calls['hotel'], 3)
self.assertFalse('sierra' in control.site_launch_calls)
overall_n = 5
hotel_cloud = make_cloud_conf('hotel', hotel_n, 1)
sierra_cloud = make_cloud_conf('sierra', sierra_n, 2)
newconf = make_conf([hotel_cloud, sierra_cloud, ], overall_n, 'testdt', 'm1.small')
de.reconfigure(control, newconf)
de.decide(control, control.get_state())
self.assertEqual(control._launch_calls, overall_n)
self.assertEqual(control.site_launch_calls['hotel'], hotel_n)
self.assertEqual(control.site_launch_calls['sierra'], 1)
overall_n = hotel_n + sierra_n + 10
hotel_cloud = make_cloud_conf('hotel', hotel_n, 1)
sierra_cloud = make_cloud_conf('sierra', sierra_n, 2)
newconf = make_conf([hotel_cloud, sierra_cloud, ], overall_n, 'testdt', 'm1.small')
de.reconfigure(control, newconf)
de.decide(control, control.get_state())
self.assertEqual(control._launch_calls, hotel_n + sierra_n)
self.assertEqual(control.site_launch_calls['hotel'], hotel_n)
self.assertEqual(control.site_launch_calls['sierra'], sierra_n)
def test_basic_node_failed(self):
control = MockControl()
state = MockState()
n = 4
cloud = make_cloud_conf('hotel', n, 1)
conf = make_conf([cloud, ], n, 'testdt', 'm1.small')
de = PhantomMultiSiteOverflowEngine()
de.initialize(control, state, conf)
de.decide(control, control.get_state())
self.assertEqual(control._launch_calls, n)
i = random.choice(control.instances)
i.state = InstanceState.FAILED
de.decide(control, control.get_state())
self.assertEqual(control._launch_calls, n + 1)
def test_ran_out_of_uniques(self):
control = MockControl()
state = control.get_state()
de = NeedyEngine()
de.initialize(control, state, self._get_config(2, "dt1",
unique_key="somekey", unique_values=[1, ]))
de.decide(control, state)
self.assertEqual(control._launch_calls, 1)
self.assertEqual(control.instances[0].extravars, {"somekey": 1})
# kill first. replacement should get same unique
control.instances[0].state = InstanceState.FAILED
state = control.get_state()
de.decide(control, state)
self.assertEqual(control._launch_calls, 2)
self.assertEqual(control.instances[1].extravars, {"somekey": 1})
def test_basic_reconf_one_cloud_n_and_overall_n_decrease(self):
control = MockControl()
state = MockState()
n = 4
cloud = make_cloud_conf('hotel', n, 1)
conf = make_conf([cloud, ], n, 'testdt', 'm1.small')
de = PhantomMultiSiteOverflowEngine()
de.initialize(control, state, conf)
de.decide(control, control.get_state())
self.assertEqual(control._launch_calls, n)
n = 2
cloud = make_cloud_conf('hotel', n, 1)
newconf = make_conf([cloud, ], n, 'testdt', 'm1.small')
de.reconfigure(control, newconf)
de.decide(control, control.get_state())
self.assertEqual(control._launch_calls - control._destroy_calls, n)
def test_multi_cloud_n_decrease_multi_site(self):
control = MockControl()
state = MockState()
n = 14
cloud1 = make_cloud_conf('hotel', 8, 1)
cloud2 = make_cloud_conf('sierra', 4, 2)
cloud3 = make_cloud_conf('foxtrot', 2, 3)
conf = make_conf([cloud1, cloud2, cloud3], n, 'testdt', 'm1.small')
de = PhantomMultiSiteOverflowEngine()
de.initialize(control, state, conf)
de.decide(control, control.get_state())
self.assertEqual(control._launch_calls, n)
n = 10
newconf = make_conf([cloud1, cloud2, cloud3], n, 'testdt', 'm1.small')
de.reconfigure(control, newconf)
de.decide(control, control.get_state())
self.assertEqual(control._launch_calls - control._destroy_calls, n)
healthy_instances = control.get_instances(states=HEALTHY_STATES, site='foxtrot')
self.assertEqual(len(healthy_instances), 0)
healthy_instances = control.get_instances(states=HEALTHY_STATES, site='sierra')
self.assertEqual(len(healthy_instances), 2)
healthy_instances = control.get_instances(states=HEALTHY_STATES, site='hotel')
self.assertEqual(len(healthy_instances), 8)
def test_capacity_overflow(self):
control = MockControl()
state = MockState()
desired_n = 10
capacity = 5
hotel_cloud = make_cloud_conf('hotel', desired_n * 2, 1) # set hotel to have plenty of room
sierra_cloud = make_cloud_conf('sierra', desired_n, 2) # set hotel to have enough for the overflow
conf = make_conf([hotel_cloud, sierra_cloud, ], desired_n, 'testdt', 'm1.small')
de = PhantomMultiSiteOverflowEngine()
de.initialize(control, state, conf)
de.decide(control, control.get_state())
healthy_instances = control.get_instances(states=HEALTHY_STATES)
self.assertEqual(len(healthy_instances), desired_n)
for i in range(100):
de.decide(control, control.get_state())
skip_count = 0
for i in control.instances:
if i.site == 'hotel':
if skip_count >= capacity:
i.state = InstanceState.FAILED
skip_count = skip_count + 1
healthy_instances = control.get_instances(states=HEALTHY_STATES, site='hotel')
self.assertEqual(len(healthy_instances), capacity)
# at this point the DE should have figured out that we are at capacity so it should
# not decide to try to add more
de.decide(control, control.get_state())
healthy_instances = control.get_instances(states=HEALTHY_STATES, site='hotel')
self.assertEqual(len(healthy_instances), capacity)
sierra_healthy_instances = control.get_instances(states=HEALTHY_STATES, site='sierra')
self.assertEqual(len(sierra_healthy_instances), desired_n - capacity)
healthy_instances = control.get_instances(states=HEALTHY_STATES)
self.assertEqual(len(healthy_instances), desired_n)
def test_reconf_two_clouds_n_stays_the_same_but_node_dies(self):
control = MockControl()
state = MockState()
hotel_n = 4
sierra_n = 2
overall_n = hotel_n + sierra_n
hotel_cloud = make_cloud_conf('hotel', hotel_n, 1)
sierra_cloud = make_cloud_conf('sierra', sierra_n, 2)
conf = make_conf([hotel_cloud, sierra_cloud, ], overall_n, 'testdt', 'm1.small')
de = PhantomMultiSiteOverflowEngine()
de.initialize(control, state, conf)
de.decide(control, control.get_state())
self.assertEqual(control._launch_calls, overall_n)
self.assertEqual(control.site_launch_calls['hotel'], hotel_n)
self.assertEqual(control.site_launch_calls['sierra'], sierra_n)
hotel_cloud = make_cloud_conf('hotel', hotel_n + 5, 1)
sierra_cloud = make_cloud_conf('sierra', sierra_n, 2)
newconf = make_conf([hotel_cloud, sierra_cloud, ], overall_n, 'testdt', 'm1.small')
de.reconfigure(control, newconf)
de.decide(control, control.get_state())
# since we only rebalance optimistically nothing should change
self.assertEqual(control._launch_calls, overall_n)
self.assertEqual(control.site_launch_calls['hotel'], hotel_n)
self.assertEqual(control.site_launch_calls['sierra'], sierra_n)
for i in control.instances:
if i.site == 'sierra':
i.state = InstanceState.FAILED
break
# check to verify that we optimistically rebalanced
de.decide(control, control.get_state())
healthy_instances = control.get_instances(states=HEALTHY_STATES)
sierra_instances = control.get_instances(site='sierra', states=HEALTHY_STATES)
hotel_instances = control.get_instances(site='hotel', states=HEALTHY_STATES)
self.assertEqual(len(healthy_instances), overall_n)
self.assertEqual(len(hotel_instances), hotel_n + 1)
self.assertEqual(len(sierra_instances), sierra_n - 1)
def test_basic(self):
control = MockControl()
state = control.get_state()
minimum_n = 1
maximum_n = 3
metric = 'fake'
sample_function = Statistics.AVERAGE
scale_up_threshold = 2.0
scale_down_threshold = 0.5
scale_up_n_vms = 1
scale_down_n_vms = 1
config = self._get_config(minimum_n, maximum_n, metric, sample_function,
scale_up_threshold, scale_up_n_vms, scale_down_threshold,
scale_down_n_vms, "dt1")
sensor_series_up = [1, 3, 5]
sensor_average_up = sum(sensor_series_up) / len(sensor_series_up)
sensor_data_up = {metric: {Statistics.SERIES: sensor_series_up, Statistics.AVERAGE: sensor_average_up}}
sensor_series_down = [0, 1, 0]
sensor_average_down = float(sum(sensor_series_down)) / len(sensor_series_down)
sensor_data_down = {metric: {Statistics.SERIES: sensor_series_down, Statistics.AVERAGE: sensor_average_down}}
de = SensorEngine()
de.initialize(control, state, config)
de.decide(control, state)
self.assertEqual(control._launch_calls, 1)
state = control.get_state()
control.set_instance_sensor_data(sensor_data_up)
state = control.get_state()
de.decide(control, state)
self.assertEqual(control._launch_calls, 2)
control.set_instance_sensor_data(sensor_data_up)
state = control.get_state()
de.decide(control, state)
self.assertEqual(control._launch_calls, 3)
# We should see this stay at 3, since we hit max_instances
control.set_instance_sensor_data(sensor_data_up)
state = control.get_state()
de.decide(control, state)
self.assertEqual(control._launch_calls, 3)
# Now, change the sensor data to scale us down
control.set_instance_sensor_data(sensor_data_down)
state = control.get_state()
de.decide(control, state)
self.assertEqual(control._launch_calls, 3)
self.assertEqual(control._destroy_calls, 1)
control.set_instance_sensor_data(sensor_data_down)
state = control.get_state()
de.decide(control, state)
self.assertEqual(control._destroy_calls, 2)
# We are now at min, so we shouldn't see any more scaling
control.set_instance_sensor_data(sensor_data_down)
state = control.get_state()
de.decide(control, state)
self.assertEqual(control._destroy_calls, 2)
def test_basic_sensor(self):
control = MockControl()
state = MockState()
minimum_vms = 4
maximum_vms = 6
metric = "something"
sample_function = "Average"
scale_up_threshold = 2
scale_down_threshold = 0.5
scale_up_n_vms = 1
scale_down_n_vms = 1
cooldown_period = 0
cloud = make_cloud_conf('hotel', maximum_vms, 1)
conf = make_sensor_conf([cloud, ], minimum_vms, maximum_vms, metric,
sample_function, scale_up_threshold, scale_up_n_vms,
scale_down_threshold, scale_down_n_vms, 'testdt', 'm1.small',
cooldown_period)
sensor_series_up = [1, 3, 5]
sensor_average_up = sum(sensor_series_up) / len(sensor_series_up)
sensor_data_up = {metric: {Statistics.SERIES: sensor_series_up, Statistics.AVERAGE: sensor_average_up}}
sensor_series_down = [0, 1, 0]
sensor_average_down = float(sum(sensor_series_down)) / len(sensor_series_down)
sensor_data_down = {metric: {Statistics.SERIES: sensor_series_down, Statistics.AVERAGE: sensor_average_down}}
de = PhantomMultiSiteOverflowEngine()
de.initialize(control, state, conf)
# after a start, we should see the DE start the minimum number of VMs
state = control.get_state()
de.decide(control, state)
self.assertEqual(control._launch_calls, minimum_vms)
previous_launch_calls = control._launch_calls
# Now we add some sensor data, and we should see a scale up until we get to max VMs (n=5)
control.set_instance_sensor_data(sensor_data_up)
state = control.get_state()
de.decide(control, state)
self.assertEqual(control._launch_calls, previous_launch_calls + scale_up_n_vms)
previous_launch_calls = control._launch_calls
# We should see one more (n=6)
control.set_instance_sensor_data(sensor_data_up)
state = control.get_state()
de.decide(control, state)
self.assertEqual(control._launch_calls, previous_launch_calls + scale_up_n_vms)
# Now we should be at max vms, and see no more starts
control.set_instance_sensor_data(sensor_data_up)
state = control.get_state()
de.decide(control, state)
self.assertEqual(control._launch_calls, maximum_vms)
# Now we scale down with some vms die
control.set_instance_sensor_data(sensor_data_down)
state = control.get_state()
de.decide(control, state)
self.assertEqual(control._launch_calls, maximum_vms)
self.assertEqual(control._destroy_calls, 1)
# and another one
control.set_instance_sensor_data(sensor_data_down)
state = control.get_state()
de.decide(control, state)
self.assertEqual(control._launch_calls, maximum_vms)
self.assertEqual(control._destroy_calls, 2)
# and we should now be at minimum vms
control.set_instance_sensor_data(sensor_data_down)
state = control.get_state()
de.decide(control, state)
self.assertEqual(control._launch_calls - control._destroy_calls, minimum_vms)
