def test_sort_by_health_system_instance_health_system_status_failed(): mock_slave_1 = Mock(name="slave1") mock_slave_1.task_counts = SlaveTaskCount(count=3, slave=Mock(), batch_count=1) mock_slave_1.instance_status = { "Events": [ { "Code": "instance-reboot", "Description": "string", "NotBefore": datetime(2015, 1, 1), "NotAfter": datetime(2015, 1, 1), } ], "SystemStatus": {"Status": "impaired"}, "InstanceStatus": {"Status": "ok"}, } mock_slave_2 = Mock(name="slave2") mock_slave_2.task_counts = (SlaveTaskCount(count=3, slave=Mock(), batch_count=1),) mock_slave_2.instance_status = { "Events": [ { "Code": "instance-reboot", "Description": "string", "NotBefore": datetime(2015, 1, 1), "NotAfter": datetime(2015, 1, 1), } ], "SystemStatus": {"Status": "ok"}, "InstanceStatus": {"Status": "ok"}, } ret = ec2_fitness.sort_by_system_instance_health([mock_slave_1, mock_slave_2]) assert ret == [mock_slave_2, mock_slave_1]
def test_sort_by_running_batch_count(): mock_slave_1 = Mock( task_counts=SlaveTaskCount(count=3, slave=Mock(), chronos_count=1)) mock_slave_2 = Mock( task_counts=SlaveTaskCount(count=2, slave=Mock(), chronos_count=2)) mock_slave_3 = Mock( task_counts=SlaveTaskCount(count=5, slave=Mock(), chronos_count=3)) ret = ec2_fitness.sort_by_running_batch_count( [mock_slave_1, mock_slave_2, mock_slave_3]) assert ret == [mock_slave_3, mock_slave_2, mock_slave_1]
def test_sort_by_total_tasks(): mock_slave_1 = Mock( task_counts=SlaveTaskCount(count=3, slave=Mock(), chronos_count=0)) mock_slave_2 = Mock( task_counts=SlaveTaskCount(count=2, slave=Mock(), chronos_count=1)) mock_slave_3 = Mock( task_counts=SlaveTaskCount(count=5, slave=Mock(), chronos_count=0)) ret = ec2_fitness.sort_by_total_tasks( [mock_slave_1, mock_slave_2, mock_slave_3]) assert ret == [mock_slave_3, mock_slave_1, mock_slave_2]
def test_sort_by_fitness(): mock_slave_1 = Mock(name="slave1") mock_slave_1.task_counts = SlaveTaskCount(count=3, slave=Mock(), batch_count=1) mock_slave_1.instance_status = { "Events": [], "SystemStatus": {"Status": "impaired"}, "InstanceStatus": {"Status": "ok"}, } mock_slave_2 = Mock(name="slave2") mock_slave_2.task_counts = SlaveTaskCount(count=3, slave=Mock(), batch_count=1) mock_slave_2.instance_status = { "Events": [ { "Code": "instance-reboot", "Description": "foo", "NotBefore": datetime(2015, 1, 1), "NotAfter": datetime(2015, 1, 1), } ], "SystemStatus": {"Status": "ok"}, "InstanceStatus": {"Status": "ok"}, } mock_slave_3 = Mock(name="slave3") mock_slave_3.task_counts = SlaveTaskCount(count=2, slave=Mock(), batch_count=3) mock_slave_3.instance_status = { "Events": [], "SystemStatus": {"Status": "ok"}, "InstanceStatus": {"Status": "ok"}, } mock_slave_4 = Mock(name="slave4") mock_slave_4.task_counts = SlaveTaskCount(count=3, slave=Mock(), batch_count=1) mock_slave_4.instance_status = { "Events": [], "SystemStatus": {"Status": "ok"}, "InstanceStatus": {"Status": "ok"}, } mock_slave_5 = Mock(name="slave5") mock_slave_5.task_counts = SlaveTaskCount(count=1, slave=Mock(), batch_count=1) mock_slave_5.instance_status = { "Events": [], "SystemStatus": {"Status": "ok"}, "InstanceStatus": {"Status": "ok"}, } ret = ec2_fitness.sort_by_ec2_fitness( [mock_slave_1, mock_slave_2, mock_slave_3, mock_slave_4, mock_slave_5] ) # we expect this order for the following reason: # mock_slave_1 is impaired and so should be killed asap # mock_slave_2 has an upcoming event # mock_slave_5 and mock_slave_4 have the fewest batch tasks, and so should be killed before # mock_slave_3 (we cant drain batch tasks, so try and save them) # mock_slave_5 has fewer tasks than mock_slave_4, and so is a better candidate for killing assert ret == [mock_slave_3, mock_slave_4, mock_slave_5, mock_slave_2, mock_slave_1]
def test_sort_by_health_system_instance_health_system_status_failed(): mock_slave_1 = Mock( task_counts=SlaveTaskCount( count=3, slave=Mock(), chronos_count=1, ), instance_status={ 'Events': [ { 'Code': 'instance-reboot', 'Description': 'string', 'NotBefore': datetime(2015, 1, 1), 'NotAfter': datetime(2015, 1, 1) }, ], 'SystemStatus': { 'Status': 'impaired', }, 'InstanceStatus': { 'Status': 'ok', } }, ) mock_slave_2 = Mock( task_counts=SlaveTaskCount( count=3, slave=Mock(), chronos_count=1, ), instance_status={ 'Events': [ { 'Code': 'instance-reboot', 'Description': 'string', 'NotBefore': datetime(2015, 1, 1), 'NotAfter': datetime(2015, 1, 1) }, ], 'SystemStatus': { 'Status': 'ok', }, 'InstanceStatus': { 'Status': 'ok', } }, ) mock_slave_2 = Mock(task_counts=SlaveTaskCount(count=2, slave=Mock(), chronos_count=2)) ret = ec2_fitness.sort_by_running_batch_count([mock_slave_1, mock_slave_2]) assert ret == [mock_slave_1, mock_slave_2]
def test_sort_by_health_system_instance_health_system_status_failed(): mock_slave_1 = Mock(name='slave1') mock_slave_1.task_counts = SlaveTaskCount( count=3, slave=Mock(), batch_count=1, ) mock_slave_1.instance_status = { 'Events': [ { 'Code': 'instance-reboot', 'Description': 'string', 'NotBefore': datetime(2015, 1, 1), 'NotAfter': datetime(2015, 1, 1), }, ], 'SystemStatus': { 'Status': 'impaired', }, 'InstanceStatus': { 'Status': 'ok', }, } mock_slave_2 = Mock(name='slave2') mock_slave_2.task_counts = SlaveTaskCount( count=3, slave=Mock(), batch_count=1, ), mock_slave_2.instance_status = { 'Events': [ { 'Code': 'instance-reboot', 'Description': 'string', 'NotBefore': datetime(2015, 1, 1), 'NotAfter': datetime(2015, 1, 1), }, ], 'SystemStatus': { 'Status': 'ok', }, 'InstanceStatus': { 'Status': 'ok', }, } ret = ec2_fitness.sort_by_system_instance_health([mock_slave_1, mock_slave_2]) assert ret == [mock_slave_2, mock_slave_1]
def test_sort_by_upcoming_events(): mock_slave_1 = Mock( task_counts=SlaveTaskCount( count=3, slave=Mock(), chronos_count=1, ), instance_status={ 'Events': [], 'SystemStatus': { 'Status': 'ok', }, 'InstanceStatus': { 'Status': 'ok', } }, ) mock_slave_2 = Mock( task_counts=SlaveTaskCount( count=3, slave=Mock(), chronos_count=1, ), instance_status={ 'Events': [ { 'Code': 'instance-reboot', 'Description': 'string', 'NotBefore': datetime(2015, 1, 1), 'NotAfter': datetime(2015, 1, 1) }, ], 'SystemStatus': { 'Status': 'ok', }, 'InstanceStatus': { 'Status': 'ok', } }, ) ret = ec2_fitness.sort_by_upcoming_events([mock_slave_1, mock_slave_2]) assert ret == [mock_slave_1, mock_slave_2]
def test_sort_slaves_to_kill(): with contextlib.nested( mock.patch('paasta_tools.autoscaling_lib.get_mesos_task_count_by_slave', autospec=True), ) as ( mock_get_task_count, ): # test no slaves mock_mesos_state = mock.Mock() ret = autoscaling_lib.sort_slaves_to_kill(mock_mesos_state) assert ret == [] mock_get_task_count.return_value = {} mock_slave_1 = mock.Mock() mock_slave_2 = mock.Mock() mock_slave_3 = mock.Mock() mock_task_count = {'pid1': SlaveTaskCount(count=3, slave=mock_slave_1, chronos_count=0), 'pid2': SlaveTaskCount(count=2, slave=mock_slave_2, chronos_count=1), 'pid3': SlaveTaskCount(count=5, slave=mock_slave_3, chronos_count=0)} mock_get_task_count.return_value = mock_task_count ret = autoscaling_lib.sort_slaves_to_kill(mock_mesos_state) mock_get_task_count.assert_called_with(mock_mesos_state, pool='default') assert ret == [mock_slave_1, mock_slave_3, mock_slave_2]
def test_sort_by_upcoming_events(): mock_slave_1 = Mock() mock_slave_1.task_counts = SlaveTaskCount(count=3, slave=Mock(), batch_count=1) mock_slave_1.instance_status = { "Events": [], "SystemStatus": {"Status": "ok"}, "InstanceStatus": {"Status": "ok"}, } mock_slave_2 = Mock() mock_slave_2.task_counts = SlaveTaskCount(count=3, slave=Mock(), batch_count=1) mock_slave_2.instance_status = { "Events": [ { "Code": "instance-reboot", "Description": "string", "NotBefore": datetime(2015, 1, 1), "NotAfter": datetime(2015, 1, 1), } ], "SystemStatus": {"Status": "ok"}, "InstanceStatus": {"Status": "ok"}, } ret = ec2_fitness.sort_by_upcoming_events([mock_slave_1, mock_slave_2]) assert ret == [mock_slave_1, mock_slave_2]
def test_sort_by_fitness(): mock_slave_1 = Mock(name='slave1') mock_slave_1.task_counts = SlaveTaskCount( count=3, slave=Mock(), batch_count=1, ) mock_slave_1.instance_status = { 'Events': [], 'SystemStatus': {'Status': 'impaired', }, 'InstanceStatus': {'Status': 'ok', }, } mock_slave_2 = Mock(name='slave2') mock_slave_2.task_counts = SlaveTaskCount( count=3, slave=Mock(), batch_count=1, ) mock_slave_2.instance_status = { 'Events': [ { 'Code': 'instance-reboot', 'Description': 'foo', 'NotBefore': datetime(2015, 1, 1), 'NotAfter': datetime(2015, 1, 1), }, ], 'SystemStatus': {'Status': 'ok', }, 'InstanceStatus': {'Status': 'ok', }, } mock_slave_3 = Mock(name='slave3') mock_slave_3.task_counts = SlaveTaskCount( count=2, slave=Mock(), batch_count=3, ) mock_slave_3.instance_status = { 'Events': [], 'SystemStatus': {'Status': 'ok', }, 'InstanceStatus': {'Status': 'ok', }, } mock_slave_4 = Mock(name='slave4') mock_slave_4.task_counts = SlaveTaskCount( count=3, slave=Mock(), batch_count=1, ) mock_slave_4.instance_status = { 'Events': [], 'SystemStatus': {'Status': 'ok', }, 'InstanceStatus': {'Status': 'ok', }, } mock_slave_5 = Mock(name='slave5') mock_slave_5.task_counts = SlaveTaskCount( count=1, slave=Mock(), batch_count=1, ) mock_slave_5.instance_status = { 'Events': [], 'SystemStatus': {'Status': 'ok', }, 'InstanceStatus': {'Status': 'ok', }, } ret = ec2_fitness.sort_by_ec2_fitness([mock_slave_1, mock_slave_2, mock_slave_3, mock_slave_4, mock_slave_5]) # we expect this order for the following reason: # mock_slave_1 is impaired and so should be killed asap # mock_slave_2 has an upcoming event # mock_slave_5 and mock_slave_4 have the fewest chronos tasks, and so should be killed before # mock_slave_3 (we cant drain chronos tasks, so try and save them) # mock_slave_5 has fewer tasks than mock_slave_4, and so is a better candidate for killing assert ret == [mock_slave_3, mock_slave_4, mock_slave_5, mock_slave_2, mock_slave_1]
def test_sort_by_fitness(): mock_slave_1 = Mock( task_counts=SlaveTaskCount( count=3, slave=Mock(), chronos_count=1, ), instance_status={ 'Events': [], 'SystemStatus': {'Status': 'impaired', }, 'InstanceStatus': {'Status': 'ok', } }, ) mock_slave_2 = Mock( task_counts=SlaveTaskCount( count=3, slave=Mock(), chronos_count=1, ), instance_status={ 'Events': [ { 'Code': 'instance-reboot', 'Description': 'foo', 'NotBefore': datetime(2015, 1, 1), 'NotAfter': datetime(2015, 1, 1) }, ], 'SystemStatus': {'Status': 'ok', }, 'InstanceStatus': {'Status': 'ok', } }, ) mock_slave_3 = Mock( task_counts=SlaveTaskCount( count=2, slave=Mock(), chronos_count=3, ), instance_status={ 'Events': [], 'SystemStatus': {'Status': 'ok', }, 'InstanceStatus': {'Status': 'ok', } }, ) mock_slave_4 = Mock( task_counts=SlaveTaskCount( count=3, slave=Mock(), chronos_count=1, ), instance_status={ 'Events': [], 'SystemStatus': {'Status': 'ok', }, 'InstanceStatus': {'Status': 'ok', } }, ) mock_slave_5 = Mock( task_counts=SlaveTaskCount( count=1, slave=Mock(), chronos_count=1, ), instance_status={ 'Events': [], 'SystemStatus': {'Status': 'ok', }, 'InstanceStatus': {'Status': 'ok', } }, ) ret = ec2_fitness.sort_by_ec2_fitness([mock_slave_1, mock_slave_2, mock_slave_3, mock_slave_4, mock_slave_5]) assert ret == [mock_slave_5, mock_slave_4, mock_slave_3, mock_slave_2, mock_slave_1]