def test_scale_down(self): """Test scale down.""" autoscale._state.return_value = autoscale._STATE( running=10, pending=1, busy_srv_cnt=5, idle_servers=['a', 'b'], ) self.assertEqual((0, ['a']), autoscale.scale(min_servers=0, max_servers=200, default_app_srv_ratio=0.5, max_batch=10)) autoscale._state.return_value = autoscale._STATE( running=10, pending=1, busy_srv_cnt=5, idle_servers=['a'], ) self.assertEqual((0, []), autoscale.scale(min_servers=0, max_servers=200, default_app_srv_ratio=0.5, max_batch=10)) autoscale._state.return_value = autoscale._STATE( running=10, pending=0, busy_srv_cnt=5, idle_servers=['a', 'b', 'c'], ) self.assertEqual((0, ['a', 'b', 'c']), autoscale.scale(min_servers=0, max_servers=200, default_app_srv_ratio=0.5, max_batch=10)) # There are 5 busy servers, so with 5 min servers, all idle will be # removed. self.assertEqual((0, ['a', 'b', 'c']), autoscale.scale(min_servers=5, max_servers=200, default_app_srv_ratio=0.5, max_batch=10)) # There are 5 busy servers, so with 6 min servers, two idle will be # removed. self.assertEqual((0, ['b', 'c']), autoscale.scale(min_servers=6, max_servers=200, default_app_srv_ratio=0.5, max_batch=10))
def autoscale_cmd(interval, server_app_ratio, workers): """Autoscale Treadmill cell based on scheduler queue.""" pool = None if workers: pool = multiprocessing.Pool(processes=workers) pool.workers = workers context.GLOBAL.zk.add_listener(zkutils.exit_on_lost) while True: autoscale.scale(server_app_ratio, pool=pool) time.sleep(interval)
def test_scale_up_no_idle(self): """Test scale up.""" autoscale._state.return_value = autoscale._STATE( running=10, pending=100, busy_srv_cnt=5, idle_servers=[], ) self.assertEqual((10, []), autoscale.scale(min_servers=0, max_servers=200, default_app_srv_ratio=0.5, max_batch=10)) self.assertEqual((1, []), autoscale.scale(min_servers=0, max_servers=6, default_app_srv_ratio=0.5, max_batch=10)) self.assertEqual((0, []), autoscale.scale(min_servers=0, max_servers=5, default_app_srv_ratio=0.5, max_batch=10)) autoscale._state.return_value = autoscale._STATE( running=10, pending=2, busy_srv_cnt=5, idle_servers=[], ) self.assertEqual((1, []), autoscale.scale(min_servers=0, max_servers=200, default_app_srv_ratio=0.5, max_batch=10)) autoscale._state.return_value = autoscale._STATE( running=1000, pending=1, busy_srv_cnt=10, idle_servers=[], ) self.assertEqual((1, []), autoscale.scale(min_servers=0, max_servers=2000, default_app_srv_ratio=0.5, max_batch=10))
def test_scale_up_with_idle(self): """Test scale up with some idle servers present.""" autoscale._state.return_value = autoscale._STATE( running=10, pending=100, busy_srv_cnt=5, idle_servers=['a', 'b'], ) self.assertEqual((10, []), autoscale.scale(min_servers=0, max_servers=200, default_app_srv_ratio=0.5, max_batch=10)) # Two apps, two server idle, with current ratio .5, will not ask for # new servers. autoscale._state.return_value = autoscale._STATE( running=10, pending=2, busy_srv_cnt=5, idle_servers=['a', 'b'], ) self.assertEqual((0, []), autoscale.scale(min_servers=0, max_servers=200, default_app_srv_ratio=0.5, max_batch=10)) # 6 pending apps, two server idle, with current ratio .5. expect # additional 1 server. autoscale._state.return_value = autoscale._STATE( running=10, pending=6, busy_srv_cnt=5, idle_servers=['a', 'b'], ) self.assertEqual((1, []), autoscale.scale(min_servers=0, max_servers=200, default_app_srv_ratio=0.5, max_batch=10))
def autoscale_cmd(timeout, max_count, min_count, batch_count, app_srv_ratio): """Autoscale Treadmill cell based on scheduler queue.""" while True: create_cnt, extra_servers = autoscale.scale( max_servers=max_count, min_servers=min_count, default_app_srv_ratio=app_srv_ratio, max_batch=batch_count) if create_cnt > 0: autoscale.create_n_servers(create_cnt, partition=None) if extra_servers: autoscale.delete_servers_by_name(extra_servers) time.sleep(timeout)
def test_scale_up_no_idle(self, admin_mock, stateapi_mock): """Test scaling up with no idle servers present.""" mock_zkclient = context.GLOBAL.zk.conn mock_zkclient.get_children.return_value = [] # Ratio: 0.5 # Pending apps: 3, no servers - create 2 servers. _mock_cell( admin_mock, stateapi_mock, partitions=[ {'_id': 'partition', 'data': {'autoscale': {'min_servers': 1, 'max_servers': 9}}}, ], servers=[], servers_state=[], apps_state=[ ('proid.app#001', 'partition', None), ('proid.app#002', 'partition', None), ('proid.app#003', 'partition', None), ], ) autoscale.scale(0.5, 0) autoscale.create_n_servers.assert_called_once_with( 2, 'partition', pool=None ) autoscale.delete_servers_by_name.assert_not_called() autoscale.create_n_servers.reset_mock() autoscale.delete_servers_by_name.reset_mock() # Ratio: 0.5 # Pending apps: 100, no servers, max servers: 9 - create 9 severs. _mock_cell( admin_mock, stateapi_mock, partitions=[ {'_id': 'partition', 'data': {'autoscale': {'min_servers': 1, 'max_servers': 9}}}, ], servers=[], servers_state=[], apps_state=[ ('proid.app#%03d' % i, 'partition', None) for i in range(100) ], ) autoscale.scale(0.5, 0) autoscale.create_n_servers.assert_called_once_with( 9, 'partition', pool=None ) autoscale.delete_servers_by_name.assert_not_called() autoscale.create_n_servers.reset_mock() autoscale.delete_servers_by_name.reset_mock() # Ratio: 1.0 (down and frozen servers excluded). # Pending apps: 3, no idle servers - create 3 servers. # Down and frozen servers have apps placed on them - don't delete. _mock_cell( admin_mock, stateapi_mock, partitions=[ {'_id': 'partition', 'data': {'autoscale': {'min_servers': 1, 'max_servers': 9}}}, ], servers=[ {'_id': 'server1', 'partition': 'partition', '_create_timestamp': 100.0}, {'_id': 'server2', 'partition': 'partition', '_create_timestamp': 100.0}, {'_id': 'server3', 'partition': 'partition', '_create_timestamp': 100.0}, ], servers_state=[ ('server1', 'up', 100, 100, 100), ('server2', 'down', 100, 100, 100), ('server3', 'frozen', 100, 100, 100), ], apps_state=[ ('proid.app#001', 'partition', 'server1'), ('proid.app#002', 'partition', 'server2'), ('proid.app#003', 'partition', 'server3'), ('proid.app#004', 'partition', 'server3'), ('proid.app#005', 'partition', 'server3'), ('proid.app#006', 'partition', 'server3'), ('proid.app#007', 'partition', None), ('proid.app#008', 'partition', None), ('proid.app#009', 'partition', None), ], ) autoscale.scale(0.5, 0) autoscale.create_n_servers.assert_called_once_with( 3, 'partition', pool=None ) autoscale.delete_servers_by_name.assert_not_called()
def test_scale_down(self, admin_mock, stateapi_mock): """Test scaling down.""" mock_zkclient = context.GLOBAL.zk.conn mock_zkclient.get_children.return_value = [] # Pending apps: 1, idle servers: 2 - delete 1 server. _mock_cell( admin_mock, stateapi_mock, partitions=[ {'_id': 'partition', 'data': {'autoscale': {'min_servers': 3, 'max_servers': 9}}}, ], servers=[ {'_id': 'server1', 'partition': 'partition', '_create_timestamp': 100.0}, {'_id': 'server2', 'partition': 'partition', '_create_timestamp': 100.0}, {'_id': 'server3', 'partition': 'partition', '_create_timestamp': 100.0}, {'_id': 'server4', 'partition': 'partition', '_create_timestamp': 100.0}, {'_id': 'server5', 'partition': 'partition', '_create_timestamp': 100.0}, ], servers_state=[ ('server1', 'up', 100, 100, 100), ('server2', 'up', 100, 100, 100), ('server3', 'up', 100, 100, 100), ('server4', 'up', 100, 100, 100), ('server5', 'up', 100, 100, 100), ], apps_state=[ ('proid.app#001', 'partition', 'server1'), ('proid.app#002', 'partition', 'server2'), ('proid.app#003', 'partition', 'server3'), ('proid.app#004', 'partition', None), ], ) autoscale.scale(0.5, 0) autoscale.create_n_servers.assert_not_called() autoscale.delete_servers_by_name.assert_called_once_with( ['server4'], pool=None ) autoscale.create_n_servers.reset_mock() autoscale.delete_servers_by_name.reset_mock() # No pending apps, idle servers: 5, min servers: 3 - delete 2 servers. _mock_cell( admin_mock, stateapi_mock, partitions=[ {'_id': 'partition', 'data': {'autoscale': {'min_servers': 3, 'max_servers': 9}}}, ], servers=[ {'_id': 'server1', 'partition': 'partition', '_create_timestamp': 100.0}, {'_id': 'server2', 'partition': 'partition', '_create_timestamp': 100.0}, {'_id': 'server3', 'partition': 'partition', '_create_timestamp': 100.0}, {'_id': 'server4', 'partition': 'partition', '_create_timestamp': 100.0}, {'_id': 'server5', 'partition': 'partition', '_create_timestamp': 100.0}, ], servers_state=[ ('server1', 'up', 100, 100, 100), ('server2', 'up', 100, 100, 100), ('server3', 'up', 100, 100, 100), ('server4', 'up', 100, 100, 100), ('server5', 'up', 100, 100, 100), ], apps_state=[], ) autoscale.scale(0.5, 0) autoscale.create_n_servers.assert_not_called() autoscale.delete_servers_by_name.assert_called_once_with( ['server1', 'server2'], pool=None ) autoscale.create_n_servers.reset_mock() autoscale.delete_servers_by_name.reset_mock() # Delete empty down server, don't delete blackedout and frozen servers. _mock_cell( admin_mock, stateapi_mock, partitions=[ {'_id': 'partition', 'data': {'autoscale': {'min_servers': 3, 'max_servers': 9}}}, ], servers=[ {'_id': 'server1', 'partition': 'partition', '_create_timestamp': 100.0}, {'_id': 'server2', 'partition': 'partition', '_create_timestamp': 100.0}, {'_id': 'server3', 'partition': 'partition', '_create_timestamp': 100.0}, {'_id': 'server4', 'partition': 'partition', '_create_timestamp': 100.0}, {'_id': 'server5', 'partition': 'partition', '_create_timestamp': 100.0}, {'_id': 'server6', 'partition': 'partition', '_create_timestamp': 100.0}, {'_id': 'server7', 'partition': 'partition', '_create_timestamp': 100.0}, ], servers_state=[ ('server1', 'up', 100, 100, 100), ('server2', 'up', 100, 100, 100), ('server3', 'up', 100, 100, 100), ('server4', 'down', 100, 100, 100), ('server5', 'down', 100, 100, 100), ('server6', 'down', 100, 100, 100), ('server7', 'frozen', 100, 100, 100), ], apps_state=[ ('proid.app#001', 'partition', 'server6'), ], ) mock_zkclient.get_children.return_value = ['server5'] autoscale.scale(0.5, 0) autoscale.create_n_servers.assert_not_called() autoscale.delete_servers_by_name.assert_called_once_with( ['server4'], pool=None ) autoscale.create_n_servers.reset_mock() autoscale.delete_servers_by_name.reset_mock() # Delete idle servers when grace period expires. idle_servers_tracker = collections.defaultdict(dict) _mock_cell( admin_mock, stateapi_mock, partitions=[ {'_id': 'partition', 'data': {'autoscale': {'min_servers': 0, 'max_servers': 9}}}, ], servers=[ {'_id': 'server1', 'partition': 'partition', '_create_timestamp': 100.0}, {'_id': 'server2', 'partition': 'partition', '_create_timestamp': 100.0}, {'_id': 'server3', 'partition': 'partition', '_create_timestamp': 100.0}, ], servers_state=[ ('server1', 'up', 100, 100, 100), ('server2', 'up', 100, 100, 100), ('server3', 'up', 100, 100, 100), ], apps_state=[ ('proid.app#001', 'partition', 'server1'), ], ) idle_servers_tracker = autoscale.scale(0.5, 5 * 60) print(idle_servers_tracker) autoscale.create_n_servers.assert_not_called() autoscale.delete_servers_by_name.assert_not_called() self.assertEqual( idle_servers_tracker['partition'], {'server2': 1000.0, 'server3': 1000.0} ) time.time.return_value = 1000.0 + (5 * 60) + 1 # server2 is no longer idle, server3 continues to be idle. _mock_cell( admin_mock, stateapi_mock, partitions=[ {'_id': 'partition', 'data': {'autoscale': {'min_servers': 0, 'max_servers': 9}}}, ], servers=[ {'_id': 'server1', 'partition': 'partition', '_create_timestamp': 100.0}, {'_id': 'server2', 'partition': 'partition', '_create_timestamp': 100.0}, {'_id': 'server3', 'partition': 'partition', '_create_timestamp': 100.0}, ], servers_state=[ ('server1', 'up', 100, 100, 100), ('server2', 'up', 100, 100, 100), ('server3', 'up', 100, 100, 100), ], apps_state=[ ('proid.app#001', 'partition', 'server1'), ('proid.app#002', 'partition', 'server2'), ], ) idle_servers_tracker = autoscale.scale( 0.5, 5 * 60, idle_servers_tracker=idle_servers_tracker ) print(idle_servers_tracker) autoscale.create_n_servers.assert_not_called() autoscale.delete_servers_by_name.assert_called_once_with( ['server3'], pool=None ) self.assertEqual( idle_servers_tracker['partition'], {'server3': 1000.0} )
def test_scale_up_min_servers(self, admin_mock, stateapi_mock): """Test scaling up to min (active) servers.""" mock_zkclient = context.GLOBAL.zk.conn mock_zkclient.get_children.return_value = [] # Empty partition, min servers: 3 - create 3 servers. _mock_cell( admin_mock, stateapi_mock, partitions=[ {'_id': 'partition', 'data': {'autoscale': {'min_servers': 3, 'max_servers': 9}}}, ], servers=[], servers_state=[], apps_state=[], ) autoscale.scale(0.5, 0) autoscale.create_n_servers.assert_called_once_with( 3, 'partition', pool=None ) autoscale.delete_servers_by_name.assert_not_called() autoscale.create_n_servers.reset_mock() autoscale.delete_servers_by_name.reset_mock() # 1 up, 1 down and 1 frozen server, min servers: 3 - create 2 servers. # Down and frozen servers have apps placed on them - don't delete. _mock_cell( admin_mock, stateapi_mock, partitions=[ {'_id': 'partition', 'data': {'autoscale': {'min_servers': 3, 'max_servers': 9}}}, ], servers=[ {'_id': 'server1', 'partition': 'partition', '_create_timestamp': 100.0}, {'_id': 'server2', 'partition': 'partition', '_create_timestamp': 100.0}, {'_id': 'server3', 'partition': 'partition', '_create_timestamp': 100.0}, ], servers_state=[ ('server1', 'up', 100, 100, 100), ('server2', 'down', 100, 100, 100), ('server3', 'frozen', 100, 100, 100), ], apps_state=[ ('proid.app#001', 'partition', 'server1'), ('proid.app#002', 'partition', 'server2'), ('proid.app#003', 'partition', 'server3'), ], ) autoscale.scale(0.5, 0) autoscale.create_n_servers.assert_called_once_with( 2, 'partition', pool=None ) autoscale.delete_servers_by_name.assert_not_called()
def test_scale_up_with_idle(self, admin_mock, stateapi_mock): """Test scaling up with some idle servers present.""" mock_zkclient = context.GLOBAL.zk.conn mock_zkclient.get_children.return_value = [] # Ratio: 0.5 # Pending apps: 1, idle servers: 1 - don't create anything. _mock_cell( admin_mock, stateapi_mock, partitions=[ {'_id': 'partition', 'data': {'autoscale': {'min_servers': 1, 'max_servers': 9}}}, ], servers=[ {'_id': 'server1', 'partition': 'partition', '_create_timestamp': 100.0}, ], servers_state=[ ('server1', 'up', 100, 100, 100), ], apps_state=[ ('proid.app#001', 'partition', None), ], ) autoscale.scale(0.5, 0) autoscale.create_n_servers.assert_not_called() autoscale.delete_servers_by_name.assert_not_called() autoscale.create_n_servers.reset_mock() autoscale.delete_servers_by_name.reset_mock() # Ratio: 0.5 # Pending apps: 3, idle servers: 2 - don't create anything. _mock_cell( admin_mock, stateapi_mock, partitions=[ {'_id': 'partition', 'data': {'autoscale': {'min_servers': 1, 'max_servers': 9}}}, ], servers=[ {'_id': 'server1', 'partition': 'partition', '_create_timestamp': 100.0}, {'_id': 'server2', 'partition': 'partition', '_create_timestamp': 100.0}, ], servers_state=[ ('server1', 'up', 100, 100, 100), ('server2', 'up', 100, 100, 100), ], apps_state=[ ('proid.app#001', 'partition', None), ('proid.app#002', 'partition', None), ('proid.app#003', 'partition', None), ], ) autoscale.scale(0.5, 0) autoscale.create_n_servers.assert_not_called() autoscale.delete_servers_by_name.assert_not_called() autoscale.create_n_servers.reset_mock() autoscale.delete_servers_by_name.reset_mock() # Ratio: 1.0. # Pending apps: 3, idle servers: 2 - create 1 server. _mock_cell( admin_mock, stateapi_mock, partitions=[ {'_id': 'partition', 'data': {'autoscale': {'min_servers': 1, 'max_servers': 9}}}, ], servers=[ {'_id': 'server1', 'partition': 'partition', '_create_timestamp': 100.0}, {'_id': 'server2', 'partition': 'partition', '_create_timestamp': 100.0}, {'_id': 'server3', 'partition': 'partition', '_create_timestamp': 999.0}, {'_id': 'server4', 'partition': 'partition', '_create_timestamp': 100.0}, ], servers_state=[ ('server1', 'up', 100, 100, 100), ('server2', 'up', 100, 100, 100), ('server3', 'down', None, None, None), # Didn't report - new. ('server4', 'down', None, None, None), # Didn't report - down. ], apps_state=[ ('proid.app#001', 'partition', 'server1'), ('proid.app#002', 'partition', None), ('proid.app#003', 'partition', None), ('proid.app#004', 'partition', None), ], ) autoscale.scale(0.5, 0) autoscale.create_n_servers.assert_called_once_with( 1, 'partition', pool=None ) autoscale.delete_servers_by_name.assert_called_once_with( ['server4'], pool=None )
def autoscale_cmd(interval, server_app_ratio): """Autoscale Treadmill cell based on scheduler queue.""" while True: autoscale.scale(server_app_ratio) time.sleep(interval)
def test_scale_down(self, admin_mock, stateapi_mock): """Test scaling down.""" mock_zkclient = context.GLOBAL.zk.conn mock_zkclient.get_children.return_value = [] # Pending apps: 1, idle servers: 2 - delete 1 server. _mock_cell( admin_mock, stateapi_mock, partitions=[ { '_id': 'partition', 'data': { 'autoscale': { 'min_servers': 3, 'max_servers': 9 } } }, ], servers=[ { '_id': 'server1', 'partition': 'partition', '_create_timestamp': 100.0 }, { '_id': 'server2', 'partition': 'partition', '_create_timestamp': 100.0 }, { '_id': 'server3', 'partition': 'partition', '_create_timestamp': 100.0 }, { '_id': 'server4', 'partition': 'partition', '_create_timestamp': 100.0 }, { '_id': 'server5', 'partition': 'partition', '_create_timestamp': 100.0 }, ], servers_state=[ ('server1', 'up', 100, 100, 100), ('server2', 'up', 100, 100, 100), ('server3', 'up', 100, 100, 100), ('server4', 'up', 100, 100, 100), ('server5', 'up', 100, 100, 100), ], apps_state=[ ('proid.app#001', 'partition', 'server1'), ('proid.app#002', 'partition', 'server2'), ('proid.app#003', 'partition', 'server3'), ('proid.app#004', 'partition', None), ], ) autoscale.scale(0.5) autoscale.create_n_servers.assert_not_called() autoscale.delete_servers_by_name.assert_called_once_with(['server4']) autoscale.create_n_servers.reset_mock() autoscale.delete_servers_by_name.reset_mock() # No pending apps, idle servers: 5, min servers: 3 - delete 2 servers. _mock_cell( admin_mock, stateapi_mock, partitions=[ { '_id': 'partition', 'data': { 'autoscale': { 'min_servers': 3, 'max_servers': 9 } } }, ], servers=[ { '_id': 'server1', 'partition': 'partition', '_create_timestamp': 100.0 }, { '_id': 'server2', 'partition': 'partition', '_create_timestamp': 100.0 }, { '_id': 'server3', 'partition': 'partition', '_create_timestamp': 100.0 }, { '_id': 'server4', 'partition': 'partition', '_create_timestamp': 100.0 }, { '_id': 'server5', 'partition': 'partition', '_create_timestamp': 100.0 }, ], servers_state=[ ('server1', 'up', 100, 100, 100), ('server2', 'up', 100, 100, 100), ('server3', 'up', 100, 100, 100), ('server4', 'up', 100, 100, 100), ('server5', 'up', 100, 100, 100), ], apps_state=[], ) autoscale.scale(0.5) autoscale.create_n_servers.assert_not_called() autoscale.delete_servers_by_name.assert_called_once_with( ['server1', 'server2']) autoscale.create_n_servers.reset_mock() autoscale.delete_servers_by_name.reset_mock() # Delete empty down and frozen servers. _mock_cell( admin_mock, stateapi_mock, partitions=[ { '_id': 'partition', 'data': { 'autoscale': { 'min_servers': 3, 'max_servers': 9 } } }, ], servers=[ { '_id': 'server1', 'partition': 'partition', '_create_timestamp': 100.0 }, { '_id': 'server2', 'partition': 'partition', '_create_timestamp': 100.0 }, { '_id': 'server3', 'partition': 'partition', '_create_timestamp': 100.0 }, { '_id': 'server4', 'partition': 'partition', '_create_timestamp': 100.0 }, { '_id': 'server5', 'partition': 'partition', '_create_timestamp': 100.0 }, { '_id': 'server6', 'partition': 'partition', '_create_timestamp': 100.0 }, { '_id': 'server7', 'partition': 'partition', '_create_timestamp': 100.0 }, ], servers_state=[ ('server1', 'up', 100, 100, 100), ('server2', 'up', 100, 100, 100), ('server3', 'up', 100, 100, 100), ('server4', 'down', 100, 100, 100), ('server5', 'down', 100, 100, 100), ('server6', 'down', 100, 100, 100), ('server7', 'frozen', 100, 100, 100), ], apps_state=[ ('proid.app#001', 'partition', 'server6'), ], ) mock_zkclient.get_children.return_value = ['server5'] autoscale.scale(0.5) autoscale.create_n_servers.assert_not_called() autoscale.delete_servers_by_name.assert_called_once_with(['server4'])
def test_scale_down_with_broken(self, admin_mock, stateapi_mock): """Test scaling down with some broken servers present.""" mock_zkclient = context.GLOBAL.zk.conn # Delete empty down, frozen and blackedout servers. _mock_cell( admin_mock, stateapi_mock, partitions=[ { '_id': 'partition', 'data': { 'autoscale': { 'min_servers': 3, 'max_servers': 9 } } }, ], servers=[ { '_id': 'server1', 'partition': 'partition', '_create_timestamp': 100.0 }, { '_id': 'server2', 'partition': 'partition', '_create_timestamp': 100.0 }, { '_id': 'server3', 'partition': 'partition', '_create_timestamp': 100.0 }, { '_id': 'server4', 'partition': 'partition', '_create_timestamp': 100.0 }, { '_id': 'server5', 'partition': 'partition', '_create_timestamp': 100.0 }, { '_id': 'server6', 'partition': 'partition', '_create_timestamp': 100.0 }, { '_id': 'server7', 'partition': 'partition', '_create_timestamp': 100.0 }, ], servers_state=[ ('server1', 'up', 100, 100, 100), ('server2', 'up', 100, 100, 100), ('server3', 'up', 100, 100, 100), ('server4', 'down', 100, 100, 100), ('server5', 'down', 100, 100, 100), ('server6', 'down', 100, 100, 100), ('server7', 'frozen', 100, 100, 100), ], apps_state=[ ('proid.app#001', 'partition', 'server6'), ], ) mock_zkclient.get_children.return_value = ['server5'] autoscale.scale(0.5, 0) autoscale.create_n_servers.assert_not_called() autoscale.delete_servers_by_name.assert_called_once_with( ['server4', 'server5', 'server7'], pool=None) autoscale.create_n_servers.reset_mock() autoscale.delete_servers_by_name.reset_mock() # Keep 1 frozen/blackedout server. _mock_cell( admin_mock, stateapi_mock, partitions=[ { '_id': 'partition', 'data': { 'autoscale': { 'min_servers': 0, 'max_servers': 9, 'max_broken_servers': 1, }, }, }, ], servers=[ { '_id': 'server1', 'partition': 'partition', '_create_timestamp': 100.0 }, { '_id': 'server2', 'partition': 'partition', '_create_timestamp': 100.0 }, { '_id': 'server3', 'partition': 'partition', '_create_timestamp': 100.0 }, ], servers_state=[ ('server1', 'down', 100, 100, 100), ('server2', 'down', 100, 100, 100), ('server3', 'frozen', 100, 100, 100), ], apps_state=[], ) mock_zkclient.get_children.return_value = ['server2'] autoscale.scale(0.5, 0) autoscale.create_n_servers.assert_not_called() args, kwargs = autoscale.delete_servers_by_name.call_args self.assertIn(args, [ (['server1', 'server2'], ), (['server1', 'server3'], ), ]) autoscale.create_n_servers.reset_mock() autoscale.delete_servers_by_name.reset_mock() # Keep 2 frozen/blackedout servers. _mock_cell( admin_mock, stateapi_mock, partitions=[ { '_id': 'partition', 'data': { 'autoscale': { 'min_servers': 0, 'max_servers': 9, 'max_broken_servers': 2, }, }, }, ], servers=[ { '_id': 'server1', 'partition': 'partition', '_create_timestamp': 100.0 }, { '_id': 'server2', 'partition': 'partition', '_create_timestamp': 100.0 }, { '_id': 'server3', 'partition': 'partition', '_create_timestamp': 100.0 }, ], servers_state=[ ('server1', 'down', 100, 100, 100), ('server2', 'down', 100, 100, 100), ('server3', 'frozen', 100, 100, 100), ], apps_state=[], ) mock_zkclient.get_children.return_value = ['server2'] autoscale.scale(0.5, 0) autoscale.create_n_servers.assert_not_called() autoscale.delete_servers_by_name.assert_called_once_with(['server1'], pool=None)