def test_scale_down(self):
"""Test scale down."""
autoscale._state.return_value = autoscale._STATE(
running=10,
pending=1,
busy_srv_cnt=5,
idle_servers=['a', 'b'],
)
self.assertEqual((0, ['a']),
autoscale.scale(min_servers=0,
max_servers=200,
default_app_srv_ratio=0.5,
max_batch=10))
autoscale._state.return_value = autoscale._STATE(
running=10,
pending=1,
busy_srv_cnt=5,
idle_servers=['a'],
)
self.assertEqual((0, []),
autoscale.scale(min_servers=0,
max_servers=200,
default_app_srv_ratio=0.5,
max_batch=10))
autoscale._state.return_value = autoscale._STATE(
running=10,
pending=0,
busy_srv_cnt=5,
idle_servers=['a', 'b', 'c'],
)
self.assertEqual((0, ['a', 'b', 'c']),
autoscale.scale(min_servers=0,
max_servers=200,
default_app_srv_ratio=0.5,
max_batch=10))
# There are 5 busy servers, so with 5 min servers, all idle will be
# removed.
self.assertEqual((0, ['a', 'b', 'c']),
autoscale.scale(min_servers=5,
max_servers=200,
default_app_srv_ratio=0.5,
max_batch=10))
# There are 5 busy servers, so with 6 min servers, two idle will be
# removed.
self.assertEqual((0, ['b', 'c']),
autoscale.scale(min_servers=6,
max_servers=200,
default_app_srv_ratio=0.5,
max_batch=10))
def autoscale_cmd(interval, server_app_ratio, workers):
"""Autoscale Treadmill cell based on scheduler queue."""
pool = None
if workers:
pool = multiprocessing.Pool(processes=workers)
pool.workers = workers
context.GLOBAL.zk.add_listener(zkutils.exit_on_lost)
while True:
autoscale.scale(server_app_ratio, pool=pool)
time.sleep(interval)
def test_scale_up_no_idle(self):
"""Test scale up."""
autoscale._state.return_value = autoscale._STATE(
running=10,
pending=100,
busy_srv_cnt=5,
idle_servers=[],
)
self.assertEqual((10, []),
autoscale.scale(min_servers=0,
max_servers=200,
default_app_srv_ratio=0.5,
max_batch=10))
self.assertEqual((1, []),
autoscale.scale(min_servers=0,
max_servers=6,
default_app_srv_ratio=0.5,
max_batch=10))
self.assertEqual((0, []),
autoscale.scale(min_servers=0,
max_servers=5,
default_app_srv_ratio=0.5,
max_batch=10))
autoscale._state.return_value = autoscale._STATE(
running=10,
pending=2,
busy_srv_cnt=5,
idle_servers=[],
)
self.assertEqual((1, []),
autoscale.scale(min_servers=0,
max_servers=200,
default_app_srv_ratio=0.5,
max_batch=10))
autoscale._state.return_value = autoscale._STATE(
running=1000,
pending=1,
busy_srv_cnt=10,
idle_servers=[],
)
self.assertEqual((1, []),
autoscale.scale(min_servers=0,
max_servers=2000,
default_app_srv_ratio=0.5,
max_batch=10))
def test_scale_up_with_idle(self):
"""Test scale up with some idle servers present."""
autoscale._state.return_value = autoscale._STATE(
running=10,
pending=100,
busy_srv_cnt=5,
idle_servers=['a', 'b'],
)
self.assertEqual((10, []),
autoscale.scale(min_servers=0,
max_servers=200,
default_app_srv_ratio=0.5,
max_batch=10))
# Two apps, two server idle, with current ratio .5, will not ask for
# new servers.
autoscale._state.return_value = autoscale._STATE(
running=10,
pending=2,
busy_srv_cnt=5,
idle_servers=['a', 'b'],
)
self.assertEqual((0, []),
autoscale.scale(min_servers=0,
max_servers=200,
default_app_srv_ratio=0.5,
max_batch=10))
# 6 pending apps, two server idle, with current ratio .5. expect
# additional 1 server.
autoscale._state.return_value = autoscale._STATE(
running=10,
pending=6,
busy_srv_cnt=5,
idle_servers=['a', 'b'],
)
self.assertEqual((1, []),
autoscale.scale(min_servers=0,
max_servers=200,
default_app_srv_ratio=0.5,
max_batch=10))
def autoscale_cmd(timeout, max_count, min_count, batch_count,
app_srv_ratio):
"""Autoscale Treadmill cell based on scheduler queue."""
while True:
create_cnt, extra_servers = autoscale.scale(
max_servers=max_count,
min_servers=min_count,
default_app_srv_ratio=app_srv_ratio,
max_batch=batch_count)
if create_cnt > 0:
autoscale.create_n_servers(create_cnt, partition=None)
if extra_servers:
autoscale.delete_servers_by_name(extra_servers)
time.sleep(timeout)
def test_scale_up_no_idle(self, admin_mock, stateapi_mock):
"""Test scaling up with no idle servers present."""
mock_zkclient = context.GLOBAL.zk.conn
mock_zkclient.get_children.return_value = []
# Ratio: 0.5
# Pending apps: 3, no servers - create 2 servers.
_mock_cell(
admin_mock, stateapi_mock,
partitions=[
{'_id': 'partition',
'data': {'autoscale': {'min_servers': 1, 'max_servers': 9}}},
],
servers=[],
servers_state=[],
apps_state=[
('proid.app#001', 'partition', None),
('proid.app#002', 'partition', None),
('proid.app#003', 'partition', None),
],
)
autoscale.scale(0.5, 0)
autoscale.create_n_servers.assert_called_once_with(
2, 'partition', pool=None
)
autoscale.delete_servers_by_name.assert_not_called()
autoscale.create_n_servers.reset_mock()
autoscale.delete_servers_by_name.reset_mock()
# Ratio: 0.5
# Pending apps: 100, no servers, max servers: 9 - create 9 severs.
_mock_cell(
admin_mock, stateapi_mock,
partitions=[
{'_id': 'partition',
'data': {'autoscale': {'min_servers': 1, 'max_servers': 9}}},
],
servers=[],
servers_state=[],
apps_state=[
('proid.app#%03d' % i, 'partition', None) for i in range(100)
],
)
autoscale.scale(0.5, 0)
autoscale.create_n_servers.assert_called_once_with(
9, 'partition', pool=None
)
autoscale.delete_servers_by_name.assert_not_called()
autoscale.create_n_servers.reset_mock()
autoscale.delete_servers_by_name.reset_mock()
# Ratio: 1.0 (down and frozen servers excluded).
# Pending apps: 3, no idle servers - create 3 servers.
# Down and frozen servers have apps placed on them - don't delete.
_mock_cell(
admin_mock, stateapi_mock,
partitions=[
{'_id': 'partition',
'data': {'autoscale': {'min_servers': 1, 'max_servers': 9}}},
],
servers=[
{'_id': 'server1', 'partition': 'partition',
'_create_timestamp': 100.0},
{'_id': 'server2', 'partition': 'partition',
'_create_timestamp': 100.0},
{'_id': 'server3', 'partition': 'partition',
'_create_timestamp': 100.0},
],
servers_state=[
('server1', 'up', 100, 100, 100),
('server2', 'down', 100, 100, 100),
('server3', 'frozen', 100, 100, 100),
],
apps_state=[
('proid.app#001', 'partition', 'server1'),
('proid.app#002', 'partition', 'server2'),
('proid.app#003', 'partition', 'server3'),
('proid.app#004', 'partition', 'server3'),
('proid.app#005', 'partition', 'server3'),
('proid.app#006', 'partition', 'server3'),
('proid.app#007', 'partition', None),
('proid.app#008', 'partition', None),
('proid.app#009', 'partition', None),
],
)
autoscale.scale(0.5, 0)
autoscale.create_n_servers.assert_called_once_with(
3, 'partition', pool=None
)
autoscale.delete_servers_by_name.assert_not_called()
def test_scale_down(self, admin_mock, stateapi_mock):
"""Test scaling down."""
mock_zkclient = context.GLOBAL.zk.conn
mock_zkclient.get_children.return_value = []
# Pending apps: 1, idle servers: 2 - delete 1 server.
_mock_cell(
admin_mock, stateapi_mock,
partitions=[
{'_id': 'partition',
'data': {'autoscale': {'min_servers': 3, 'max_servers': 9}}},
],
servers=[
{'_id': 'server1', 'partition': 'partition',
'_create_timestamp': 100.0},
{'_id': 'server2', 'partition': 'partition',
'_create_timestamp': 100.0},
{'_id': 'server3', 'partition': 'partition',
'_create_timestamp': 100.0},
{'_id': 'server4', 'partition': 'partition',
'_create_timestamp': 100.0},
{'_id': 'server5', 'partition': 'partition',
'_create_timestamp': 100.0},
],
servers_state=[
('server1', 'up', 100, 100, 100),
('server2', 'up', 100, 100, 100),
('server3', 'up', 100, 100, 100),
('server4', 'up', 100, 100, 100),
('server5', 'up', 100, 100, 100),
],
apps_state=[
('proid.app#001', 'partition', 'server1'),
('proid.app#002', 'partition', 'server2'),
('proid.app#003', 'partition', 'server3'),
('proid.app#004', 'partition', None),
],
)
autoscale.scale(0.5, 0)
autoscale.create_n_servers.assert_not_called()
autoscale.delete_servers_by_name.assert_called_once_with(
['server4'], pool=None
)
autoscale.create_n_servers.reset_mock()
autoscale.delete_servers_by_name.reset_mock()
# No pending apps, idle servers: 5, min servers: 3 - delete 2 servers.
_mock_cell(
admin_mock, stateapi_mock,
partitions=[
{'_id': 'partition',
'data': {'autoscale': {'min_servers': 3, 'max_servers': 9}}},
],
servers=[
{'_id': 'server1', 'partition': 'partition',
'_create_timestamp': 100.0},
{'_id': 'server2', 'partition': 'partition',
'_create_timestamp': 100.0},
{'_id': 'server3', 'partition': 'partition',
'_create_timestamp': 100.0},
{'_id': 'server4', 'partition': 'partition',
'_create_timestamp': 100.0},
{'_id': 'server5', 'partition': 'partition',
'_create_timestamp': 100.0},
],
servers_state=[
('server1', 'up', 100, 100, 100),
('server2', 'up', 100, 100, 100),
('server3', 'up', 100, 100, 100),
('server4', 'up', 100, 100, 100),
('server5', 'up', 100, 100, 100),
],
apps_state=[],
)
autoscale.scale(0.5, 0)
autoscale.create_n_servers.assert_not_called()
autoscale.delete_servers_by_name.assert_called_once_with(
['server1', 'server2'], pool=None
)
autoscale.create_n_servers.reset_mock()
autoscale.delete_servers_by_name.reset_mock()
# Delete empty down server, don't delete blackedout and frozen servers.
_mock_cell(
admin_mock, stateapi_mock,
partitions=[
{'_id': 'partition',
'data': {'autoscale': {'min_servers': 3, 'max_servers': 9}}},
],
servers=[
{'_id': 'server1', 'partition': 'partition',
'_create_timestamp': 100.0},
{'_id': 'server2', 'partition': 'partition',
'_create_timestamp': 100.0},
{'_id': 'server3', 'partition': 'partition',
'_create_timestamp': 100.0},
{'_id': 'server4', 'partition': 'partition',
'_create_timestamp': 100.0},
{'_id': 'server5', 'partition': 'partition',
'_create_timestamp': 100.0},
{'_id': 'server6', 'partition': 'partition',
'_create_timestamp': 100.0},
{'_id': 'server7', 'partition': 'partition',
'_create_timestamp': 100.0},
],
servers_state=[
('server1', 'up', 100, 100, 100),
('server2', 'up', 100, 100, 100),
('server3', 'up', 100, 100, 100),
('server4', 'down', 100, 100, 100),
('server5', 'down', 100, 100, 100),
('server6', 'down', 100, 100, 100),
('server7', 'frozen', 100, 100, 100),
],
apps_state=[
('proid.app#001', 'partition', 'server6'),
],
)
mock_zkclient.get_children.return_value = ['server5']
autoscale.scale(0.5, 0)
autoscale.create_n_servers.assert_not_called()
autoscale.delete_servers_by_name.assert_called_once_with(
['server4'], pool=None
)
autoscale.create_n_servers.reset_mock()
autoscale.delete_servers_by_name.reset_mock()
# Delete idle servers when grace period expires.
idle_servers_tracker = collections.defaultdict(dict)
_mock_cell(
admin_mock, stateapi_mock,
partitions=[
{'_id': 'partition',
'data': {'autoscale': {'min_servers': 0, 'max_servers': 9}}},
],
servers=[
{'_id': 'server1', 'partition': 'partition',
'_create_timestamp': 100.0},
{'_id': 'server2', 'partition': 'partition',
'_create_timestamp': 100.0},
{'_id': 'server3', 'partition': 'partition',
'_create_timestamp': 100.0},
],
servers_state=[
('server1', 'up', 100, 100, 100),
('server2', 'up', 100, 100, 100),
('server3', 'up', 100, 100, 100),
],
apps_state=[
('proid.app#001', 'partition', 'server1'),
],
)
idle_servers_tracker = autoscale.scale(0.5, 5 * 60)
print(idle_servers_tracker)
autoscale.create_n_servers.assert_not_called()
autoscale.delete_servers_by_name.assert_not_called()
self.assertEqual(
idle_servers_tracker['partition'],
{'server2': 1000.0, 'server3': 1000.0}
)
time.time.return_value = 1000.0 + (5 * 60) + 1
# server2 is no longer idle, server3 continues to be idle.
_mock_cell(
admin_mock, stateapi_mock,
partitions=[
{'_id': 'partition',
'data': {'autoscale': {'min_servers': 0, 'max_servers': 9}}},
],
servers=[
{'_id': 'server1', 'partition': 'partition',
'_create_timestamp': 100.0},
{'_id': 'server2', 'partition': 'partition',
'_create_timestamp': 100.0},
{'_id': 'server3', 'partition': 'partition',
'_create_timestamp': 100.0},
],
servers_state=[
('server1', 'up', 100, 100, 100),
('server2', 'up', 100, 100, 100),
('server3', 'up', 100, 100, 100),
],
apps_state=[
('proid.app#001', 'partition', 'server1'),
('proid.app#002', 'partition', 'server2'),
],
)
idle_servers_tracker = autoscale.scale(
0.5, 5 * 60, idle_servers_tracker=idle_servers_tracker
)
print(idle_servers_tracker)
autoscale.create_n_servers.assert_not_called()
autoscale.delete_servers_by_name.assert_called_once_with(
['server3'], pool=None
)
self.assertEqual(
idle_servers_tracker['partition'],
{'server3': 1000.0}
)
def test_scale_up_min_servers(self, admin_mock, stateapi_mock):
"""Test scaling up to min (active) servers."""
mock_zkclient = context.GLOBAL.zk.conn
mock_zkclient.get_children.return_value = []
# Empty partition, min servers: 3 - create 3 servers.
_mock_cell(
admin_mock, stateapi_mock,
partitions=[
{'_id': 'partition',
'data': {'autoscale': {'min_servers': 3, 'max_servers': 9}}},
],
servers=[],
servers_state=[],
apps_state=[],
)
autoscale.scale(0.5, 0)
autoscale.create_n_servers.assert_called_once_with(
3, 'partition', pool=None
)
autoscale.delete_servers_by_name.assert_not_called()
autoscale.create_n_servers.reset_mock()
autoscale.delete_servers_by_name.reset_mock()
# 1 up, 1 down and 1 frozen server, min servers: 3 - create 2 servers.
# Down and frozen servers have apps placed on them - don't delete.
_mock_cell(
admin_mock, stateapi_mock,
partitions=[
{'_id': 'partition',
'data': {'autoscale': {'min_servers': 3, 'max_servers': 9}}},
],
servers=[
{'_id': 'server1', 'partition': 'partition',
'_create_timestamp': 100.0},
{'_id': 'server2', 'partition': 'partition',
'_create_timestamp': 100.0},
{'_id': 'server3', 'partition': 'partition',
'_create_timestamp': 100.0},
],
servers_state=[
('server1', 'up', 100, 100, 100),
('server2', 'down', 100, 100, 100),
('server3', 'frozen', 100, 100, 100),
],
apps_state=[
('proid.app#001', 'partition', 'server1'),
('proid.app#002', 'partition', 'server2'),
('proid.app#003', 'partition', 'server3'),
],
)
autoscale.scale(0.5, 0)
autoscale.create_n_servers.assert_called_once_with(
2, 'partition', pool=None
)
autoscale.delete_servers_by_name.assert_not_called()
def test_scale_up_with_idle(self, admin_mock, stateapi_mock):
"""Test scaling up with some idle servers present."""
mock_zkclient = context.GLOBAL.zk.conn
mock_zkclient.get_children.return_value = []
# Ratio: 0.5
# Pending apps: 1, idle servers: 1 - don't create anything.
_mock_cell(
admin_mock, stateapi_mock,
partitions=[
{'_id': 'partition',
'data': {'autoscale': {'min_servers': 1, 'max_servers': 9}}},
],
servers=[
{'_id': 'server1', 'partition': 'partition',
'_create_timestamp': 100.0},
],
servers_state=[
('server1', 'up', 100, 100, 100),
],
apps_state=[
('proid.app#001', 'partition', None),
],
)
autoscale.scale(0.5, 0)
autoscale.create_n_servers.assert_not_called()
autoscale.delete_servers_by_name.assert_not_called()
autoscale.create_n_servers.reset_mock()
autoscale.delete_servers_by_name.reset_mock()
# Ratio: 0.5
# Pending apps: 3, idle servers: 2 - don't create anything.
_mock_cell(
admin_mock, stateapi_mock,
partitions=[
{'_id': 'partition',
'data': {'autoscale': {'min_servers': 1, 'max_servers': 9}}},
],
servers=[
{'_id': 'server1', 'partition': 'partition',
'_create_timestamp': 100.0},
{'_id': 'server2', 'partition': 'partition',
'_create_timestamp': 100.0},
],
servers_state=[
('server1', 'up', 100, 100, 100),
('server2', 'up', 100, 100, 100),
],
apps_state=[
('proid.app#001', 'partition', None),
('proid.app#002', 'partition', None),
('proid.app#003', 'partition', None),
],
)
autoscale.scale(0.5, 0)
autoscale.create_n_servers.assert_not_called()
autoscale.delete_servers_by_name.assert_not_called()
autoscale.create_n_servers.reset_mock()
autoscale.delete_servers_by_name.reset_mock()
# Ratio: 1.0.
# Pending apps: 3, idle servers: 2 - create 1 server.
_mock_cell(
admin_mock, stateapi_mock,
partitions=[
{'_id': 'partition',
'data': {'autoscale': {'min_servers': 1, 'max_servers': 9}}},
],
servers=[
{'_id': 'server1', 'partition': 'partition',
'_create_timestamp': 100.0},
{'_id': 'server2', 'partition': 'partition',
'_create_timestamp': 100.0},
{'_id': 'server3', 'partition': 'partition',
'_create_timestamp': 999.0},
{'_id': 'server4', 'partition': 'partition',
'_create_timestamp': 100.0},
],
servers_state=[
('server1', 'up', 100, 100, 100),
('server2', 'up', 100, 100, 100),
('server3', 'down', None, None, None), # Didn't report - new.
('server4', 'down', None, None, None), # Didn't report - down.
],
apps_state=[
('proid.app#001', 'partition', 'server1'),
('proid.app#002', 'partition', None),
('proid.app#003', 'partition', None),
('proid.app#004', 'partition', None),
],
)
autoscale.scale(0.5, 0)
autoscale.create_n_servers.assert_called_once_with(
1, 'partition', pool=None
)
autoscale.delete_servers_by_name.assert_called_once_with(
['server4'], pool=None
)
def autoscale_cmd(interval, server_app_ratio):
"""Autoscale Treadmill cell based on scheduler queue."""
while True:
autoscale.scale(server_app_ratio)
time.sleep(interval)
def test_scale_down(self, admin_mock, stateapi_mock):
"""Test scaling down."""
mock_zkclient = context.GLOBAL.zk.conn
mock_zkclient.get_children.return_value = []
# Pending apps: 1, idle servers: 2 - delete 1 server.
_mock_cell(
admin_mock,
stateapi_mock,
partitions=[
{
'_id': 'partition',
'data': {
'autoscale': {
'min_servers': 3,
'max_servers': 9
}
}
},
],
servers=[
{
'_id': 'server1',
'partition': 'partition',
'_create_timestamp': 100.0
},
{
'_id': 'server2',
'partition': 'partition',
'_create_timestamp': 100.0
},
{
'_id': 'server3',
'partition': 'partition',
'_create_timestamp': 100.0
},
{
'_id': 'server4',
'partition': 'partition',
'_create_timestamp': 100.0
},
{
'_id': 'server5',
'partition': 'partition',
'_create_timestamp': 100.0
},
],
servers_state=[
('server1', 'up', 100, 100, 100),
('server2', 'up', 100, 100, 100),
('server3', 'up', 100, 100, 100),
('server4', 'up', 100, 100, 100),
('server5', 'up', 100, 100, 100),
],
apps_state=[
('proid.app#001', 'partition', 'server1'),
('proid.app#002', 'partition', 'server2'),
('proid.app#003', 'partition', 'server3'),
('proid.app#004', 'partition', None),
],
)
autoscale.scale(0.5)
autoscale.create_n_servers.assert_not_called()
autoscale.delete_servers_by_name.assert_called_once_with(['server4'])
autoscale.create_n_servers.reset_mock()
autoscale.delete_servers_by_name.reset_mock()
# No pending apps, idle servers: 5, min servers: 3 - delete 2 servers.
_mock_cell(
admin_mock,
stateapi_mock,
partitions=[
{
'_id': 'partition',
'data': {
'autoscale': {
'min_servers': 3,
'max_servers': 9
}
}
},
],
servers=[
{
'_id': 'server1',
'partition': 'partition',
'_create_timestamp': 100.0
},
{
'_id': 'server2',
'partition': 'partition',
'_create_timestamp': 100.0
},
{
'_id': 'server3',
'partition': 'partition',
'_create_timestamp': 100.0
},
{
'_id': 'server4',
'partition': 'partition',
'_create_timestamp': 100.0
},
{
'_id': 'server5',
'partition': 'partition',
'_create_timestamp': 100.0
},
],
servers_state=[
('server1', 'up', 100, 100, 100),
('server2', 'up', 100, 100, 100),
('server3', 'up', 100, 100, 100),
('server4', 'up', 100, 100, 100),
('server5', 'up', 100, 100, 100),
],
apps_state=[],
)
autoscale.scale(0.5)
autoscale.create_n_servers.assert_not_called()
autoscale.delete_servers_by_name.assert_called_once_with(
['server1', 'server2'])
autoscale.create_n_servers.reset_mock()
autoscale.delete_servers_by_name.reset_mock()
# Delete empty down and frozen servers.
_mock_cell(
admin_mock,
stateapi_mock,
partitions=[
{
'_id': 'partition',
'data': {
'autoscale': {
'min_servers': 3,
'max_servers': 9
}
}
},
],
servers=[
{
'_id': 'server1',
'partition': 'partition',
'_create_timestamp': 100.0
},
{
'_id': 'server2',
'partition': 'partition',
'_create_timestamp': 100.0
},
{
'_id': 'server3',
'partition': 'partition',
'_create_timestamp': 100.0
},
{
'_id': 'server4',
'partition': 'partition',
'_create_timestamp': 100.0
},
{
'_id': 'server5',
'partition': 'partition',
'_create_timestamp': 100.0
},
{
'_id': 'server6',
'partition': 'partition',
'_create_timestamp': 100.0
},
{
'_id': 'server7',
'partition': 'partition',
'_create_timestamp': 100.0
},
],
servers_state=[
('server1', 'up', 100, 100, 100),
('server2', 'up', 100, 100, 100),
('server3', 'up', 100, 100, 100),
('server4', 'down', 100, 100, 100),
('server5', 'down', 100, 100, 100),
('server6', 'down', 100, 100, 100),
('server7', 'frozen', 100, 100, 100),
],
apps_state=[
('proid.app#001', 'partition', 'server6'),
],
)
mock_zkclient.get_children.return_value = ['server5']
autoscale.scale(0.5)
autoscale.create_n_servers.assert_not_called()
autoscale.delete_servers_by_name.assert_called_once_with(['server4'])
def test_scale_down_with_broken(self, admin_mock, stateapi_mock):
"""Test scaling down with some broken servers present."""
mock_zkclient = context.GLOBAL.zk.conn
# Delete empty down, frozen and blackedout servers.
_mock_cell(
admin_mock,
stateapi_mock,
partitions=[
{
'_id': 'partition',
'data': {
'autoscale': {
'min_servers': 3,
'max_servers': 9
}
}
},
],
servers=[
{
'_id': 'server1',
'partition': 'partition',
'_create_timestamp': 100.0
},
{
'_id': 'server2',
'partition': 'partition',
'_create_timestamp': 100.0
},
{
'_id': 'server3',
'partition': 'partition',
'_create_timestamp': 100.0
},
{
'_id': 'server4',
'partition': 'partition',
'_create_timestamp': 100.0
},
{
'_id': 'server5',
'partition': 'partition',
'_create_timestamp': 100.0
},
{
'_id': 'server6',
'partition': 'partition',
'_create_timestamp': 100.0
},
{
'_id': 'server7',
'partition': 'partition',
'_create_timestamp': 100.0
},
],
servers_state=[
('server1', 'up', 100, 100, 100),
('server2', 'up', 100, 100, 100),
('server3', 'up', 100, 100, 100),
('server4', 'down', 100, 100, 100),
('server5', 'down', 100, 100, 100),
('server6', 'down', 100, 100, 100),
('server7', 'frozen', 100, 100, 100),
],
apps_state=[
('proid.app#001', 'partition', 'server6'),
],
)
mock_zkclient.get_children.return_value = ['server5']
autoscale.scale(0.5, 0)
autoscale.create_n_servers.assert_not_called()
autoscale.delete_servers_by_name.assert_called_once_with(
['server4', 'server5', 'server7'], pool=None)
autoscale.create_n_servers.reset_mock()
autoscale.delete_servers_by_name.reset_mock()
# Keep 1 frozen/blackedout server.
_mock_cell(
admin_mock,
stateapi_mock,
partitions=[
{
'_id': 'partition',
'data': {
'autoscale': {
'min_servers': 0,
'max_servers': 9,
'max_broken_servers': 1,
},
},
},
],
servers=[
{
'_id': 'server1',
'partition': 'partition',
'_create_timestamp': 100.0
},
{
'_id': 'server2',
'partition': 'partition',
'_create_timestamp': 100.0
},
{
'_id': 'server3',
'partition': 'partition',
'_create_timestamp': 100.0
},
],
servers_state=[
('server1', 'down', 100, 100, 100),
('server2', 'down', 100, 100, 100),
('server3', 'frozen', 100, 100, 100),
],
apps_state=[],
)
mock_zkclient.get_children.return_value = ['server2']
autoscale.scale(0.5, 0)
autoscale.create_n_servers.assert_not_called()
args, kwargs = autoscale.delete_servers_by_name.call_args
self.assertIn(args, [
(['server1', 'server2'], ),
(['server1', 'server3'], ),
])
autoscale.create_n_servers.reset_mock()
autoscale.delete_servers_by_name.reset_mock()
# Keep 2 frozen/blackedout servers.
_mock_cell(
admin_mock,
stateapi_mock,
partitions=[
{
'_id': 'partition',
'data': {
'autoscale': {
'min_servers': 0,
'max_servers': 9,
'max_broken_servers': 2,
},
},
},
],
servers=[
{
'_id': 'server1',
'partition': 'partition',
'_create_timestamp': 100.0
},
{
'_id': 'server2',
'partition': 'partition',
'_create_timestamp': 100.0
},
{
'_id': 'server3',
'partition': 'partition',
'_create_timestamp': 100.0
},
],
servers_state=[
('server1', 'down', 100, 100, 100),
('server2', 'down', 100, 100, 100),
('server3', 'frozen', 100, 100, 100),
],
apps_state=[],
)
mock_zkclient.get_children.return_value = ['server2']
autoscale.scale(0.5, 0)
autoscale.create_n_servers.assert_not_called()
autoscale.delete_servers_by_name.assert_called_once_with(['server1'],
pool=None)
