def rotate_cmd(count, partition):
"""Rotate nodes, deleting old nodes and starting new."""
if partition in ('-', '_default'):
partition = None
autoscale.create_n_servers(count, partition)
autoscale.delete_n_servers(count, partition)
def scale_cmd(count, partition):
"""Scale nodes to specified count."""
if partition in ('-', '_default'):
partition = None
cell = context.GLOBAL.cell
admin_srv = admin.Server(context.GLOBAL.ldap.conn)
all_servers = sorted(admin_srv.list({'cell': cell}),
key=lambda x: x.get('partition'))
by_partition = {}
for part, srvs in itertools.groupby(all_servers,
lambda x: x.get('partition')):
by_partition[part] = list(srvs)
current_count = len(
by_partition.get(partition if partition else '_default', []))
count_by_partition = collections.Counter(
{p: len(s)
for p, s in by_partition.items()})
if count not in {None, current_count}:
if count > current_count:
autoscale.create_n_servers(count - current_count, partition)
else:
autoscale.delete_n_servers(current_count - count, partition)
count_by_partition[partition if partition else '_default'] = count
for part in sorted(count_by_partition):
print('{: <32}: {}'.format(part, count_by_partition[part]))
def autoscale_cmd(timeout, max_count, min_count, batch_count,
app_srv_ratio):
"""Autoscale Treadmill cell based on scheduler queue."""
while True:
create_cnt, extra_servers = autoscale.scale(
max_servers=max_count,
min_servers=min_count,
default_app_srv_ratio=app_srv_ratio,
max_batch=batch_count)
if create_cnt > 0:
autoscale.create_n_servers(create_cnt, partition=None)
if extra_servers:
autoscale.delete_servers_by_name(extra_servers)
time.sleep(timeout)
def test_create_n_servers(self, create_host_mock, admin_mock):
"""Test creating new servers in the cell."""
admin_cell_mock = admin_mock.cell.return_value
admin_cell_mock.get.return_value = {
'data': {
'image': 'ami-test',
'size': 'm5.large',
'subnets': ['subnet-4c76610a', 'subnet-4c76610b'],
'secgroup': 'test',
'hostgroups': ['test'],
'instance_profile': 'test',
'disk_size': '100',
'aws_account': 'test',
}
}
admin_part_mock = admin_mock.partition.return_value
admin_part_mock.get.return_value = {
'data': {
'instance_types': ['m5.large', 'm5.xlarge'],
'spot_instance_types': ['m5.large', 'm5.xlarge', 'm5.2xlarge'],
}
}
# Create 3 on-demand hosts.
hosts_created = autoscale.create_n_servers(3, 'partition')
self.assertEqual(
hosts_created,
[
{
'hostname': 'test-partition-8s6u9ns20000.foo.com',
'type': 'm5.large',
'lifecycle': 'on-demand',
'subnet': 'subnet-4c76610a',
},
{
'hostname': 'test-partition-8s6u9ns20001.foo.com',
'type': 'm5.large',
'lifecycle': 'on-demand',
'subnet': 'subnet-4c76610a',
},
{
'hostname': 'test-partition-8s6u9ns20002.foo.com',
'type': 'm5.large',
'lifecycle': 'on-demand',
'subnet': 'subnet-4c76610a',
},
]
)
create_host_mock.reset_mock()
# Create 1 on-demand host and 2 spot hosts.
hosts_created = autoscale.create_n_servers(
3, 'partition', min_on_demand=1, max_on_demand=1
)
self.assertEqual(
hosts_created,
[
{
'hostname': 'test-partition-8s6u9ns20000.foo.com',
'type': 'm5.large',
'lifecycle': 'on-demand',
'subnet': 'subnet-4c76610a',
},
{
'hostname': 'test-partition-8s6u9ns20001.foo.com',
'type': 'm5.large',
'lifecycle': 'spot',
'subnet': 'subnet-4c76610a',
},
{
'hostname': 'test-partition-8s6u9ns20002.foo.com',
'type': 'm5.large',
'lifecycle': 'spot',
'subnet': 'subnet-4c76610a',
},
]
)
create_host_mock.reset_mock()
# Create 1 on-demand host and try 2 spot hosts, fallback to on-demand.
create_host_mock.side_effect = lambda *args, **kwargs: _raise_if(
kwargs['spot'],
botoexc.ClientError(
{'Error': {'Code': 'SpotMaxPriceTooLow'}}, None
)
)
hosts_created = autoscale.create_n_servers(
3, 'partition', min_on_demand=1, max_on_demand=3
)
self.assertEqual(
hosts_created,
[
{
'hostname': 'test-partition-8s6u9ns20000.foo.com',
'type': 'm5.large',
'lifecycle': 'on-demand',
'subnet': 'subnet-4c76610a',
},
{
'hostname': 'test-partition-8s6u9ns20001.foo.com',
'type': 'm5.large',
'lifecycle': 'on-demand',
'subnet': 'subnet-4c76610a',
},
{
'hostname': 'test-partition-8s6u9ns20002.foo.com',
'type': 'm5.large',
'lifecycle': 'on-demand',
'subnet': 'subnet-4c76610a',
},
]
)
# Check if each spot type was tried once in each subnet, check order.
self.assertEqual(
[
(kwargs['instance_type'], kwargs['subnet'])
for _args, kwargs in create_host_mock.call_args_list
if kwargs['spot']
],
[
('m5.large', 'subnet-4c76610a'),
('m5.large', 'subnet-4c76610b'),
('m5.xlarge', 'subnet-4c76610a'),
('m5.xlarge', 'subnet-4c76610b'),
('m5.2xlarge', 'subnet-4c76610a'),
('m5.2xlarge', 'subnet-4c76610b'),
]
)
create_host_mock.reset_mock()
# Create 1 on-demand host and try 2 spot hosts, no fallback - fail.
with self.assertRaisesRegex(
Exception,
'Failed to create host test-partition-8s6u9ns20001.foo.com'
):
hosts_created = autoscale.create_n_servers(
3, 'partition', min_on_demand=1, max_on_demand=1
)
create_host_mock.reset_mock()
# Create 1 on-demand host and 2 spot hosts, m5.large spot not feasible.
create_host_mock.side_effect = lambda *args, **kwargs: _raise_if(
kwargs['spot'] and kwargs['instance_type'] == 'm5.large',
botoexc.ClientError(
{'Error': {'Code': 'InsufficientInstanceCapacity'}}, None
)
)
hosts_created = autoscale.create_n_servers(
3, 'partition', min_on_demand=1, max_on_demand=1
)
self.assertEqual(
hosts_created,
[
{
'hostname': 'test-partition-8s6u9ns20000.foo.com',
'type': 'm5.large',
'lifecycle': 'on-demand',
'subnet': 'subnet-4c76610a',
},
{
'hostname': 'test-partition-8s6u9ns20001.foo.com',
'type': 'm5.xlarge',
'lifecycle': 'spot',
'subnet': 'subnet-4c76610a',
},
{
'hostname': 'test-partition-8s6u9ns20002.foo.com',
'type': 'm5.xlarge',
'lifecycle': 'spot',
'subnet': 'subnet-4c76610a',
},
]
)
# Check if m5.large spot was tried once in each subnet, then m5.xlarge.
self.assertEqual(
[
(kwargs['instance_type'], kwargs['subnet'])
for _args, kwargs in create_host_mock.call_args_list
if kwargs['spot']
],
[
('m5.large', 'subnet-4c76610a'),
('m5.large', 'subnet-4c76610b'),
('m5.xlarge', 'subnet-4c76610a'),
('m5.xlarge', 'subnet-4c76610a'),
]
)
create_host_mock.reset_mock()
# Create 1 on-demand host and 2 spot hosts, m5.large not feasible in
# subnet-4c76610a, but feasible in subnet-4c76610b.
create_host_mock.side_effect = lambda *args, **kwargs: _raise_if(
(
kwargs['instance_type'] == 'm5.large' and
kwargs['subnet'] == 'subnet-4c76610a'
),
botoexc.ClientError(
{'Error': {'Code': 'InsufficientInstanceCapacity'}}, None
)
)
hosts_created = autoscale.create_n_servers(
3, 'partition', min_on_demand=1, max_on_demand=1
)
self.assertEqual(
hosts_created,
[
{
'hostname': 'test-partition-8s6u9ns20000.foo.com',
'type': 'm5.large',
'lifecycle': 'on-demand',
'subnet': 'subnet-4c76610b',
},
{
'hostname': 'test-partition-8s6u9ns20001.foo.com',
'type': 'm5.large',
'lifecycle': 'spot',
'subnet': 'subnet-4c76610b',
},
{
'hostname': 'test-partition-8s6u9ns20002.foo.com',
'type': 'm5.large',
'lifecycle': 'spot',
'subnet': 'subnet-4c76610b',
},
]
)
# Check if subnet-4c76610a was tried once for on-demand and spot.
self.assertEqual(
[
(kwargs['instance_type'], kwargs['spot'], kwargs['subnet'])
for _args, kwargs in create_host_mock.call_args_list
],
[
('m5.large', False, 'subnet-4c76610a'),
('m5.large', False, 'subnet-4c76610b'),
('m5.large', True, 'subnet-4c76610a'),
('m5.large', True, 'subnet-4c76610b'),
('m5.large', True, 'subnet-4c76610b'),
]
)
create_host_mock.reset_mock()
# Create 1 on-demand host and 2 spot hosts, subnet-4c76610a exhausted.
create_host_mock.side_effect = lambda *args, **kwargs: _raise_if(
(
kwargs['subnet'] == 'subnet-4c76610a'
),
botoexc.ClientError(
{'Error': {'Code': 'InsufficientFreeAddressesInSubnet'}}, None
)
)
hosts_created = autoscale.create_n_servers(
3, 'partition', min_on_demand=1, max_on_demand=1
)
self.assertEqual(
hosts_created,
[
{
'hostname': 'test-partition-8s6u9ns20000.foo.com',
'type': 'm5.large',
'lifecycle': 'on-demand',
'subnet': 'subnet-4c76610b',
},
{
'hostname': 'test-partition-8s6u9ns20001.foo.com',
'type': 'm5.large',
'lifecycle': 'spot',
'subnet': 'subnet-4c76610b',
},
{
'hostname': 'test-partition-8s6u9ns20002.foo.com',
'type': 'm5.large',
'lifecycle': 'spot',
'subnet': 'subnet-4c76610b',
},
]
)
# Check if subnet-4c76610a was tried once (entire subnet excluded).
self.assertEqual(
[
(kwargs['instance_type'], kwargs['spot'], kwargs['subnet'])
for _args, kwargs in create_host_mock.call_args_list
],
[
('m5.large', False, 'subnet-4c76610a'),
('m5.large', False, 'subnet-4c76610b'),
('m5.large', True, 'subnet-4c76610b'),
('m5.large', True, 'subnet-4c76610b'),
]
)
create_host_mock.reset_mock()
# Create 1 on-demand host and 2 spot hosts, retry on InternalError.
raise_err = {
'test-partition-8s6u9ns20000.foo.com': True,
'test-partition-8s6u9ns20001.foo.com': True,
'test-partition-8s6u9ns20002.foo.com': True,
}
create_host_mock.side_effect = lambda *args, **kwargs: _raise_if(
raise_err.pop(kwargs['hostname'], False),
botoexc.ClientError(
{'Error': {'Code': 'InternalError'}}, None
)
)
hosts_created = autoscale.create_n_servers(
3, 'partition', min_on_demand=1, max_on_demand=1
)
self.assertEqual(
hosts_created,
[
{
'hostname': 'test-partition-8s6u9ns20000.foo.com',
'type': 'm5.large',
'lifecycle': 'on-demand',
'subnet': 'subnet-4c76610a',
},
{
'hostname': 'test-partition-8s6u9ns20001.foo.com',
'type': 'm5.large',
'lifecycle': 'spot',
'subnet': 'subnet-4c76610a',
},
{
'hostname': 'test-partition-8s6u9ns20002.foo.com',
'type': 'm5.large',
'lifecycle': 'spot',
'subnet': 'subnet-4c76610a',
},
]
)
# Check retries.
self.assertEqual(
[
(
kwargs['hostname'],
kwargs['instance_type'],
kwargs['spot'],
kwargs['subnet']
)
for _args, kwargs in create_host_mock.call_args_list
],
[
(
'test-partition-8s6u9ns20000.foo.com',
'm5.large', False, 'subnet-4c76610a'
),
(
'test-partition-8s6u9ns20000.foo.com',
'm5.large', False, 'subnet-4c76610a'
),
(
'test-partition-8s6u9ns20001.foo.com',
'm5.large', True, 'subnet-4c76610a'
),
(
'test-partition-8s6u9ns20001.foo.com',
'm5.large', True, 'subnet-4c76610a'
),
(
'test-partition-8s6u9ns20002.foo.com',
'm5.large', True, 'subnet-4c76610a'
),
(
'test-partition-8s6u9ns20002.foo.com',
'm5.large', True, 'subnet-4c76610a'
),
]
)
create_host_mock.reset_mock()