def test_spot_instance_blacklist(mocker):
"""check that spot requests being cancelled will result in temporary blacklisting"""
# ensure EC2Manager returns a request ID
class _status_code(object):
code = 'instance-terminated-by-service'
class _MockReq(boto.ec2.spotinstancerequest.SpotInstanceRequest):
def __init__(self, *args, **kwds):
super(_MockReq, self).__init__(*args, **kwds)
self.state = 'cancelled'
self.status = _status_code
req = _MockReq()
req.launch_specification = mocker.Mock()
req.launch_specification.instance_type = '80286'
mocker.patch('ec2spotmanager.CloudProvider.EC2SpotCloudProvider.CORES_PER_INSTANCE', new={'80286': 1})
mock_ec2mgr = mocker.patch('ec2spotmanager.CloudProvider.EC2SpotCloudProvider.EC2Manager')
mock_ec2mgr.return_value.check_spot_requests.return_value = (req,)
# set-up redis mock to return price data and image name
def _mock_redis_get(key):
if ":blacklist:" in key:
return True
if ":price:" in key:
return '{"redmond": {"mshq": [0.001]}}'
if ":image:" in key:
return 'warp'
raise UncatchableException("unhandle key in mock_get(): %s" % (key,))
def _mock_redis_set(key, value, ex=None):
assert ":blacklist:redmond:mshq:" in key
mock_redis = mocker.patch('redis.StrictRedis')
mock_redis.return_value.get = mocker.Mock(side_effect=_mock_redis_get)
mock_redis.return_value.set = mocker.Mock(side_effect=_mock_redis_set)
# create database state
config = create_config(name='config #1', size=1, cycle_interval=3600, ec2_key_name='fredsRefurbishedSshKey',
ec2_security_groups='mostlysecure', ec2_instance_types=['80286'], ec2_image_name='os/2',
ec2_allowed_regions=['redmond'], max_price='0.1', ec2_userdata=b'cleverscript')
pool = create_pool(config=config, enabled=True)
create_instance(None, pool=pool, status_code=INSTANCE_STATE['requested'],
ec2_instance_id='req123', ec2_region="redmond", ec2_zone="mshq")
# call function under test
# XXX: this message is inaccurate .. there were no allowed regions at all
with pytest.raises(UncatchableException, match="temporary-failure: Spot request.* and cancelled"):
update_requests('EC2Spot', 'redmond', pool.pk)
update_instances('EC2Spot', 'redmond')
with pytest.raises(UncatchableException, match="No allowed region was cheap enough to spawn instances"):
check_and_resize_pool(pool.pk)
# check that laniakea calls were made
cluster = mock_ec2mgr.return_value
assert {call[0] for call in cluster.method_calls} == {'connect', 'check_spot_requests', 'find'}
# check that instance was updated
assert not Instance.objects.exists()
# check that blacklist was set in redis
assert len(mock_redis.return_value.set.mock_calls) == 1
def test_nothing_to_do():
"""nothing is done if no pools are enabled"""
config = create_config(name='config #1', size=1, cycle_interval=1, ec2_key_name='a', ec2_image_name='a',
max_price='0.1', ec2_userdata='a', ec2_allowed_regions=['a'])
pool = create_pool(config=config)
update_requests('prov1', 'a', pool.pk)
update_instances('prov1', 'a')
cycle_and_terminate_disabled('prov1', 'a')
assert not Instance.objects.exists()
def test_instance_not_updatable(mocker):
"""instances are not touched while they are not tagged Updatable"""
# ensure EC2Manager returns a request ID
class _MockInstance(boto.ec2.instance.Instance):
@property
def state_code(self):
return INSTANCE_STATE['stopping']
boto_instance = _MockInstance()
boto_instance.id = 'i-123'
boto_instance.public_dns_name = 'fm-test.fuzzing.allizom.com'
mock_ec2mgr = mocker.patch(
'ec2spotmanager.CloudProvider.EC2SpotCloudProvider.EC2Manager')
mock_ec2mgr.return_value.find.return_value = (boto_instance, )
# create database state
config = create_config(name='config #1',
size=1,
cycle_interval=3600,
ec2_key_name='fredsRefurbishedSshKey',
ec2_security_groups='mostlysecure',
ec2_instance_types=['80286'],
ec2_image_name='os/2',
ec2_allowed_regions=['redmond'],
max_price='0.1',
ec2_userdata=b'cleverscript')
pool = create_pool(config=config, enabled=True, last_cycled=timezone.now())
orig = create_instance(None,
pool=pool,
status_code=INSTANCE_STATE['running'],
ec2_instance_id='i-123',
ec2_region="redmond",
ec2_zone="mshq")
# call function under test
update_instances('EC2Spot', 'redmond')
cycle_and_terminate_disabled('EC2Spot', 'redmond')
check_and_resize_pool(pool.pk)
# check that laniakea calls were made
cluster = mock_ec2mgr.return_value
assert {call[0] for call in cluster.method_calls} == {'connect', 'find'}
# check that instances were not updated
count = 0
for instance in Instance.objects.all():
assert instance.status_code == INSTANCE_STATE["running"]
assert instance.id == orig.id
count += 1
assert count == 1
def test_instance_shutting_down(mocker):
"""instances are replaced when shut down or terminated"""
# ensure EC2Manager returns a request ID
class _MockInstance(boto.ec2.instance.Instance):
@property
def state_code(self):
return INSTANCE_STATE['shutting-down']
def add_tags(self, _tags, _dry_run=False):
pass
class _MockInstance2(_MockInstance):
@property
def state_code(self):
return INSTANCE_STATE['terminated']
boto_instance1 = _MockInstance()
boto_instance1.id = 'i-123'
boto_instance1.public_dns_name = 'fm-test1.fuzzing.allizom.com'
boto_instance1.tags = {SPOTMGR_TAG + '-Updatable': '1'}
boto_instance2 = _MockInstance2()
boto_instance2.id = 'i-456'
boto_instance2.public_dns_name = 'fm-test2.fuzzing.allizom.com'
boto_instance2.tags = {SPOTMGR_TAG + '-Updatable': '1'}
mocker.patch(
'ec2spotmanager.CloudProvider.EC2SpotCloudProvider.CORES_PER_INSTANCE',
new={'80286': 1})
mock_ec2mgr = mocker.patch(
'ec2spotmanager.CloudProvider.EC2SpotCloudProvider.EC2Manager')
mock_ec2mgr.return_value.find.return_value = (boto_instance1,
boto_instance2)
# set-up redis mock to return price data and image name
def _mock_redis_get(key):
if ":blacklist:" in key:
return None
if ":price:" in key:
return '{"redmond": {"mshq": [0.005]}}'
if ":image:" in key:
return 'warp'
raise UncatchableException("unhandle key in mock_get(): %s" % (key, ))
mock_redis = mocker.patch('redis.StrictRedis.from_url')
mock_redis.return_value.get = mocker.Mock(side_effect=_mock_redis_get)
# ensure EC2Manager returns a request ID
mock_ec2mgr.return_value.create_spot_requests.return_value = ('req123',
'req456')
# create database state
config = create_config(name='config #1',
size=2,
cycle_interval=3600,
ec2_key_name='fredsRefurbishedSshKey',
ec2_security_groups='mostlysecure',
ec2_instance_types=['80286'],
ec2_image_name='os/2',
ec2_allowed_regions=['redmond'],
max_price='0.1',
ec2_userdata=b'cleverscript')
pool = create_pool(config=config, enabled=True)
orig1 = create_instance(None,
pool=pool,
status_code=INSTANCE_STATE['running'],
ec2_instance_id='i-123',
ec2_region="redmond",
ec2_zone="mshq")
orig2 = create_instance(None,
pool=pool,
status_code=INSTANCE_STATE['running'],
ec2_instance_id='i-456',
ec2_region="redmond",
ec2_zone="mshq")
# call function under test
update_instances('EC2Spot', 'redmond')
remaining = {orig1.instance_id, orig2.instance_id}
for old in Instance.objects.all():
remaining.remove(old.instance_id)
assert old.status_code in {
INSTANCE_STATE['shutting-down'], INSTANCE_STATE['terminated']
}
assert not remaining
cycle_and_terminate_disabled('EC2Spot', 'redmond')
check_and_resize_pool(pool.pk)
# check that laniakea calls were made
cluster = mock_ec2mgr.return_value
assert {call[0]
for call in cluster.method_calls
} == {'connect', 'create_spot_requests', 'find'}
# check that instances were replaced
remaining = {'req123', 'req456'}
for new in Instance.objects.all():
remaining.remove(new.instance_id)
assert new.id not in {orig1.id, orig2.id}
assert new.pool.id == pool.id
assert new.size == 1
assert new.status_code == INSTANCE_STATE["requested"]
assert not remaining
