def test_spot_instance_blacklist(mocker): """check that spot requests being cancelled will result in temporary blacklisting""" # ensure EC2Manager returns a request ID class _status_code(object): code = 'instance-terminated-by-service' class _MockReq(boto.ec2.spotinstancerequest.SpotInstanceRequest): def __init__(self, *args, **kwds): super(_MockReq, self).__init__(*args, **kwds) self.state = 'cancelled' self.status = _status_code req = _MockReq() req.launch_specification = mocker.Mock() req.launch_specification.instance_type = '80286' mocker.patch('ec2spotmanager.CloudProvider.EC2SpotCloudProvider.CORES_PER_INSTANCE', new={'80286': 1}) mock_ec2mgr = mocker.patch('ec2spotmanager.CloudProvider.EC2SpotCloudProvider.EC2Manager') mock_ec2mgr.return_value.check_spot_requests.return_value = (req,) # set-up redis mock to return price data and image name def _mock_redis_get(key): if ":blacklist:" in key: return True if ":price:" in key: return '{"redmond": {"mshq": [0.001]}}' if ":image:" in key: return 'warp' raise UncatchableException("unhandle key in mock_get(): %s" % (key,)) def _mock_redis_set(key, value, ex=None): assert ":blacklist:redmond:mshq:" in key mock_redis = mocker.patch('redis.StrictRedis') mock_redis.return_value.get = mocker.Mock(side_effect=_mock_redis_get) mock_redis.return_value.set = mocker.Mock(side_effect=_mock_redis_set) # create database state config = create_config(name='config #1', size=1, cycle_interval=3600, ec2_key_name='fredsRefurbishedSshKey', ec2_security_groups='mostlysecure', ec2_instance_types=['80286'], ec2_image_name='os/2', ec2_allowed_regions=['redmond'], max_price='0.1', ec2_userdata=b'cleverscript') pool = create_pool(config=config, enabled=True) create_instance(None, pool=pool, status_code=INSTANCE_STATE['requested'], ec2_instance_id='req123', ec2_region="redmond", ec2_zone="mshq") # call function under test # XXX: this message is inaccurate .. there were no allowed regions at all with pytest.raises(UncatchableException, match="temporary-failure: Spot request.* and cancelled"): update_requests('EC2Spot', 'redmond', pool.pk) update_instances('EC2Spot', 'redmond') with pytest.raises(UncatchableException, match="No allowed region was cheap enough to spawn instances"): check_and_resize_pool(pool.pk) # check that laniakea calls were made cluster = mock_ec2mgr.return_value assert {call[0] for call in cluster.method_calls} == {'connect', 'check_spot_requests', 'find'} # check that instance was updated assert not Instance.objects.exists() # check that blacklist was set in redis assert len(mock_redis.return_value.set.mock_calls) == 1
def test_nothing_to_do(): """nothing is done if no pools are enabled""" config = create_config(name='config #1', size=1, cycle_interval=1, ec2_key_name='a', ec2_image_name='a', max_price='0.1', ec2_userdata='a', ec2_allowed_regions=['a']) pool = create_pool(config=config) update_requests('prov1', 'a', pool.pk) update_instances('prov1', 'a') cycle_and_terminate_disabled('prov1', 'a') assert not Instance.objects.exists()
def test_instance_not_updatable(mocker): """instances are not touched while they are not tagged Updatable""" # ensure EC2Manager returns a request ID class _MockInstance(boto.ec2.instance.Instance): @property def state_code(self): return INSTANCE_STATE['stopping'] boto_instance = _MockInstance() boto_instance.id = 'i-123' boto_instance.public_dns_name = 'fm-test.fuzzing.allizom.com' mock_ec2mgr = mocker.patch( 'ec2spotmanager.CloudProvider.EC2SpotCloudProvider.EC2Manager') mock_ec2mgr.return_value.find.return_value = (boto_instance, ) # create database state config = create_config(name='config #1', size=1, cycle_interval=3600, ec2_key_name='fredsRefurbishedSshKey', ec2_security_groups='mostlysecure', ec2_instance_types=['80286'], ec2_image_name='os/2', ec2_allowed_regions=['redmond'], max_price='0.1', ec2_userdata=b'cleverscript') pool = create_pool(config=config, enabled=True, last_cycled=timezone.now()) orig = create_instance(None, pool=pool, status_code=INSTANCE_STATE['running'], ec2_instance_id='i-123', ec2_region="redmond", ec2_zone="mshq") # call function under test update_instances('EC2Spot', 'redmond') cycle_and_terminate_disabled('EC2Spot', 'redmond') check_and_resize_pool(pool.pk) # check that laniakea calls were made cluster = mock_ec2mgr.return_value assert {call[0] for call in cluster.method_calls} == {'connect', 'find'} # check that instances were not updated count = 0 for instance in Instance.objects.all(): assert instance.status_code == INSTANCE_STATE["running"] assert instance.id == orig.id count += 1 assert count == 1
def test_instance_shutting_down(mocker): """instances are replaced when shut down or terminated""" # ensure EC2Manager returns a request ID class _MockInstance(boto.ec2.instance.Instance): @property def state_code(self): return INSTANCE_STATE['shutting-down'] def add_tags(self, _tags, _dry_run=False): pass class _MockInstance2(_MockInstance): @property def state_code(self): return INSTANCE_STATE['terminated'] boto_instance1 = _MockInstance() boto_instance1.id = 'i-123' boto_instance1.public_dns_name = 'fm-test1.fuzzing.allizom.com' boto_instance1.tags = {SPOTMGR_TAG + '-Updatable': '1'} boto_instance2 = _MockInstance2() boto_instance2.id = 'i-456' boto_instance2.public_dns_name = 'fm-test2.fuzzing.allizom.com' boto_instance2.tags = {SPOTMGR_TAG + '-Updatable': '1'} mocker.patch( 'ec2spotmanager.CloudProvider.EC2SpotCloudProvider.CORES_PER_INSTANCE', new={'80286': 1}) mock_ec2mgr = mocker.patch( 'ec2spotmanager.CloudProvider.EC2SpotCloudProvider.EC2Manager') mock_ec2mgr.return_value.find.return_value = (boto_instance1, boto_instance2) # set-up redis mock to return price data and image name def _mock_redis_get(key): if ":blacklist:" in key: return None if ":price:" in key: return '{"redmond": {"mshq": [0.005]}}' if ":image:" in key: return 'warp' raise UncatchableException("unhandle key in mock_get(): %s" % (key, )) mock_redis = mocker.patch('redis.StrictRedis.from_url') mock_redis.return_value.get = mocker.Mock(side_effect=_mock_redis_get) # ensure EC2Manager returns a request ID mock_ec2mgr.return_value.create_spot_requests.return_value = ('req123', 'req456') # create database state config = create_config(name='config #1', size=2, cycle_interval=3600, ec2_key_name='fredsRefurbishedSshKey', ec2_security_groups='mostlysecure', ec2_instance_types=['80286'], ec2_image_name='os/2', ec2_allowed_regions=['redmond'], max_price='0.1', ec2_userdata=b'cleverscript') pool = create_pool(config=config, enabled=True) orig1 = create_instance(None, pool=pool, status_code=INSTANCE_STATE['running'], ec2_instance_id='i-123', ec2_region="redmond", ec2_zone="mshq") orig2 = create_instance(None, pool=pool, status_code=INSTANCE_STATE['running'], ec2_instance_id='i-456', ec2_region="redmond", ec2_zone="mshq") # call function under test update_instances('EC2Spot', 'redmond') remaining = {orig1.instance_id, orig2.instance_id} for old in Instance.objects.all(): remaining.remove(old.instance_id) assert old.status_code in { INSTANCE_STATE['shutting-down'], INSTANCE_STATE['terminated'] } assert not remaining cycle_and_terminate_disabled('EC2Spot', 'redmond') check_and_resize_pool(pool.pk) # check that laniakea calls were made cluster = mock_ec2mgr.return_value assert {call[0] for call in cluster.method_calls } == {'connect', 'create_spot_requests', 'find'} # check that instances were replaced remaining = {'req123', 'req456'} for new in Instance.objects.all(): remaining.remove(new.instance_id) assert new.id not in {orig1.id, orig2.id} assert new.pool.id == pool.id assert new.size == 1 assert new.status_code == INSTANCE_STATE["requested"] assert not remaining