def test_nothing_to_do(): """nothing is done if no pools are enabled""" config = create_config(name='config #1', size=1, cycle_interval=1, ec2_key_name='a', ec2_image_name='a', ec2_max_price='0.1', userdata='a', ec2_allowed_regions=['a']) pool = create_pool(config=config) check_instance_pool(pool.id) assert not Instance.objects.exists()
def test_instance_price_high(mock_redis, mock_ec2mgr): """check that instances are not created if the price is too high""" # set-up redis mock to return price data and image name def _mock_redis_get(key): if ":blacklist:" in key: return None if ":price:" in key: return '{"redmond": {"mshq": [0.05]}}' if ":image:" in key: return 'warp' raise UncatchableException("unhandle key in mock_get(): %s" % (key,)) mock_redis.return_value.get = Mock(side_effect=_mock_redis_get) # create database state config = create_config(name='config #1', size=1, cycle_interval=3600, ec2_key_name='fredsRefurbishedSshKey', ec2_security_groups='mostlysecure', ec2_instance_types=['80286'], ec2_image_name='os/2', ec2_allowed_regions=['redmond'], ec2_max_price='0.01', userdata=b'cleverscript') pool = create_pool(config=config, enabled=True) # call function under test with pytest.raises(UncatchableException, match="No allowed regions was cheap enough to spawn instances"): check_instance_pool(pool.id) # check that laniakea calls were made cluster = mock_ec2mgr.return_value assert not cluster.method_calls # check that no instances were created assert not Instance.objects.exists()
def test_instance_not_updatable(mock_ec2mgr): """instances are not touched while they are not tagged Updatable""" # ensure EC2Manager returns a request ID class _MockInstance(boto.ec2.instance.Instance): @property def state_code(self): return INSTANCE_STATE['stopping'] boto_instance = _MockInstance() boto_instance.id = 'i-123' boto_instance.public_dns_name = 'fm-test.fuzzing.allizom.com' mock_ec2mgr.return_value.find.return_value = (boto_instance,) # create database state config = create_config(name='config #1', size=1, cycle_interval=3600, ec2_key_name='fredsRefurbishedSshKey', ec2_security_groups='mostlysecure', ec2_instance_types=['80286'], ec2_image_name='os/2', ec2_allowed_regions=['redmond'], ec2_max_price='0.1', userdata=b'cleverscript') pool = create_pool(config=config, enabled=True, last_cycled=timezone.now()) orig = create_instance(None, pool=pool, status_code=INSTANCE_STATE['running'], ec2_instance_id='i-123', ec2_region="redmond", ec2_zone="mshq") # call function under test check_instance_pool(pool.id) # check that laniakea calls were made cluster = mock_ec2mgr.return_value assert {call[0] for call in cluster.method_calls} == {'connect', 'find'} # check that instances were not updated count = 0 for instance in Instance.objects.all(): assert instance.status_code == INSTANCE_STATE["running"] assert instance.id == orig.id count += 1 assert count == 1
def test_lock(mock_lock): config = create_config(name='config #1', size=4, cycle_interval=3600, ec2_key_name='fredsRefurbishedSshKey', ec2_security_groups='mostlysecure', ec2_instance_types=['80286'], ec2_image_name='os/2', ec2_allowed_regions=['redmond'], ec2_max_price='0.1', userdata=b'cleverscript') pool = create_pool(config=config) mock_lock.return_value.acquire.return_value = False # call function under test check_instance_pool(pool.id)
def test_pool_trim(mock_redis, mock_ec2mgr): """check that pool down-size trims older instances until we meet the requirement""" # ensure EC2Manager returns a request ID class _MockInstance(boto.ec2.instance.Instance): @property def state_code(self): return INSTANCE_STATE['running'] boto_instances = [_MockInstance(), _MockInstance(), _MockInstance()] boto_instances[0].id = 'i-123' boto_instances[0].public_dns_name = 'fm-test1.fuzzing.allizom.com' boto_instances[1].id = 'i-456' boto_instances[1].public_dns_name = 'fm-test2.fuzzing.allizom.com' boto_instances[2].id = 'i-789' boto_instances[2].public_dns_name = 'fm-test3.fuzzing.allizom.com' def _find(*args, **kwds): if "instance_ids" in kwds: return [instance for instance in boto_instances if instance.id in kwds["instance_ids"]] return list(boto_instances) mock_ec2mgr.return_value.find.side_effect = _find # set-up redis mock to return price data and image name def _mock_redis_get(key): if ":blacklist:" in key: return None if ":price:" in key: return '{"redmond": {"mshq": [0.005]}}' if ":image:" in key: return 'warp' raise UncatchableException("unhandle key in mock_get(): %s" % (key,)) mock_redis.return_value.get = Mock(side_effect=_mock_redis_get) # create database state config = create_config(name='config #1', size=4, cycle_interval=3600, ec2_key_name='fredsRefurbishedSshKey', ec2_security_groups='mostlysecure', ec2_instance_types=['80286'], ec2_image_name='os/2', ec2_allowed_regions=['redmond'], ec2_max_price='0.1', userdata=b'cleverscript') pool = create_pool(config=config, last_cycled=timezone.now() - datetime.timedelta(seconds=300), enabled=True) create_instance(None, pool=pool, status_code=INSTANCE_STATE['running'], created=timezone.now() - datetime.timedelta(seconds=100), ec2_instance_id='i-123', ec2_region="redmond", ec2_zone="mshq", size=2) create_instance(None, pool=pool, status_code=INSTANCE_STATE['running'], created=timezone.now() - datetime.timedelta(seconds=75), ec2_instance_id='i-456', ec2_region="redmond", ec2_zone="mshq", size=1) create_instance(None, pool=pool, status_code=INSTANCE_STATE['running'], created=timezone.now() - datetime.timedelta(seconds=50), ec2_instance_id='i-789', ec2_region="redmond", ec2_zone="mshq", size=2) # call function under test check_instance_pool(pool.id) # check that laniakea calls were made cluster = mock_ec2mgr.return_value assert {call[0] for call in cluster.method_calls} == {'connect', 'find', 'terminate'} cluster.terminate.assert_called_once_with([boto_instances[1]])
def test_bad_config(): """invalid configs create a pool status entry""" config = create_config(name='config #1') pool = create_pool(config=config) with pytest.raises(UncatchableException, match=r'Configuration error \(missing: '): check_instance_pool(pool.id) assert not Instance.objects.exists() config2 = create_config(name='config #2', parent=config) config.parent = config2 config.save() with pytest.raises(UncatchableException, match=r'Configuration error \(cyclic\)'): check_instance_pool(pool.id) assert not Instance.objects.exists()
def test_spot_instance_blacklist(mock_redis, mock_ec2mgr): """check that spot requests being cancelled will result in temporary blacklisting""" # ensure EC2Manager returns a request ID class _MockReq(boto.ec2.spotinstancerequest.SpotInstanceRequest): def __init__(self, *args, **kwds): super(_MockReq, self).__init__(*args, **kwds) self.state = 'cancelled' req = _MockReq() req.launch_specification = Mock() req.launch_specification.instance_type = '80286' mock_ec2mgr.return_value.check_spot_requests.return_value = (req,) # set-up redis mock to return price data and image name def _mock_redis_get(key): if ":blacklist:" in key: return True if ":price:" in key: return '{"redmond": {"mshq": [0.001]}}' if ":image:" in key: return 'warp' raise UncatchableException("unhandle key in mock_get(): %s" % (key,)) def _mock_redis_set(key, value, ex=None): assert ":blacklist:mshq:" in key mock_redis.return_value.get = Mock(side_effect=_mock_redis_get) mock_redis.return_value.set = Mock(side_effect=_mock_redis_set) # create database state config = create_config(name='config #1', size=1, cycle_interval=3600, ec2_key_name='fredsRefurbishedSshKey', ec2_security_groups='mostlysecure', ec2_instance_types=['80286'], ec2_image_name='os/2', ec2_allowed_regions=['redmond'], ec2_max_price='0.1', userdata=b'cleverscript') pool = create_pool(config=config, enabled=True) create_instance(None, pool=pool, status_code=INSTANCE_STATE['requested'], ec2_instance_id='req123', ec2_region="redmond", ec2_zone="mshq") # call function under test # XXX: this message is inaccurate .. there were no allowed regions at all with pytest.raises(UncatchableException, match="No allowed regions was cheap enough to spawn instances"): check_instance_pool(pool.id) # check that laniakea calls were made cluster = mock_ec2mgr.return_value assert {call[0] for call in cluster.method_calls} == {'connect', 'check_spot_requests', 'find'} # check that instance was updated assert not Instance.objects.exists() # check that blacklist was set in redis assert len(mock_redis.return_value.set.mock_calls) == 1
def test_pool_disabled(mock_redis, mock_ec2mgr): """check that pool disabled results in running and pending instances being terminated""" # ensure EC2Manager returns a request ID class _MockInstance(boto.ec2.instance.Instance): @property def state_code(self): return INSTANCE_STATE['running'] boto_instance = _MockInstance() boto_instance.id = 'i-123' boto_instance.public_dns_name = 'fm-test1.fuzzing.allizom.com' mock_ec2mgr.return_value.find.return_value = (boto_instance,) mock_ec2mgr.return_value.check_spot_requests.return_value = (None,) # set-up redis mock to return price data and image name def _mock_redis_get(key): if ":blacklist:" in key: return None if ":price:" in key: return '{"redmond": {"mshq": [0.005]}}' if ":image:" in key: return 'warp' raise UncatchableException("unhandle key in mock_get(): %s" % (key,)) mock_redis.return_value.get = Mock(side_effect=_mock_redis_get) # ensure EC2Manager returns a request ID mock_ec2mgr.return_value.create_spot_requests.return_value = ('req123', 'req456') # create database state config = create_config(name='config #1', size=2, cycle_interval=3600, ec2_key_name='fredsRefurbishedSshKey', ec2_security_groups='mostlysecure', ec2_instance_types=['80286'], ec2_image_name='os/2', ec2_allowed_regions=['redmond'], ec2_max_price='0.1', userdata=b'cleverscript') pool = create_pool(config=config) create_instance(None, pool=pool, status_code=INSTANCE_STATE['running'], ec2_instance_id='i-123', ec2_region="redmond", ec2_zone="mshq") create_instance(None, pool=pool, status_code=INSTANCE_STATE['requested'], ec2_instance_id='r-456', ec2_region="redmond", ec2_zone="mshq") # call function under test check_instance_pool(pool.id) # check that laniakea calls were made cluster = mock_ec2mgr.return_value assert {call[0] for call in cluster.method_calls} == {'connect', 'find', 'terminate', 'check_spot_requests', 'cancel_spot_requests'} cluster.terminate.assert_called_once_with((boto_instance,)) cluster.cancel_spot_requests.assert_called_once_with(['r-456'])
def test_fulfilled_spot_instance(mock_ec2mgr): """spot instance requests are turned into instances when fulfilled""" # ensure EC2Manager returns a request ID class _MockInstance(boto.ec2.instance.Instance): def __init__(self, *args, **kwds): super(_MockInstance, self).__init__(*args, **kwds) self._test_tags = {} @property def state_code(self): return INSTANCE_STATE['running'] def add_tags(self, tags, dry_run=False): self._test_tags.update(tags) boto_instance = _MockInstance() boto_instance.id = 'i-123' boto_instance.public_dns_name = 'fm-test.fuzzing.allizom.com' mock_ec2mgr.return_value.check_spot_requests.return_value = (boto_instance,) # create database state config = create_config(name='config #1', size=1, cycle_interval=3600, ec2_key_name='fredsRefurbishedSshKey', ec2_security_groups='mostlysecure', ec2_instance_types=['80286'], ec2_image_name='os/2', ec2_allowed_regions=['redmond'], ec2_max_price='0.1', userdata=b'cleverscript') pool = create_pool(config=config, enabled=True) orig = create_instance(None, pool=pool, status_code=INSTANCE_STATE['requested'], ec2_instance_id='req123', ec2_region="redmond", ec2_zone="mshq") # call function under test check_instance_pool(pool.id) # check that laniakea calls were made cluster = mock_ec2mgr.return_value assert {call[0] for call in cluster.method_calls} == {'connect', 'check_spot_requests', 'find'} # check that instance was updated instance = Instance.objects.get() assert instance.id == orig.id assert instance.hostname == 'fm-test.fuzzing.allizom.com' assert instance.status_code == INSTANCE_STATE["running"] assert instance.instance_id == 'i-123' assert boto_instance._test_tags == {'SpotManager-Updatable': '1'} # pylint: disable=protected-access
def test_create_instance(mock_redis, mock_ec2mgr): """spot instance requests are created when required""" # set-up redis mock to return price data and image name def _mock_redis_get(key): if ":blacklist:mshq:" in key: return True if ":blacklist:" in key: return None if ":price:" in key: return '{"redmond": {"mshq": [0.005]}, "toronto": {"markham": [0.01]}}' if ":image:" in key: return 'warp' raise UncatchableException("unhandle key in mock_get(): %s" % (key,)) mock_redis.return_value.get = Mock(side_effect=_mock_redis_get) # ensure EC2Manager returns a request ID mock_ec2mgr.return_value.create_spot_requests.return_value = ('req123',) # create database state config = create_config(name='config #1', size=1, cycle_interval=3600, ec2_key_name='fredsRefurbishedSshKey', ec2_security_groups='mostlysecure', ec2_instance_types=['80286'], ec2_image_name='os/2', ec2_allowed_regions=['redmond', 'toronto'], ec2_max_price='0.1', userdata=b'cleverscript') pool = create_pool(config=config, enabled=True) # call function under test check_instance_pool(pool.id) # check that laniakea calls were made cluster = mock_ec2mgr.return_value assert {call[0] for call in cluster.method_calls} == {'connect', 'create_spot_requests'} # check that pending instances were created instance = Instance.objects.get() assert instance.region == 'toronto' assert instance.zone == 'markham' assert instance.status_code == INSTANCE_STATE["requested"] assert instance.pool.id == pool.id assert instance.size == 1 assert instance.instance_id == 'req123'
def test_instance_shutting_down(mock_redis, mock_ec2mgr): """instances are replaced when shut down or terminated""" # ensure EC2Manager returns a request ID class _MockInstance(boto.ec2.instance.Instance): @property def state_code(self): return INSTANCE_STATE['shutting-down'] def add_tags(self, _tags, _dry_run=False): pass class _MockInstance2(_MockInstance): @property def state_code(self): return INSTANCE_STATE['terminated'] boto_instance1 = _MockInstance() boto_instance1.id = 'i-123' boto_instance1.public_dns_name = 'fm-test1.fuzzing.allizom.com' boto_instance2 = _MockInstance2() boto_instance2.id = 'i-456' boto_instance2.public_dns_name = 'fm-test2.fuzzing.allizom.com' mock_ec2mgr.return_value.find.return_value = (boto_instance1, boto_instance2) # set-up redis mock to return price data and image name def _mock_redis_get(key): if ":blacklist:" in key: return None if ":price:" in key: return '{"redmond": {"mshq": [0.005]}}' if ":image:" in key: return 'warp' raise UncatchableException("unhandle key in mock_get(): %s" % (key,)) mock_redis.return_value.get = Mock(side_effect=_mock_redis_get) # ensure EC2Manager returns a request ID mock_ec2mgr.return_value.create_spot_requests.return_value = ('req123', 'req456') # create database state config = create_config(name='config #1', size=2, cycle_interval=3600, ec2_key_name='fredsRefurbishedSshKey', ec2_security_groups='mostlysecure', ec2_instance_types=['80286'], ec2_image_name='os/2', ec2_allowed_regions=['redmond'], ec2_max_price='0.1', userdata=b'cleverscript') pool = create_pool(config=config, enabled=True) orig1 = create_instance(None, pool=pool, status_code=INSTANCE_STATE['running'], ec2_instance_id='i-123', ec2_region="redmond", ec2_zone="mshq") orig2 = create_instance(None, pool=pool, status_code=INSTANCE_STATE['running'], ec2_instance_id='i-456', ec2_region="redmond", ec2_zone="mshq") # call function under test check_instance_pool(pool.id) # check that laniakea calls were made cluster = mock_ec2mgr.return_value assert {call[0] for call in cluster.method_calls} == {'connect', 'create_spot_requests', 'find'} # check that instances were replaced remaining = {'req123', 'req456'} for new in Instance.objects.all(): assert new.id not in {orig1.id, orig2.id} assert new.pool.id == pool.id assert new.size == 1 assert new.status_code == INSTANCE_STATE["requested"] remaining.remove(new.instance_id)