def test_boto3_connection_kwargs_profile(self):
m_creds = Mock()
type(m_creds).access_key = 'ak'
type(m_creds).secret_key = 'sk'
type(m_creds).token = 'tkn'
mock_session = Mock()
m_sess = Mock()
m_sess.get_credentials.return_value = m_creds
type(mock_session)._session = m_sess
with patch('%s._get_sts_token' % pb) as mock_get_sts:
with patch('%s.logger' % pbm) as mock_logger:
with patch('%s.boto3.Session' % pbm) as mock_sess:
with patch.dict('%s._services' % pbm, {}, clear=True):
mock_sess.return_value = mock_session
cls = AwsLimitChecker(profile_name='myprof')
mock_get_sts.reset_mock()
mock_logger.reset_mock()
mock_sess.reset_mock()
res = cls._boto_conn_kwargs
assert mock_get_sts.mock_calls == []
assert mock_logger.mock_calls == [
call.debug('Using credentials profile: %s', 'myprof')
]
assert mock_sess.mock_calls == [call(profile_name='myprof')]
assert res == {
'region_name': None,
'aws_access_key_id': 'ak',
'aws_secret_access_key': 'sk',
'aws_session_token': 'tkn'
}
def test_init_sts_external_id_ta_refresh(self):
mock_svc1 = Mock(spec_set=_AwsService)
mock_svc2 = Mock(spec_set=_AwsService)
mock_foo = Mock(spec_set=_AwsService)
mock_bar = Mock(spec_set=_AwsService)
mock_ta = Mock(spec_set=TrustedAdvisor)
mock_foo.return_value = mock_svc1
mock_bar.return_value = mock_svc2
svcs = {'SvcFoo': mock_foo, 'SvcBar': mock_bar}
with patch('%s.boto3' % pbm) as mock_boto:
mock_boto.client.return_value.assume_role.return_value = {
'Credentials': {
'AccessKeyId': 'akid',
'SecretAccessKey': 'sk',
'SessionToken': 'stoken',
'Expiration': '0'
},
'AssumedRoleUser': {
'AssumedRoleId': 'arid',
'Arn': 'arn'
}
}
with patch.dict('%s._services' % pbm, values=svcs, clear=True):
with patch.multiple(
'awslimitchecker.checker',
logger=DEFAULT,
_get_version_info=DEFAULT,
TrustedAdvisor=DEFAULT,
_get_latest_version=DEFAULT,
autospec=True,
) as mocks:
mock_version = mocks['_get_version_info']
mock_version.return_value = self.mock_ver_info
mocks['TrustedAdvisor'].return_value = mock_ta
mocks['_get_latest_version'].return_value = None
cls = AwsLimitChecker(account_id='123456789012',
account_role='myrole',
region='myregion',
external_id='myextid',
mfa_serial_number=123,
mfa_token=456,
ta_refresh_mode=123,
ta_refresh_timeout=456,
role_partition='mypart')
# dict should be of _AwsService instances
services = {'SvcFoo': mock_svc1, 'SvcBar': mock_svc2}
assert cls.services == services
assert mock_svc1.mock_calls == []
assert mock_svc2.mock_calls == []
assert self.mock_version.mock_calls == [call()]
assert self.cls.vinfo == self.mock_ver_info
assert mock_boto.mock_calls == [
call.client('sts', region_name='myregion'),
call.client().assume_role(
ExternalId='myextid',
RoleArn='arn:mypart:iam::123456789012:role/myrole',
RoleSessionName='awslimitchecker',
SerialNumber=123,
TokenCode=456)
]
def test_init_sts(self):
mock_svc1 = Mock(spec_set=_AwsService)
mock_svc2 = Mock(spec_set=_AwsService)
mock_foo = Mock(spec_set=_AwsService)
mock_bar = Mock(spec_set=_AwsService)
mock_ta = Mock(spec_set=TrustedAdvisor)
mock_foo.return_value = mock_svc1
mock_bar.return_value = mock_svc2
svcs = {'SvcFoo': mock_foo, 'SvcBar': mock_bar}
with patch('%s.boto3' % pbm):
with patch.dict('%s._services' % pbm, values=svcs, clear=True):
with patch.multiple(
'awslimitchecker.checker',
logger=DEFAULT,
_get_version_info=DEFAULT,
TrustedAdvisor=DEFAULT,
_get_latest_version=DEFAULT,
autospec=True,
) as mocks:
mock_version = mocks['_get_version_info']
mock_version.return_value = self.mock_ver_info
mocks['TrustedAdvisor'].return_value = mock_ta
mocks['_get_latest_version'].return_value = None
cls = AwsLimitChecker(account_id='123456789012',
account_role='myrole',
region='myregion')
# dict should be of _AwsService instances
services = {'SvcFoo': mock_svc1, 'SvcBar': mock_svc2}
assert cls.services == services
assert mock_svc1.mock_calls == []
assert mock_svc2.mock_calls == []
assert self.mock_version.mock_calls == [call()]
assert self.cls.vinfo == self.mock_ver_info
def test_boto3_connection_kwargs_sts(self):
mock_creds = Mock()
type(mock_creds).access_key = 'sts_ak'
type(mock_creds).secret_key = 'sts_sk'
type(mock_creds).session_token = 'sts_token'
with patch('%s._get_sts_token' % pb) as mock_get_sts:
with patch('%s.logger' % pbm) as mock_logger:
with patch('%s.boto3.Session' % pbm) as mock_sess:
with patch.dict('%s._services' % pbm, {}, clear=True):
cls = AwsLimitChecker(account_id='123',
account_role='myrole',
region='myregion')
mock_get_sts.return_value = mock_creds
mock_get_sts.reset_mock()
mock_logger.reset_mock()
mock_sess.reset_mock()
res = cls._boto_conn_kwargs
assert mock_get_sts.mock_calls == [call()]
assert mock_logger.mock_calls == [
call.debug(
"Connecting for account %s role '%s' with STS "
"(region: %s)", '123', 'myrole', 'myregion')
]
assert mock_sess.mock_calls == []
assert res == {
'region_name': 'myregion',
'aws_access_key_id': 'sts_ak',
'aws_secret_access_key': 'sts_sk',
'aws_session_token': 'sts_token'
}
def setup(self):
self.mock_ver_info = Mock(release='1.2.3',
url='http://myurl',
commit='abcd',
tag='mytag',
version_str='1.2.3@mytag')
self.mock_svc1 = Mock(spec_set=_AwsService)
self.mock_svc2 = Mock(spec_set=ApiServiceSpec)
self.mock_foo = Mock(spec_set=_AwsService)
self.mock_bar = Mock(spec_set=_AwsService)
self.mock_ta = Mock(spec_set=TrustedAdvisor)
self.mock_foo.return_value = self.mock_svc1
self.mock_bar.return_value = self.mock_svc2
self.svcs = {'SvcFoo': self.mock_foo, 'SvcBar': self.mock_bar}
with patch.dict('%s._services' % pbm, values=self.svcs, clear=True):
with patch.multiple(
'awslimitchecker.checker',
logger=DEFAULT,
_get_version_info=DEFAULT,
TrustedAdvisor=DEFAULT,
_get_latest_version=DEFAULT,
ServiceQuotasClient=DEFAULT,
autospec=True,
) as mocks:
self.mock_logger = mocks['logger']
self.mock_version = mocks['_get_version_info']
self.mock_ta_constr = mocks['TrustedAdvisor']
self.mock_glv = mocks['_get_latest_version']
self.mock_quotas = mocks['ServiceQuotasClient']
mocks['TrustedAdvisor'].return_value = self.mock_ta
mocks['_get_latest_version'].return_value = None
self.mock_version.return_value = self.mock_ver_info
self.cls = AwsLimitChecker(check_version=False)
def check_limits(self, verbose=False):
"""
Run the actual usage and limit check, with overrides.
see: http://awslimitchecker.readthedocs.org/en/latest/python_usage.html#ci-deployment-checks
"""
# instantiate the class
checker = AwsLimitChecker()
# set your overrides
checker.set_threshold_overrides(AWS_THRESHOLD_OVERRIDES)
checker.set_limit_overrides(AWS_LIMIT_OVERRIDES)
print("Checking AWS resource usage; WARNING threshold {w}% of "
"limit, CRITICAL threshold {c}% of limit".format(
w=checker.warning_threshold, c=checker.critical_threshold))
# check usage against thresholds
# if we didn't support verbose output, we could just iterate the return
# value of this to be a bit more efficient.
checker.check_thresholds()
# save state for exit code and summary
warnings = []
criticals = []
# iterate the results
for service, svc_limits in sorted(checker.get_limits().items()):
for limit_name, limit in sorted(svc_limits.items()):
have_alarms = False
# check warnings and criticals for each Limit
for warn in limit.get_warnings():
warnings.append("{service} '{limit_name}' usage "
"({u}) exceeds warning threshold "
"(limit={l})".format(
service=service,
limit_name=limit_name,
u=str(warn),
l=limit.get_limit(),
))
have_alarms = True
for crit in limit.get_criticals():
criticals.append("{service} '{limit_name}' usage "
"({u}) exceeds critical threshold"
" (limit={l})".format(
service=service,
limit_name=limit_name,
u=str(crit),
l=limit.get_limit(),
))
have_alarms = True
if not have_alarms and verbose:
print(
"{service} '{limit_name}' OK: {u} (limit={l})".format(
service=service,
limit_name=limit_name,
u=limit.get_current_usage_str(),
l=limit.get_limit()))
if verbose:
print("\n\n")
return (warnings, criticals)
def test_init_region_profile(self):
mock_svc1 = Mock(spec_set=_AwsService)
mock_svc2 = Mock(spec_set=_AwsService)
mock_foo = Mock(spec_set=_AwsService)
mock_bar = Mock(spec_set=_AwsService)
mock_ta = Mock(spec_set=TrustedAdvisor)
mock_foo.return_value = mock_svc1
mock_bar.return_value = mock_svc2
svcs = {'SvcFoo': mock_foo, 'SvcBar': mock_bar}
with patch('%s.boto3' % pbm) as mock_boto3:
with patch.dict('%s._services' % pbm, values=svcs, clear=True):
with patch.multiple(
'awslimitchecker.checker',
logger=DEFAULT,
_get_version_info=DEFAULT,
TrustedAdvisor=DEFAULT,
autospec=True,
) as mocks:
mock_boto3.Session.return_value._session = Mock()
mock_version = mocks['_get_version_info']
mock_version.return_value = self.mock_ver_info
mocks['TrustedAdvisor'].return_value = mock_ta
cls = AwsLimitChecker(region='regionX', profile_name='foo')
# dict should be of _AwsService instances
services = {
'SvcFoo': mock_svc1,
'SvcBar': mock_svc2
}
assert cls.profile_name == 'foo'
assert cls.region == 'regionX'
assert cls.services == services
assert mock_svc1.mock_calls == []
assert mock_svc2.mock_calls == []
assert self.mock_version.mock_calls == [call()]
assert self.cls.vinfo == self.mock_ver_info
def test_init_sts_external_id(self):
mock_svc1 = Mock(spec_set=_AwsService)
mock_svc2 = Mock(spec_set=_AwsService)
mock_foo = Mock(spec_set=_AwsService)
mock_bar = Mock(spec_set=_AwsService)
mock_ta = Mock(spec_set=TrustedAdvisor)
mock_foo.return_value = mock_svc1
mock_bar.return_value = mock_svc2
svcs = {'SvcFoo': mock_foo, 'SvcBar': mock_bar}
with patch.dict('awslimitchecker.checker._services',
values=svcs, clear=True):
with patch.multiple(
'awslimitchecker.checker',
logger=DEFAULT,
_get_version_info=DEFAULT,
TrustedAdvisor=DEFAULT,
autospec=True,
) as mocks:
mock_version = mocks['_get_version_info']
mock_version.return_value = self.mock_ver_info
mock_ta_constr = mocks['TrustedAdvisor']
mocks['TrustedAdvisor'].return_value = mock_ta
cls = AwsLimitChecker(
account_id='123456789012',
account_role='myrole',
region='myregion',
external_id='myextid',
mfa_serial_number=None,
mfa_token=None
)
# dict should be of _AwsService instances
assert cls.services == {
'SvcFoo': mock_svc1,
'SvcBar': mock_svc2
}
# _AwsService instances should exist, but have no other calls
assert mock_foo.mock_calls == [
call(80, 99, '123456789012', 'myrole', 'myregion', 'myextid',
None, None)
]
assert mock_bar.mock_calls == [
call(80, 99, '123456789012', 'myrole', 'myregion', 'myextid',
None, None)
]
assert mock_ta_constr.mock_calls == [
call(
account_id='123456789012',
account_role='myrole',
region='myregion',
external_id='myextid',
mfa_serial_number=None,
mfa_token=None
)
]
assert mock_svc1.mock_calls == []
assert mock_svc2.mock_calls == []
assert self.mock_version.mock_calls == [call()]
assert self.cls.vinfo == self.mock_ver_info
def scrape_limits(event, context):
data = base64.b64decode(os.environ.get('CONFIG_DATA_BASE64'))
fd = io.StringIO(data.decode('utf-8'))
config = yaml.safe_load(fd)
if config is None:
log.error(
f'Unable to read configuration file "{CONFIG_NAME}" exiting...')
sys.exit(1)
alarm_actions = os.environ.get('ALARM_ACTIONS', "").split(',')
alarm_actions = list(filter(lambda a: a != '', alarm_actions))
limits = []
for service in config.get('services', []):
for limit in service.get('limits', []):
try:
limits.append(Limit(service.get('name'), limit))
except InvalidObjectInConfig as e:
log.error(e)
c = AwsLimitChecker(ta_refresh_mode=config.get('ta_refresh_mode', 21600),
ta_refresh_timeout=config.get('ta_refresh_timeout',
1800))
for limit in limits:
limit.override(c)
c.remove_services(config.get('skip', []))
cwc = CloudWatchClient(METRICS_NAMESPACE)
metrics = []
alarms = []
results = c.get_limits()
log.info('----------------------------------------------')
for service, limits in results.items():
for limit, value in limits.items():
log.info(
f'service={service} limit={limit} usage={value.get_current_usage_str()}'
)
for usage in value.get_current_usage():
metric = Metric(value, usage)
metrics.append(metric)
if not value.has_resource_limits():
log.warning(f'Limit {value.name} has no limit')
else:
alarms.append(
Alarm(value, 'warn', metric.dimensions, alarm_actions))
alarms.append(
Alarm(value, 'crit', metric.dimensions, alarm_actions))
cwc.put_metric_data(metrics)
cwc.put_metric_alarms(alarms)
return {'message': 'Done'}
def test_boto3_connection_kwargs(self):
cls = AwsLimitChecker()
with patch('%s._get_sts_token' % pb) as mock_get_sts:
with patch('%s.logger' % pbm) as mock_logger:
with patch('%s.boto3.Session' % pbm) as mock_sess:
res = cls._boto_conn_kwargs
assert mock_get_sts.mock_calls == []
assert mock_logger.mock_calls == [
call.debug('Connecting to region %s', None)
]
assert mock_sess.mock_calls == []
assert res == {'region_name': None}
def test_init_thresholds(self):
mock_svc1 = Mock(spec_set=_AwsService)
mock_svc2 = Mock(spec_set=_AwsService)
mock_foo = Mock(spec_set=_AwsService)
mock_bar = Mock(spec_set=_AwsService)
mock_ta = Mock(spec_set=TrustedAdvisor)
mock_foo.return_value = mock_svc1
mock_bar.return_value = mock_svc2
svcs = {'SvcFoo': mock_foo, 'SvcBar': mock_bar}
with patch.dict('awslimitchecker.checker._services',
values=svcs,
clear=True):
with patch.multiple(
'awslimitchecker.checker',
logger=DEFAULT,
_get_version_info=DEFAULT,
TrustedAdvisor=DEFAULT,
autospec=True,
) as mocks:
mock_version = mocks['_get_version_info']
mock_version.return_value = self.mock_ver_info
mock_ta_constr = mocks['TrustedAdvisor']
mocks['TrustedAdvisor'].return_value = mock_ta
cls = AwsLimitChecker(
warning_threshold=5,
critical_threshold=22,
)
# dict should be of _AwsService instances
services = {'SvcFoo': mock_svc1, 'SvcBar': mock_svc2}
assert cls.services == services
# _AwsService instances should exist, but have no other calls
assert mock_foo.mock_calls == [
call(5, 22, None, None, None, None, None, None)
]
assert mock_bar.mock_calls == [
call(5, 22, None, None, None, None, None, None)
]
assert mock_ta_constr.mock_calls == [
call(services,
account_id=None,
account_role=None,
region=None,
external_id=None,
mfa_serial_number=None,
mfa_token=None)
]
assert mock_svc1.mock_calls == []
assert mock_svc2.mock_calls == []
assert self.mock_version.mock_calls == [call()]
assert self.cls.vinfo == self.mock_ver_info
def build_docs():
"""
Trigger rebuild of all documentation that is dynamically generated
from awslimitchecker.
"""
if os.environ.get('CI', None) is not None:
print("Not building dynamic docs in CI environment")
raise SystemExit(0)
logger.info("Beginning build of dynamically-generated docs")
logger.info("Instantiating AwsLimitChecker")
c = AwsLimitChecker()
build_iam_policy(c)
build_limits(c)
build_runner_examples()
def main():
port = int(os.environ.get('ALC_PORT', '8080'))
interval = int(os.environ.get('ALC_INTERVAL', '60'))
logger = logging.getLogger()
logger.setLevel(logging.ERROR)
checkers = {}
for region in ['us-east-1', 'us-east-2', 'us-west-1', 'us-west-2']:
checkers[region] = AwsLimitChecker(region=region)
start_http_server(port)
for region, checker in checkers.items():
update(checker, region)
time.sleep(interval)
def test_init_thresholds(self):
mock_svc1 = Mock(spec_set=_AwsService)
mock_svc2 = Mock(spec_set=_AwsService)
mock_foo = Mock(spec_set=_AwsService)
mock_bar = Mock(spec_set=_AwsService)
mock_ta = Mock(spec_set=TrustedAdvisor)
mock_foo.return_value = mock_svc1
mock_bar.return_value = mock_svc2
svcs = {'SvcFoo': mock_foo, 'SvcBar': mock_bar}
with patch.dict('%s._services' % pbm, values=svcs, clear=True):
with patch.multiple(
'awslimitchecker.checker',
logger=DEFAULT,
_get_version_info=DEFAULT,
TrustedAdvisor=DEFAULT,
_get_latest_version=DEFAULT,
ServiceQuotasClient=DEFAULT,
autospec=True,
) as mocks:
mock_version = mocks['_get_version_info']
mock_version.return_value = self.mock_ver_info
mock_ta_constr = mocks['TrustedAdvisor']
mocks['TrustedAdvisor'].return_value = mock_ta
mocks['_get_latest_version'].return_value = None
cls = AwsLimitChecker(
warning_threshold=5,
critical_threshold=22,
)
# dict should be of _AwsService instances
services = {'SvcFoo': mock_svc1, 'SvcBar': mock_svc2}
assert cls.services == services
# _AwsService instances should exist, but have no other calls
assert mock_foo.mock_calls == [
call(5, 22, {'region_name': None},
mocks['ServiceQuotasClient'].return_value)
]
assert mock_bar.mock_calls == [
call(5, 22, {'region_name': None},
mocks['ServiceQuotasClient'].return_value)
]
assert mock_ta_constr.mock_calls == [
call(services, {'region_name': None},
ta_api_region='us-east-1',
ta_refresh_mode=None,
ta_refresh_timeout=None)
]
assert mock_svc1.mock_calls == []
assert mock_svc2.mock_calls == []
assert self.mock_version.mock_calls == [call()]
assert self.cls.vinfo == self.mock_ver_info
def limits_for_ec2():
limit_info = '.. _limits.EC2:\n\n'
limit_info += "EC2\n---\n\n"
limit_info += dedent("""
As of October 2019, the "standard" EC2 regions use the new
`vCPU-based limits <https://aws.amazon.com/blogs/compute/preview-vcpu-based-
instance-limits/>`__, while the China (``cn-``) and GovCloud (``us-gov-``)
regions still use the old per-instance-type limits. Please see the sections
for either :ref:`limits.ec2-standard` or :ref:`limits.ec2-nonvcpu` for
details.
""")
limit_info += '.. _limits.ec2-standard:\n\n'
limit_info += "EC2 - Standard Regions\n"
limit_info += "----------------------\n"
limit_info += "\n" + dedent("""
**Note on On-Demand vs Reserved Instances:** The EC2 limits for
"Running On-Demand" EC2 Instances apply only to On-Demand instances,
not Reserved Instances. If you list all EC2 instances that are
running in the Console or API, you'll get back instances of all types
(On-Demand, Reserved, etc.). The value that awslimitchecker reports
for Running On-Demand Instances current usage will *not* match the
number of instances you see in the Console or API.
**Important:** The limits for **Running On-Demand Instances** are now
measured in vCPU count per instance family, not instance count per instance
type.
""") + "\n"
limit_info += "\n"
limit_info += format_limits_for_service(
AwsLimitChecker(region='us-east-1').get_limits()['EC2'])
limit_info += '.. _limits.ec2-nonvcpu:\n\n'
limit_info += "EC2 - China and GovCloud\n"
limit_info += "------------------------\n"
limit_info += "\n" + dedent("""
**Note on On-Demand vs Reserved Instances:** The EC2 limits for
"Running On-Demand" EC2 Instances apply only to On-Demand instances,
not Reserved Instances. If you list all EC2 instances that are
running in the Console or API, you'll get back instances of all types
(On-Demand, Reserved, etc.). The value that awslimitchecker reports
for Running On-Demand Instances current usage will *not* match the
number of instances you see in the Console or API.
""") + "\n"
limit_info += "\n"
fname = os.path.join(my_dir, 'source', 'ec2_nonvcpu_limits.txt')
with open(fname, 'r') as fh:
limit_info += fh.read()
limit_info += "\n\n"
return limit_info
def test_check_version_not_old(self):
with patch.multiple(
'awslimitchecker.checker',
logger=DEFAULT,
_get_version_info=DEFAULT,
TrustedAdvisor=DEFAULT,
_get_latest_version=DEFAULT,
autospec=True,
) as mocks:
mocks['_get_version_info'].return_value = self.mock_ver_info
mocks['_get_latest_version'].return_value = None
AwsLimitChecker()
assert mocks['_get_latest_version'].mock_calls == [call()]
assert mocks['logger'].mock_calls == [
call.debug('Connecting to region %s', None)
]
def test_init_region_profile_role_partition_ta_region(self):
mock_svc1 = Mock(spec_set=_AwsService)
mock_svc2 = Mock(spec_set=_AwsService)
mock_foo = Mock(spec_set=_AwsService)
mock_bar = Mock(spec_set=_AwsService)
mock_ta = Mock(spec_set=TrustedAdvisor)
mock_foo.return_value = mock_svc1
mock_bar.return_value = mock_svc2
svcs = {'SvcFoo': mock_foo, 'SvcBar': mock_bar}
with patch('%s.boto3' % pbm) as mock_boto3:
with patch.dict('%s._services' % pbm, values=svcs, clear=True):
with patch.multiple(
'awslimitchecker.checker',
logger=DEFAULT,
_get_version_info=DEFAULT,
TrustedAdvisor=DEFAULT,
_get_latest_version=DEFAULT,
autospec=True,
) as mocks:
mock_boto3.Session.return_value._session = Mock()
mock_version = mocks['_get_version_info']
mock_version.return_value = self.mock_ver_info
mocks['TrustedAdvisor'].return_value = mock_ta
mocks['_get_latest_version'].return_value = None
with patch('%s._boto_conn_kwargs' % pb,
new_callable=PropertyMock) as m_bck:
m_bck.return_value = {'region_name': 'rName'}
cls = AwsLimitChecker(region='regionX',
profile_name='foo',
role_partition='rpName',
ta_api_region='taRegion')
# dict should be of _AwsService instances
services = {'SvcFoo': mock_svc1, 'SvcBar': mock_svc2}
assert cls.profile_name == 'foo'
assert cls.region == 'regionX'
assert cls.services == services
assert cls.role_partition == 'rpName'
assert mock_svc1.mock_calls == []
assert mock_svc2.mock_calls == []
assert self.mock_version.mock_calls == [call()]
assert self.cls.vinfo == self.mock_ver_info
assert mocks['TrustedAdvisor'].mock_calls == [
call(services, {'region_name': 'rName'},
ta_api_region='taRegion',
ta_refresh_mode=None,
ta_refresh_timeout=None)
]
def build_docs():
"""
Trigger rebuild of all documentation that is dynamically generated
from awslimitchecker.
"""
if os.environ.get('CI', None) is not None:
print("Not building dynamic docs in CI environment")
raise SystemExit(0)
region = os.environ.get('AWS_DEFAULT_REGION', None)
if region is None:
raise SystemExit("ERROR: Please export AWS_DEFAULT_REGION")
logger.info("Beginning build of dynamically-generated docs")
logger.info("Instantiating AwsLimitChecker")
c = AwsLimitChecker(region=region)
build_iam_policy(c)
build_limits(c)
build_runner_examples()
def test_check_version_old(self):
with patch.multiple(
'awslimitchecker.checker',
logger=DEFAULT,
_get_version_info=DEFAULT,
TrustedAdvisor=DEFAULT,
_get_latest_version=DEFAULT,
autospec=True,
) as mocks:
mocks['_get_version_info'].return_value = self.mock_ver_info
mocks['_get_latest_version'].return_value = '3.4.5'
AwsLimitChecker()
assert mocks['_get_latest_version'].mock_calls == [call()]
assert mocks['logger'].mock_calls == [
call.warning(
'You are running awslimitchecker %s, but the latest version'
' is %s; please consider upgrading.', '1.2.3', '3.4.5'),
call.debug('Connecting to region %s', None)
]
def get_iam_policy(self):
"""Return the current IAM policy as a json-serialized string"""
checker = AwsLimitChecker()
policy = checker.get_required_iam_policy()
return json.dumps(policy, sort_keys=True, indent=2)
def verify_usage(self, checker_args, creds, service_name, expect_usage,
allow_endpoint_error):
"""
This essentially replicates what's done when awslimitchecker is called
from the command line with ``-u``. This replicates some of the internal
logic of :py:class:`~awslimitchecker.runner.Runner`.
The main purpose is:
1. to allow passing in an existing
:py:class:`awslimitchecker.checker.Checker` instance for testing
the various authentication options, and,
2. to verify that at least some usage is found
This method is largely a duplication of
:py:meth:`~awslimitchecker.runner.Runner.show_usage`.
:param checker_args: dict of kwargs to pass to
:py:class:`awslimitchecker.checker.Checker` constructor
:type checker_args: dict
:param creds: AWS access key ID and secret key
:type creds: tuple
:param service_name: the Service name to test usage for; if None,
check for all.
:type service_name: str
:param expect_usage: whether or not to expect non-zero usage
:type expect_usage: bool
:param allow_endpoint_error: passed on to
:py:meth:`~.support.LogRecordHelper.unexpected_logs`
:type allow_endpoint_error: bool
"""
# clear the Connectable credentials
Connectable.credentials = None
# destroy boto3's session, so it creates a new one
boto3.DEFAULT_SESSION = None
# set the env vars to the creds we want
os.environ['AWS_ACCESS_KEY_ID'] = creds[0]
os.environ['AWS_SECRET_ACCESS_KEY'] = creds[1]
# this has to be generated inside the method, not in the method that
# yields it
if 'mfa_token' in checker_args:
checker_args['mfa_token'] = self.totp_code(creds[3])
# pytest-capturelog looked good, but won't work with our yielded
# test functions, per https://github.com/pytest-dev/pytest/issues/227
with LogCapture() as l:
checker = AwsLimitChecker(**checker_args)
checker.find_usage(service=service_name)
limits = checker.get_limits(service=service_name)
logs = LogRecordHelper(l)
have_usage = False
data = {}
for svc in sorted(limits.keys()):
for lim in sorted(limits[svc].keys()):
limit = limits[svc][lim]
data["{s}/{l}".format(
s=svc,
l=lim)] = '{v}'.format(v=limit.get_current_usage_str())
for usage in limit.get_current_usage():
if usage.get_value() != 0:
have_usage = True
# this is the normal Runner command line output
print(dict2cols(data))
if expect_usage:
assert have_usage is True
# ensure we didn't log anything at WARN or above, except possibly
# a TrustedAdvisor subscription required message
records = logs.unexpected_logs(
allow_endpoint_error=allow_endpoint_error)
assert len(records) == 0, "awslimitchecker emitted unexpected log " \
"messages at WARN or higher: \n%s" % "\n".join(records)
def __init__(self, boto_cfg = None):
""" Initialize this object AwsLimitChecker object.
boto_cfg: boto3 library configuration
"""
self.checker = AwsLimitChecker() if boto_cfg is None else AwsLimitChecker(region=boto_cfg.region_name)
def get_aws_limit_checker():
return AwsLimitChecker(region=settings.REGION_NAME,
account_id=settings.ACCOUNT_ID,
account_role=settings.ACCOUNT_ROLE)
def check_limits(self,
acct_id,
region_name,
role_name=None,
limit_overrides={},
threshold_overrides={}):
"""
Run the actual usage and limit check, with overrides, against a specific
account in a specific region, optionally assuming a role in the account
and optionally setting limit and/or threshold overrides.
Return a 2-tuple of lists, warning strings and critical strings.
see: http://awslimitchecker.readthedocs.org/en/latest/python_usage.html
:returns: 2-tuple of lists of strings, warnings and criticals
"""
# instantiate the class
if role_name is not None:
checker = AwsLimitChecker(account_id=acct_id,
region=region_name,
account_role=role_name)
else:
checker = AwsLimitChecker(region=region_name)
# set your overrides
if len(threshold_overrides) > 0:
checker.set_threshold_overrides(threshold_overrides)
if len(limit_overrides) > 0:
checker.set_limit_overrides(limit_overrides)
# check usage against thresholds
checker.check_thresholds()
# save state for exit code and summary
warnings = []
criticals = []
# iterate the results
for service, svc_limits in sorted(checker.get_limits().items()):
for limit_name, limit in sorted(svc_limits.items()):
# check warnings and criticals for each Limit
for warn in limit.get_warnings():
warnings.append(
colored(
"{service} '{limit_name}' usage "
"({u}) exceeds warning threshold "
"(limit={l})".format(
service=service,
limit_name=limit_name,
u=str(warn),
l=limit.get_limit(),
), 'yellow'))
for crit in limit.get_criticals():
criticals.append(
colored(
"{service} '{limit_name}' usage "
"({u}) exceeds critical threshold"
" (limit={l})".format(
service=service,
limit_name=limit_name,
u=str(crit),
l=limit.get_limit(),
), 'red'))
return warnings, criticals
def test_verify_limits(self, checker_args, creds_type, service_name,
use_ta, expect_api_source, allow_endpoint_error):
"""
This essentially replicates what's done when awslimitchecker is called
from the command line with ``-l``. This replicates some of the internal
logic of :py:class:`~awslimitchecker.runner.Runner`.
The main purpose is:
1. to allow passing in an existing
:py:class:`awslimitchecker.checker.Checker` instance for testing
the various authentication options, and,
2. to verify that at least some limits are found
This method is largely a duplication of
:py:meth:`~awslimitchecker.runner.Runner.list_limits`.
:param checker_args: dict of kwargs to pass to
:py:class:`awslimitchecker.checker.Checker` constructor
:type checker_args: dict
:param creds_type: Type of credentials to use; 'normal', 'sts', or
'sts_mfa'
:type creds_type: str
:param service_name: the Service name to test limits for; if None,
check for all.
:type service_name: str
:param use_ta: whether or not to use TrustedAdvisor
:type use_ta: bool
:param expect_api_source: whether or not to expect a limit with an
API source
:type expect_api_source: bool
:param allow_endpoint_error: passed on to
:py:meth:`~.support.LogRecordHelper.unexpected_logs`
:type allow_endpoint_error: bool
"""
# destroy boto3's session, so it creates a new one
boto3.DEFAULT_SESSION = None
# set the env vars to the creds we want
if creds_type == 'normal':
creds = self.normal_creds()
elif creds_type == 'sts':
creds = self.sts_creds()
elif creds_type == 'sts_mfa':
creds = self.sts_mfa_creds()
checker_args['mfa_serial_number'] = creds[2]
else:
raise RuntimeError("unknown creds type: '%s'" % creds_type)
os.environ['AWS_ACCESS_KEY_ID'] = creds[0]
os.environ['AWS_SECRET_ACCESS_KEY'] = creds[1]
# this has to be generated inside the method, not in the method that
# yields it
if 'mfa_token' in checker_args:
checker_args['mfa_token'] = self.totp_code(creds[3])
# pytest-capturelog looked good, but won't work with our yielded
# test functions, per https://github.com/pytest-dev/pytest/issues/227
with LogCapture() as l:
checker = AwsLimitChecker(**checker_args)
limits = checker.get_limits(use_ta=use_ta, service=service_name)
logs = LogRecordHelper(l)
have_api_source = False
data = {}
for svc in sorted(limits.keys()):
for lim in sorted(limits[svc].keys()):
src_str = ''
if limits[svc][lim].get_limit_source() == SOURCE_API:
have_api_source = True
src_str = ' (API)'
if limits[svc][lim].get_limit_source() == SOURCE_TA:
src_str = ' (TA)'
data["{s}/{l}".format(s=svc, l=lim)] = '{v}{t}'.format(
v=limits[svc][lim].get_limit(), t=src_str)
# check that we connected to the right region
logs.verify_region(checker_args.get('region', REGION))
# this is the normal Runner output
print(dict2cols(data))
if expect_api_source:
assert have_api_source is True
# ensure we didn't log anything at WARN or above, except possibly
# a TrustedAdvisor subscription required message
records = logs.unexpected_logs(
allow_endpoint_error=allow_endpoint_error)
assert len(records) == 0, "awslimitchecker emitted unexpected log " \
"messages at WARN or higher: \n%s" % "\n".join(records)
polls = logs.num_ta_polls
assert polls == 0, "awslimitchecker should have polled Trusted " \
"Advisor ZERO times, but polled %s times" % polls
logger = logging.getLogger('AWSlimitchecker')
hdlr = logging.FileHandler(
'/opt/zabbix/externalscripts/externalscriptsLogs/AWSlimitchecker/AWSlimitchecker.log'
)
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
hdlr.setFormatter(formatter)
logger.addHandler(hdlr)
logger.setLevel(logging.INFO)
#parameter for the initiator script#
# You can find the Project AwsLimitChecker in this link: https://github.com/jantman/awslimitchecker
region = args.myRegionVariable
print region
region = 'ca-central-1'
checker = AwsLimitChecker(region=region)
if region in problems_regions_dict.keys():
limit_path = str(problems_regions_dict[region]) + str(auto_complete)
with open('//opt//zabbix//externalscripts//AWSlimitchecker//%s' %
limit_path) as limit_conf:
problems_regions_dict[region] = json.load(limit_conf)
try:
checker.find_usage()
except:
pass
#print args.myRegionVariable
#print args.myZabbixHost
for aws_module, svc_limits in sorted(checker.get_limits().items()):
for limit_name, limit in sorted(svc_limits.items()):
if 'Firehose' not in aws_module:
try:
