def test_no_data_path(self):
func = Mock()
with pytest.raises(Exception) as excinfo:
paginate_dict(func, alc_marker_path=[])
ex_str = "alc_data_path must be specified for queries " \
"that return a dict."
assert ex_str in str(excinfo)
def test_no_data_path(self):
func = Mock()
with pytest.raises(Exception) as excinfo:
paginate_dict(func, alc_marker_path=[])
ex_str = "alc_data_path must be specified for queries " \
"that return a dict."
assert ex_str in str(excinfo)
def _find_usage_apis(self):
"""
Find usage on APIs / RestAPIs, and resources that are limited per-API.
Update `self.limits`.
"""
api_ids = []
logger.debug('Finding usage for APIs')
paginator = self.conn.get_paginator('get_rest_apis')
for resp in paginator.paginate():
for api in resp['items']:
api_ids.append(api['id'])
logger.debug('Found %d APIs', len(api_ids))
self.limits['APIs per account']._add_current_usage(
len(api_ids), aws_type='AWS::ApiGateway::RestApi')
# now the per-API limits...
warn_stages_paginated = None
logger.debug('Finding usage for per-API limits')
for api_id in api_ids:
res_count = 0
paginator = self.conn.get_paginator('get_resources')
for resp in paginator.paginate(restApiId=api_id):
res_count += len(resp['items'])
self.limits['Resources per API']._add_current_usage(
res_count,
resource_id=api_id,
aws_type='AWS::ApiGateway::Resource')
doc_parts = paginate_dict(self.conn.get_documentation_parts,
restApiId=api_id,
alc_marker_path=['position'],
alc_data_path=['items'],
alc_marker_param='position')
self.limits['Documentation parts per API']._add_current_usage(
len(doc_parts),
resource_id=api_id,
aws_type='AWS::ApiGateway::DocumentationPart')
# note that per the boto3 docs, there's no pagination of this...
stages = self.conn.get_stages(restApiId=api_id)
if len(set(stages.keys()) - set(['item', 'ResponseMetadata'])) > 0:
warn_stages_paginated = stages.keys()
self.limits['Stages per API']._add_current_usage(
len(stages['item']),
resource_id=api_id,
aws_type='AWS::ApiGateway::Stage')
authorizers = paginate_dict(self.conn.get_authorizers,
restApiId=api_id,
alc_marker_path=['position'],
alc_data_path=['items'],
alc_marker_param='position')
self.limits['Custom authorizers per API']._add_current_usage(
len(authorizers),
resource_id=api_id,
aws_type='AWS::ApiGateway::Authorizer')
if warn_stages_paginated is not None:
logger.warning(
'APIGateway get_stages returned more keys than present in '
'boto3 docs: %s', sorted(warn_stages_paginated))
def test_no_marker(self):
result = {'k1': {'k2': {'Data': []}}}
func = Mock()
func.return_value = result
res = paginate_dict(func,
alc_marker_path=['k1', 'k2', 'Marker'],
alc_data_path=['k1', 'k2', 'Data'],
alc_marker_param='Marker')
assert res == result
assert func.mock_calls == [call()]
def test_two_iterations(self):
e1 = Mock()
e2 = Mock()
e3 = Mock()
e4 = Mock()
e5 = Mock()
e6 = Mock()
func = Mock()
res1 = {
'k1': {
'k2': {
'Data': [e1, e2],
'Foo1': 'bar1',
'Marker': 'marker1'
}
}
}
res2 = {
'k1': {
'k2': {
'Data': [e3, e4],
'Foo2': 'bar2',
'Marker': 'marker2'
}
}
}
res3 = {'k1': {'k2': {'Data': [e5, e6], 'Foo3': 'bar3'}}}
expected = {
'k1': {
'k2': {
'Data': [e1, e2, e3, e4, e5, e6],
'Foo3': 'bar3'
}
}
}
func.side_effect = [res1, res2, res3]
res = paginate_dict(func,
'foo',
bar='baz',
alc_marker_path=['k1', 'k2', 'Marker'],
alc_data_path=['k1', 'k2', 'Data'],
alc_marker_param='MarkerParam')
assert res == expected
assert func.mock_calls == [
call('foo', bar='baz'),
call('foo', bar='baz', MarkerParam='marker1'),
call('foo', bar='baz', MarkerParam='marker2')
]
def test_bad_path(self):
result = {
'k1': {
'badpath': {}
}
}
func = Mock()
func.return_value = result
res = paginate_dict(
func,
alc_marker_path=['k1', 'k2', 'Marker'],
alc_data_path=['k1', 'k2', 'Data'],
alc_marker_param='Marker'
)
assert res == result
assert func.mock_calls == [call()]
def test_two_iterations(self):
e1 = Mock()
e2 = Mock()
e3 = Mock()
e4 = Mock()
e5 = Mock()
e6 = Mock()
func = Mock()
res1 = {
'k1': {
'k2': {
'Data': [e1, e2],
'Foo1': 'bar1',
'Marker': 'marker1'
}
}
}
res2 = {
'k1': {
'k2': {
'Data': [e3, e4],
'Foo2': 'bar2',
'Marker': 'marker2'
}
}
}
res3 = {
'k1': {
'k2': {
'Data': [e5, e6],
'Foo3': 'bar3'
}
}
}
expected = {
'k1': {
'k2': {
'Data': [e1, e2, e3, e4, e5, e6],
'Foo3': 'bar3'
}
}
}
func.side_effect = [res1, res2, res3]
res = paginate_dict(
func,
'foo',
bar='baz',
alc_marker_path=['k1', 'k2', 'Marker'],
alc_data_path=['k1', 'k2', 'Data'],
alc_marker_param='MarkerParam'
)
assert res == expected
assert func.mock_calls == [
call('foo', bar='baz'),
call(
'foo',
bar='baz',
MarkerParam='marker1'
),
call(
'foo',
bar='baz',
MarkerParam='marker2'
)
]
def _find_usage_apis(self):
"""
Find usage on APIs / RestAPIs, and resources that are limited per-API.
Update `self.limits`.
"""
api_ids = []
logger.debug('Finding usage for APIs')
regional_count = 0
private_count = 0
edge_count = 0
paginator = self.conn.get_paginator('get_rest_apis')
for resp in paginator.paginate():
for api in resp['items']:
api_ids.append(api['id'])
epconf = api.get('endpointConfiguration', {}).get('types', [])
if epconf == ['PRIVATE']:
private_count += 1
elif epconf == ['EDGE']:
edge_count += 1
else:
regional_count += 1
self.limits['Regional APIs per account']._add_current_usage(
regional_count, aws_type='AWS::ApiGateway::RestApi'
)
self.limits['Private APIs per account']._add_current_usage(
private_count, aws_type='AWS::ApiGateway::RestApi'
)
self.limits['Edge APIs per account']._add_current_usage(
edge_count, aws_type='AWS::ApiGateway::RestApi'
)
logger.debug('Found %d APIs', len(api_ids))
# now the per-API limits...
warn_stages_paginated = None
logger.debug('Finding usage for per-API limits')
for api_id in api_ids:
res_count = 0
paginator = self.conn.get_paginator('get_resources')
for resp in paginator.paginate(restApiId=api_id):
res_count += len(resp['items'])
self.limits['Resources per API']._add_current_usage(
res_count, resource_id=api_id,
aws_type='AWS::ApiGateway::Resource'
)
doc_parts = paginate_dict(
self.conn.get_documentation_parts,
restApiId=api_id,
alc_marker_path=['position'],
alc_data_path=['items'],
alc_marker_param='position'
)
self.limits['Documentation parts per API']._add_current_usage(
len(doc_parts), resource_id=api_id,
aws_type='AWS::ApiGateway::DocumentationPart'
)
# note that per the boto3 docs, there's no pagination of this...
stages = self.conn.get_stages(restApiId=api_id)
if len(set(stages.keys()) - set(['item', 'ResponseMetadata'])) > 0:
warn_stages_paginated = stages.keys()
self.limits['Stages per API']._add_current_usage(
len(stages['item']), resource_id=api_id,
aws_type='AWS::ApiGateway::Stage'
)
authorizers = paginate_dict(
self.conn.get_authorizers,
restApiId=api_id,
alc_marker_path=['position'],
alc_data_path=['items'],
alc_marker_param='position'
)
self.limits['Custom authorizers per API']._add_current_usage(
len(authorizers), resource_id=api_id,
aws_type='AWS::ApiGateway::Authorizer'
)
if warn_stages_paginated is not None:
logger.warning(
'APIGateway get_stages returned more keys than present in '
'boto3 docs: %s', sorted(warn_stages_paginated)
)
