def test_no_data_path(self): func = Mock() with pytest.raises(Exception) as excinfo: paginate_dict(func, alc_marker_path=[]) ex_str = "alc_data_path must be specified for queries " \ "that return a dict." assert ex_str in str(excinfo)
def test_no_data_path(self): func = Mock() with pytest.raises(Exception) as excinfo: paginate_dict(func, alc_marker_path=[]) ex_str = "alc_data_path must be specified for queries " \ "that return a dict." assert ex_str in str(excinfo)
def _find_usage_apis(self): """ Find usage on APIs / RestAPIs, and resources that are limited per-API. Update `self.limits`. """ api_ids = [] logger.debug('Finding usage for APIs') paginator = self.conn.get_paginator('get_rest_apis') for resp in paginator.paginate(): for api in resp['items']: api_ids.append(api['id']) logger.debug('Found %d APIs', len(api_ids)) self.limits['APIs per account']._add_current_usage( len(api_ids), aws_type='AWS::ApiGateway::RestApi') # now the per-API limits... warn_stages_paginated = None logger.debug('Finding usage for per-API limits') for api_id in api_ids: res_count = 0 paginator = self.conn.get_paginator('get_resources') for resp in paginator.paginate(restApiId=api_id): res_count += len(resp['items']) self.limits['Resources per API']._add_current_usage( res_count, resource_id=api_id, aws_type='AWS::ApiGateway::Resource') doc_parts = paginate_dict(self.conn.get_documentation_parts, restApiId=api_id, alc_marker_path=['position'], alc_data_path=['items'], alc_marker_param='position') self.limits['Documentation parts per API']._add_current_usage( len(doc_parts), resource_id=api_id, aws_type='AWS::ApiGateway::DocumentationPart') # note that per the boto3 docs, there's no pagination of this... stages = self.conn.get_stages(restApiId=api_id) if len(set(stages.keys()) - set(['item', 'ResponseMetadata'])) > 0: warn_stages_paginated = stages.keys() self.limits['Stages per API']._add_current_usage( len(stages['item']), resource_id=api_id, aws_type='AWS::ApiGateway::Stage') authorizers = paginate_dict(self.conn.get_authorizers, restApiId=api_id, alc_marker_path=['position'], alc_data_path=['items'], alc_marker_param='position') self.limits['Custom authorizers per API']._add_current_usage( len(authorizers), resource_id=api_id, aws_type='AWS::ApiGateway::Authorizer') if warn_stages_paginated is not None: logger.warning( 'APIGateway get_stages returned more keys than present in ' 'boto3 docs: %s', sorted(warn_stages_paginated))
def test_no_marker(self): result = {'k1': {'k2': {'Data': []}}} func = Mock() func.return_value = result res = paginate_dict(func, alc_marker_path=['k1', 'k2', 'Marker'], alc_data_path=['k1', 'k2', 'Data'], alc_marker_param='Marker') assert res == result assert func.mock_calls == [call()]
def test_two_iterations(self): e1 = Mock() e2 = Mock() e3 = Mock() e4 = Mock() e5 = Mock() e6 = Mock() func = Mock() res1 = { 'k1': { 'k2': { 'Data': [e1, e2], 'Foo1': 'bar1', 'Marker': 'marker1' } } } res2 = { 'k1': { 'k2': { 'Data': [e3, e4], 'Foo2': 'bar2', 'Marker': 'marker2' } } } res3 = {'k1': {'k2': {'Data': [e5, e6], 'Foo3': 'bar3'}}} expected = { 'k1': { 'k2': { 'Data': [e1, e2, e3, e4, e5, e6], 'Foo3': 'bar3' } } } func.side_effect = [res1, res2, res3] res = paginate_dict(func, 'foo', bar='baz', alc_marker_path=['k1', 'k2', 'Marker'], alc_data_path=['k1', 'k2', 'Data'], alc_marker_param='MarkerParam') assert res == expected assert func.mock_calls == [ call('foo', bar='baz'), call('foo', bar='baz', MarkerParam='marker1'), call('foo', bar='baz', MarkerParam='marker2') ]
def test_bad_path(self): result = { 'k1': { 'badpath': {} } } func = Mock() func.return_value = result res = paginate_dict( func, alc_marker_path=['k1', 'k2', 'Marker'], alc_data_path=['k1', 'k2', 'Data'], alc_marker_param='Marker' ) assert res == result assert func.mock_calls == [call()]
def test_two_iterations(self): e1 = Mock() e2 = Mock() e3 = Mock() e4 = Mock() e5 = Mock() e6 = Mock() func = Mock() res1 = { 'k1': { 'k2': { 'Data': [e1, e2], 'Foo1': 'bar1', 'Marker': 'marker1' } } } res2 = { 'k1': { 'k2': { 'Data': [e3, e4], 'Foo2': 'bar2', 'Marker': 'marker2' } } } res3 = { 'k1': { 'k2': { 'Data': [e5, e6], 'Foo3': 'bar3' } } } expected = { 'k1': { 'k2': { 'Data': [e1, e2, e3, e4, e5, e6], 'Foo3': 'bar3' } } } func.side_effect = [res1, res2, res3] res = paginate_dict( func, 'foo', bar='baz', alc_marker_path=['k1', 'k2', 'Marker'], alc_data_path=['k1', 'k2', 'Data'], alc_marker_param='MarkerParam' ) assert res == expected assert func.mock_calls == [ call('foo', bar='baz'), call( 'foo', bar='baz', MarkerParam='marker1' ), call( 'foo', bar='baz', MarkerParam='marker2' ) ]
def _find_usage_apis(self): """ Find usage on APIs / RestAPIs, and resources that are limited per-API. Update `self.limits`. """ api_ids = [] logger.debug('Finding usage for APIs') regional_count = 0 private_count = 0 edge_count = 0 paginator = self.conn.get_paginator('get_rest_apis') for resp in paginator.paginate(): for api in resp['items']: api_ids.append(api['id']) epconf = api.get('endpointConfiguration', {}).get('types', []) if epconf == ['PRIVATE']: private_count += 1 elif epconf == ['EDGE']: edge_count += 1 else: regional_count += 1 self.limits['Regional APIs per account']._add_current_usage( regional_count, aws_type='AWS::ApiGateway::RestApi' ) self.limits['Private APIs per account']._add_current_usage( private_count, aws_type='AWS::ApiGateway::RestApi' ) self.limits['Edge APIs per account']._add_current_usage( edge_count, aws_type='AWS::ApiGateway::RestApi' ) logger.debug('Found %d APIs', len(api_ids)) # now the per-API limits... warn_stages_paginated = None logger.debug('Finding usage for per-API limits') for api_id in api_ids: res_count = 0 paginator = self.conn.get_paginator('get_resources') for resp in paginator.paginate(restApiId=api_id): res_count += len(resp['items']) self.limits['Resources per API']._add_current_usage( res_count, resource_id=api_id, aws_type='AWS::ApiGateway::Resource' ) doc_parts = paginate_dict( self.conn.get_documentation_parts, restApiId=api_id, alc_marker_path=['position'], alc_data_path=['items'], alc_marker_param='position' ) self.limits['Documentation parts per API']._add_current_usage( len(doc_parts), resource_id=api_id, aws_type='AWS::ApiGateway::DocumentationPart' ) # note that per the boto3 docs, there's no pagination of this... stages = self.conn.get_stages(restApiId=api_id) if len(set(stages.keys()) - set(['item', 'ResponseMetadata'])) > 0: warn_stages_paginated = stages.keys() self.limits['Stages per API']._add_current_usage( len(stages['item']), resource_id=api_id, aws_type='AWS::ApiGateway::Stage' ) authorizers = paginate_dict( self.conn.get_authorizers, restApiId=api_id, alc_marker_path=['position'], alc_data_path=['items'], alc_marker_param='position' ) self.limits['Custom authorizers per API']._add_current_usage( len(authorizers), resource_id=api_id, aws_type='AWS::ApiGateway::Authorizer' ) if warn_stages_paginated is not None: logger.warning( 'APIGateway get_stages returned more keys than present in ' 'boto3 docs: %s', sorted(warn_stages_paginated) )