def get_all_env_peerings(config: Sequence[dict], metadata: Mapping) -> List[dict]: """ Given the configuration, find all the peerings which exist across all accounts and regions. See https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ec2.html#EC2.Client.describe_vpc_peering_connections :param config: A list containing dictionaries representing the configuration for the environment :type config: list :param key: The key to deduplicate on. :returns: A deduplicated list of all peerings for this environment. :rtype: list """ # Get a dictionary of accounts, regions, vpcs and the vpcs they should be peered to config_peer_map = deepcopy(get_peering_map(config)) # Initialise a filter to get vpc peerings relevant to the current environment being worked on filters = [{'Name': 'tag:peerd_created', 'Values': ['true']}, {'Name': 'tag:peerd_environment', 'Values': [metadata['environment']]}, {'Name': 'status-code', 'Values': ['active']}] # Get all existing peerings from accounts listed in the config file peerings: List[dict] = [] for account_id in config_peer_map.keys(): for region in config_peer_map[account_id].keys(): peerings.extend(get_all_peerings(account_id, region, filters)) # Now get all existing peerings from all accounts that participate in peerings # (There might be accounts not listed in the config file) for peering in peerings: for side in ('RequesterVpcInfo', 'AccepterVpcInfo'): account_id = peering[side]['OwnerId'] region = peering[side]['Region'] if not config_peer_map[account_id][region]: LOGGER.debug(f'Found peering to unconfigured account {account_id} region {region}. Inspecting...') peerings.extend(get_all_peerings(account_id, region, filters)) config_peer_map[account_id][region] = True LOGGER.debug(f'Discovered additional peerings in {account_id} {region}') # Add the 2 lists, and ceduplicate peerings. These will be all existing peerings even between accounts deleted from the config file return deduplicate_list_dicts(peerings, 'VpcPeeringConnectionId')
def describe_vpc_cached(vpc_id: str, account_id: str, region: str, **kwargs) -> dict: """ Describe a specific VPC from our cache of VPC descriptions for a given account and region Return a dict if things go as expected. We accept additional arguments here to handle the case where the arguments are given as a dictionary with additional keys. Such as when filtering the main config for only VPCs which actually exist. :param vpc_id: A vpc id e.g. vpc-abc123 :type vpc_id: str :param account_id: An AWS account id number :type account: str :param region: An AWS region where the service client should be created. e.g. us-east-1 :type region: str :param **kwargs: Handle trailing arguments in case where dictionary of arguments is provided. :type **kwargs: dict :returns: A list of dictionaries representing all the VPCs for a given account and region. :rtype: list """ LOGGER.debug( f'Fetching description of VPC {vpc_id} for account {account_id} in region {region}' ) vpcs = describe_account_vpcs_cached(account_id, region) if vpc_dict := next((x for x in vpcs if x['VpcId'] == vpc_id), None): return vpc_dict
def get_vpc_peering(vpc_id: str, remote_vpc_id: str, account_id: str, region: str, filters: list = []) -> dict: """ Returns a dictionary describing a specific VPC peering between two VPCs From the perspective of a given account. Note: Same peering can have slightly different content based on perspective account. Note: This function is greedy in the sense that the requester does not have to be the requester as AWS defines it in https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ec2.html This function will match peerings even when the remote_vpc_id is the requester. Example usage: ``` filters = [{'Name': 'tag:peerd_created', 'Values': ['true']}, {'Name': 'tag:peerd_environment', 'Values': [metadata['environment']]}, {'Name': 'status-code', 'Values': ['active']}] if not (peering := get_vpc_peering(vpc_id, remote_vpc_id, account_id, region, filters)): LOGGER.warning(f'No active peering between {vpc_id} and {remote_vpc_id} for this environment' f' {metadata["environment"]}. It may exist as part of another environment.') continue ``` :param vpc_id: A vpc id e.g. vpc-abc123 :type vpc_id: str :param remote_vpc_id: A vpc id e.g. vpc-abc123 :type remote_vpc_id: str :param account_id: An AWS account id number :type account: str :param region: An AWS region. e.g. us-east-1 :type region: str :param filters: A standard list of boto3 filters (list of dics). :type filters: list :returns: A list containing all peering dictionaries for an account-region. :rtype: list """ vpc_peerings = get_all_peerings(account_id, region, filters) for peering in vpc_peerings: if peering['AccepterVpcInfo']['VpcId'] in [vpc_id, remote_vpc_id]: if peering['RequesterVpcInfo']['VpcId'] in [vpc_id, remote_vpc_id]: LOGGER.debug( f"Found peering {peering['VpcPeeringConnectionId']} between {vpc_id} and {remote_vpc_id}" ) return peering LOGGER.debug( f"No active peering between {vpc_id} and {remote_vpc_id} for given filters {filters}" ) return None
def get_deletable_peerings(peerings: list, vpc_list: list) -> list: """ Given a list of peerings that exist in the infrastructure, and a list of vpcs that are configured to be members of this environment, returns a list of peerings which should be deleted. :param config: A list containing dictionaries representing the configuration for the environment :type config: list :param key: The key to deduplicate on. :returns: A deduplicated list of all peerings for this environment. :rtype: list """ peerings_to_delete = [] for peering in peerings: accepter = peering['AccepterVpcInfo'] requester = peering['RequesterVpcInfo'] if (accepter['VpcId'] not in vpc_list) or (requester['VpcId'] not in vpc_list): LOGGER.debug(f"Peering between {requester['VpcId']} and {accepter['VpcId']} can be deleted.") peerings_to_delete.append(peering) return deduplicate_list_dicts(peerings_to_delete, 'VpcPeeringConnectionId')
def tag_resource(client, resource: str, tags: dict, dryrun: bool = False): """ Tags an AWS resource with the provided tags. Example Usage: ``` ec2_client = aws_client(account_id, 'ec2', region) tags = {'peerd_support': '*****@*****.**', 'peerd_datetime': str(datetime.now())} tag_resource(ec2_client, peerding_id, tags) ``` :param client: An AWS boto3 client connection to an AWS resource :type client: boto3 client :param resource: The name of the resource e.g. rt-abc123 :type resource: string :param tags: Dictionary of tags to apply to a resource :type tags: dict :returns: Nothing :raises BaseException: Raises an exception if there was some problem tagging the resource """ tags = [{'Key': key, 'Value': value} for (key, value) in tags.items()] for x in range(5): try: LOGGER.debug(f'Tagging {resource}') client.create_tags(Resources=[resource], Tags=tags, DryRun=dryrun) LOGGER.debug(f'Tagging {resource} successful') return except botocore.exceptions.ClientError as err: if err.response['Error']['Code'] == 'DryRunOperation': LOGGER.debug(f'Tagging {resource} successful') return LOGGER.info( f'Tagging {resource} encountered error: {err.response["Error"]["Message"]}. Will retry.' ) sleep(1) continue except BaseException: LOGGER.warning("Unexpected error: %s", sys.exc_info()[1], exc_info=True) raise Exception(f'Could not tag resource {resource}')
# If we can't we return none and cache none. if not (credentials := get_role_credentials(account, aws_sts_client())): LOGGER.warning(f'Unable to tokenise into {account} for service {service} in region {region}. Moving on') return None # In this block we use the account credentials we got above, to create and cache a client # connection to an AWS service. try: client = boto3.client( service, region_name=region, aws_access_key_id=credentials['AccessKeyId'], aws_secret_access_key=credentials['SecretAccessKey'], aws_session_token=credentials['SessionToken'], config=Config(retries=dict(max_attempts=10))) LOGGER.debug('Obtained fresh client connection.') return client except BaseException: LOGGER.error(f'Unexpected error: {sys.exc_info()[1]}', exc_info=True) def tag_resource(client: Any, resource: str, tags: Mapping, dryrun: bool = False) -> None: """ Tags an AWS resource with the provided tags. Example Usage: ``` ec2_client = aws_client(account_id, 'ec2', region) tags = {'peerd_support': '*****@*****.**', 'peerd_datetime': str(datetime.now())} tag_resource(ec2_client, peerding_id, tags)
if service not in CLIENT_CACHE[account][region]: try: CLIENT_CACHE[account][region][service] = boto3.client( service, region_name=region, aws_access_key_id=credentials['AccessKeyId'], aws_secret_access_key=credentials['SecretAccessKey'], aws_session_token=credentials['SessionToken'], config=Config(retries=dict(max_attempts=10))) LOGGER.info('Obtained fresh client connection.') except BaseException: LOGGER.error(f'Unexpected error: {sys.exc_info()[1]}', exc_info=True) CLIENT_CACHE[account][region][service] = None else: LOGGER.debug('Fetched a cached client connection.') return CLIENT_CACHE[account][region][service] def tag_resource(client, resource: str, tags: dict, dryrun: bool = False): """ Tags an AWS resource with the provided tags. Example Usage: ``` ec2_client = aws_client(account_id, 'ec2', region) tags = {'peerd_support': '*****@*****.**', 'peerd_datetime': str(datetime.now())} tag_resource(ec2_client, peerding_id, tags) ```
continue # Break if the cidr is not pointing at a peering id if 'VpcPeeringConnectionId' not in route: LOGGER.warning( f'CIDR {cidr} not pointing at VPC peering in {account_id} {vpc_id} {route_table_id}' ) break # Break if the cidr is pointing at a different peering id if route['VpcPeeringConnectionId'] != peering_id: LOGGER.warning( f'Peering for {cidr} not pointing at correct peering in {account_id} {vpc_id} {route_table_id}' ) break # The cidr already exists and is pointing at the correct peering LOGGER.debug( f'{account_id} {vpc_id} {route_table_id} {cidr} pointing at correct peering {peering_id}' ) break else: # This block is only executed if we did not break in the above for loop # Install peering route in route table LOGGER.info( f'Intalling cidr {cidr} via {peering_id} to {remote_vpc_id} in {account_id} {vpc_id} {route_table_id}' ) try: ec2_client.create_route( RouteTableId=route_table_id, VpcPeeringConnectionId=peering_id, DestinationCidrBlock=cidr, DryRun=dryrun) except ClientError as err:
# Handle case the route is not a ipv4 route, e.g. S3 endpoint if route.get('DestinationCidrBlock', None) == cidr: # Delete the cidr if it points at a blackhole if route['State'] == 'blackhole': LOGGER.warning(f'Blackhole in: {account_id} {vpc_id} {route_table_id} for {cidr}. Deleting.') try: ec2_client.delete_route(RouteTableId=route_table_id, DestinationCidrBlock=route['DestinationCidrBlock'], DryRun=dryrun) except ClientError as err: if err.response['Error']['Code'] != 'DryRunOperation': raise # We continue here instead of breaking so that we install the correct route. continue if route.get('Origin', None) == 'EnableVgwRoutePropagation': LOGGER.debug(f"{account_id} {vpc_id} {route_table_id} {cidr} pointing at {route['GatewayId']}. Static route may co-exist.") continue # Break if the cidr is not pointing at a peering id if 'VpcPeeringConnectionId' not in route: LOGGER.warning(f'CIDR {cidr} not pointing at VPC peering in {account_id} {vpc_id} {route_table_id}') break # Break if the cidr is pointing at a different peering id if route['VpcPeeringConnectionId'] != peering_id: LOGGER.warning(f'Peering for {cidr} not pointing at correct peering in {account_id} {vpc_id} {route_table_id}') break # The cidr already exists and is pointing at the correct peering LOGGER.debug(f'{account_id} {vpc_id} {route_table_id} {cidr} pointing at correct peering {peering_id}') break else: # This block is only executed if we did not break in the above for loop # Install peering route in route table