def get_all_env_peerings(config: Sequence[dict], metadata: Mapping) -> List[dict]:
"""
Given the configuration, find all the peerings which exist across all accounts
and regions.
See https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ec2.html#EC2.Client.describe_vpc_peering_connections
:param config: A list containing dictionaries representing the configuration for the environment
:type config: list
:param key: The key to deduplicate on.
:returns: A deduplicated list of all peerings for this environment.
:rtype: list
"""
# Get a dictionary of accounts, regions, vpcs and the vpcs they should be peered to
config_peer_map = deepcopy(get_peering_map(config))
# Initialise a filter to get vpc peerings relevant to the current environment being worked on
filters = [{'Name': 'tag:peerd_created', 'Values': ['true']},
{'Name': 'tag:peerd_environment', 'Values': [metadata['environment']]},
{'Name': 'status-code', 'Values': ['active']}]
# Get all existing peerings from accounts listed in the config file
peerings: List[dict] = []
for account_id in config_peer_map.keys():
for region in config_peer_map[account_id].keys():
peerings.extend(get_all_peerings(account_id, region, filters))
# Now get all existing peerings from all accounts that participate in peerings
# (There might be accounts not listed in the config file)
for peering in peerings:
for side in ('RequesterVpcInfo', 'AccepterVpcInfo'):
account_id = peering[side]['OwnerId']
region = peering[side]['Region']
if not config_peer_map[account_id][region]:
LOGGER.debug(f'Found peering to unconfigured account {account_id} region {region}. Inspecting...')
peerings.extend(get_all_peerings(account_id, region, filters))
config_peer_map[account_id][region] = True
LOGGER.debug(f'Discovered additional peerings in {account_id} {region}')
# Add the 2 lists, and ceduplicate peerings. These will be all existing peerings even between accounts deleted from the config file
return deduplicate_list_dicts(peerings, 'VpcPeeringConnectionId')
def describe_vpc_cached(vpc_id: str, account_id: str, region: str,
**kwargs) -> dict:
"""
Describe a specific VPC from our cache of VPC descriptions for a given account and region
Return a dict if things go as expected.
We accept additional arguments here to handle the case where the arguments are given as
a dictionary with additional keys. Such as when filtering the main config for only VPCs
which actually exist.
:param vpc_id: A vpc id e.g. vpc-abc123
:type vpc_id: str
:param account_id: An AWS account id number
:type account: str
:param region: An AWS region where the service client should be created. e.g. us-east-1
:type region: str
:param **kwargs: Handle trailing arguments in case where dictionary of arguments is provided.
:type **kwargs: dict
:returns: A list of dictionaries representing all the VPCs for a given account and region.
:rtype: list
"""
LOGGER.debug(
f'Fetching description of VPC {vpc_id} for account {account_id} in region {region}'
)
vpcs = describe_account_vpcs_cached(account_id, region)
if vpc_dict := next((x for x in vpcs if x['VpcId'] == vpc_id), None):
return vpc_dict
def get_vpc_peering(vpc_id: str,
remote_vpc_id: str,
account_id: str,
region: str,
filters: list = []) -> dict:
"""
Returns a dictionary describing a specific VPC peering between two VPCs
From the perspective of a given account.
Note: Same peering can have slightly different content based on perspective account.
Note: This function is greedy in the sense that the requester does not have to be
the requester as AWS defines it in https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ec2.html
This function will match peerings even when the remote_vpc_id is the requester.
Example usage:
```
filters = [{'Name': 'tag:peerd_created', 'Values': ['true']},
{'Name': 'tag:peerd_environment', 'Values': [metadata['environment']]},
{'Name': 'status-code', 'Values': ['active']}]
if not (peering := get_vpc_peering(vpc_id, remote_vpc_id, account_id, region, filters)):
LOGGER.warning(f'No active peering between {vpc_id} and {remote_vpc_id} for this environment'
f' {metadata["environment"]}. It may exist as part of another environment.')
continue
```
:param vpc_id: A vpc id e.g. vpc-abc123
:type vpc_id: str
:param remote_vpc_id: A vpc id e.g. vpc-abc123
:type remote_vpc_id: str
:param account_id: An AWS account id number
:type account: str
:param region: An AWS region. e.g. us-east-1
:type region: str
:param filters: A standard list of boto3 filters (list of dics).
:type filters: list
:returns: A list containing all peering dictionaries for an account-region.
:rtype: list
"""
vpc_peerings = get_all_peerings(account_id, region, filters)
for peering in vpc_peerings:
if peering['AccepterVpcInfo']['VpcId'] in [vpc_id, remote_vpc_id]:
if peering['RequesterVpcInfo']['VpcId'] in [vpc_id, remote_vpc_id]:
LOGGER.debug(
f"Found peering {peering['VpcPeeringConnectionId']} between {vpc_id} and {remote_vpc_id}"
)
return peering
LOGGER.debug(
f"No active peering between {vpc_id} and {remote_vpc_id} for given filters {filters}"
)
return None
def get_deletable_peerings(peerings: list, vpc_list: list) -> list:
"""
Given a list of peerings that exist in the infrastructure, and a
list of vpcs that are configured to be members of this environment,
returns a list of peerings which should be deleted.
:param config: A list containing dictionaries representing the configuration for the environment
:type config: list
:param key: The key to deduplicate on.
:returns: A deduplicated list of all peerings for this environment.
:rtype: list
"""
peerings_to_delete = []
for peering in peerings:
accepter = peering['AccepterVpcInfo']
requester = peering['RequesterVpcInfo']
if (accepter['VpcId'] not in vpc_list) or (requester['VpcId'] not in vpc_list):
LOGGER.debug(f"Peering between {requester['VpcId']} and {accepter['VpcId']} can be deleted.")
peerings_to_delete.append(peering)
return deduplicate_list_dicts(peerings_to_delete, 'VpcPeeringConnectionId')
def tag_resource(client, resource: str, tags: dict, dryrun: bool = False):
"""
Tags an AWS resource with the provided tags.
Example Usage:
```
ec2_client = aws_client(account_id, 'ec2', region)
tags = {'peerd_support': '*****@*****.**',
'peerd_datetime': str(datetime.now())}
tag_resource(ec2_client, peerding_id, tags)
```
:param client: An AWS boto3 client connection to an AWS resource
:type client: boto3 client
:param resource: The name of the resource e.g. rt-abc123
:type resource: string
:param tags: Dictionary of tags to apply to a resource
:type tags: dict
:returns: Nothing
:raises BaseException: Raises an exception if there was some problem tagging the resource
"""
tags = [{'Key': key, 'Value': value} for (key, value) in tags.items()]
for x in range(5):
try:
LOGGER.debug(f'Tagging {resource}')
client.create_tags(Resources=[resource], Tags=tags, DryRun=dryrun)
LOGGER.debug(f'Tagging {resource} successful')
return
except botocore.exceptions.ClientError as err:
if err.response['Error']['Code'] == 'DryRunOperation':
LOGGER.debug(f'Tagging {resource} successful')
return
LOGGER.info(
f'Tagging {resource} encountered error: {err.response["Error"]["Message"]}. Will retry.'
)
sleep(1)
continue
except BaseException:
LOGGER.warning("Unexpected error: %s",
sys.exc_info()[1],
exc_info=True)
raise Exception(f'Could not tag resource {resource}')
# If we can't we return none and cache none.
if not (credentials := get_role_credentials(account, aws_sts_client())):
LOGGER.warning(f'Unable to tokenise into {account} for service {service} in region {region}. Moving on')
return None
# In this block we use the account credentials we got above, to create and cache a client
# connection to an AWS service.
try:
client = boto3.client(
service,
region_name=region,
aws_access_key_id=credentials['AccessKeyId'],
aws_secret_access_key=credentials['SecretAccessKey'],
aws_session_token=credentials['SessionToken'],
config=Config(retries=dict(max_attempts=10)))
LOGGER.debug('Obtained fresh client connection.')
return client
except BaseException:
LOGGER.error(f'Unexpected error: {sys.exc_info()[1]}', exc_info=True)
def tag_resource(client: Any, resource: str, tags: Mapping, dryrun: bool = False) -> None:
"""
Tags an AWS resource with the provided tags.
Example Usage:
```
ec2_client = aws_client(account_id, 'ec2', region)
tags = {'peerd_support': '*****@*****.**',
'peerd_datetime': str(datetime.now())}
tag_resource(ec2_client, peerding_id, tags)
if service not in CLIENT_CACHE[account][region]:
try:
CLIENT_CACHE[account][region][service] = boto3.client(
service,
region_name=region,
aws_access_key_id=credentials['AccessKeyId'],
aws_secret_access_key=credentials['SecretAccessKey'],
aws_session_token=credentials['SessionToken'],
config=Config(retries=dict(max_attempts=10)))
LOGGER.info('Obtained fresh client connection.')
except BaseException:
LOGGER.error(f'Unexpected error: {sys.exc_info()[1]}',
exc_info=True)
CLIENT_CACHE[account][region][service] = None
else:
LOGGER.debug('Fetched a cached client connection.')
return CLIENT_CACHE[account][region][service]
def tag_resource(client, resource: str, tags: dict, dryrun: bool = False):
"""
Tags an AWS resource with the provided tags.
Example Usage:
```
ec2_client = aws_client(account_id, 'ec2', region)
tags = {'peerd_support': '*****@*****.**',
'peerd_datetime': str(datetime.now())}
tag_resource(ec2_client, peerding_id, tags)
```
continue
# Break if the cidr is not pointing at a peering id
if 'VpcPeeringConnectionId' not in route:
LOGGER.warning(
f'CIDR {cidr} not pointing at VPC peering in {account_id} {vpc_id} {route_table_id}'
)
break
# Break if the cidr is pointing at a different peering id
if route['VpcPeeringConnectionId'] != peering_id:
LOGGER.warning(
f'Peering for {cidr} not pointing at correct peering in {account_id} {vpc_id} {route_table_id}'
)
break
# The cidr already exists and is pointing at the correct peering
LOGGER.debug(
f'{account_id} {vpc_id} {route_table_id} {cidr} pointing at correct peering {peering_id}'
)
break
else:
# This block is only executed if we did not break in the above for loop
# Install peering route in route table
LOGGER.info(
f'Intalling cidr {cidr} via {peering_id} to {remote_vpc_id} in {account_id} {vpc_id} {route_table_id}'
)
try:
ec2_client.create_route(
RouteTableId=route_table_id,
VpcPeeringConnectionId=peering_id,
DestinationCidrBlock=cidr,
DryRun=dryrun)
except ClientError as err:
# Handle case the route is not a ipv4 route, e.g. S3 endpoint
if route.get('DestinationCidrBlock', None) == cidr:
# Delete the cidr if it points at a blackhole
if route['State'] == 'blackhole':
LOGGER.warning(f'Blackhole in: {account_id} {vpc_id} {route_table_id} for {cidr}. Deleting.')
try:
ec2_client.delete_route(RouteTableId=route_table_id,
DestinationCidrBlock=route['DestinationCidrBlock'],
DryRun=dryrun)
except ClientError as err:
if err.response['Error']['Code'] != 'DryRunOperation':
raise
# We continue here instead of breaking so that we install the correct route.
continue
if route.get('Origin', None) == 'EnableVgwRoutePropagation':
LOGGER.debug(f"{account_id} {vpc_id} {route_table_id} {cidr} pointing at {route['GatewayId']}. Static route may co-exist.")
continue
# Break if the cidr is not pointing at a peering id
if 'VpcPeeringConnectionId' not in route:
LOGGER.warning(f'CIDR {cidr} not pointing at VPC peering in {account_id} {vpc_id} {route_table_id}')
break
# Break if the cidr is pointing at a different peering id
if route['VpcPeeringConnectionId'] != peering_id:
LOGGER.warning(f'Peering for {cidr} not pointing at correct peering in {account_id} {vpc_id} {route_table_id}')
break
# The cidr already exists and is pointing at the correct peering
LOGGER.debug(f'{account_id} {vpc_id} {route_table_id} {cidr} pointing at correct peering {peering_id}')
break
else:
# This block is only executed if we did not break in the above for loop
# Install peering route in route table
