def import_elbs(event, context):
"""Import AWS ELB resource
"""
dynamodb = boto3.resource('dynamodb')
client = boto3.client('ec2')
elb_table = dynamodb.Table(os.environ['DYNAMODB_TABLE_ELB'])
account = event['account']
region = event['region']
# ttl time to expire items in DynamoDB table, default 48 hours
# ttl provided in seconds
ttl_expire_time = (int(time.time()) +
os.environ.get('TTL_EXPIRE_TIME', 172800))
ACCESS_KEY, SECRET_KEY, SESSION_TOKEN = establish_role(account)
classic_elb_client = boto3.client('elb',
aws_access_key_id=ACCESS_KEY,
aws_secret_access_key=SECRET_KEY,
aws_session_token=SESSION_TOKEN,
region_name=region)
classic_elbs = classic_elb_client.describe_load_balancers()
if not classic_elbs['LoadBalancerDescriptions']:
logger.info("No ELBs allocated for account: {0} in region {1}".format(
account, region))
else:
classic_elb_importer(classic_elbs, elb_table, account, region,
ttl_expire_time)
def import_eips(event, context):
dynamodb = boto3.resource('dynamodb')
client = boto3.client('ec2')
eip_table = dynamodb.Table(os.environ['DYNAMODB_TABLE_EIP'])
accounts_table = dynamodb.Table(os.environ['DYNAMODB_TABLE_ACCOUNTS'])
accounts = accounts_table.scan()['Items']
regions = [
region['RegionName'] for region in client.describe_regions()['Regions']
]
for region in regions:
for acct in accounts:
ACCESS_KEY, SECRET_KEY, SESSION_TOKEN = establish_role(acct)
client = boto3.client('ec2',
aws_access_key_id=ACCESS_KEY,
aws_secret_access_key=SECRET_KEY,
aws_session_token=SESSION_TOKEN,
region_name=region)
eips = client.describe_addresses()
if not eips['Addresses']:
logger.info(
"No allocated EIPs for acct: {0} in region {1}".format(
acct['id'], region))
else:
for eip in eips['Addresses']:
acct_id = acct['id']
eip_address = eip['PublicIp']
if 'AssociationId' in eip:
eip_association_id = eip['AssociationId']
else:
logger.info(
"No AllocadtionId found for EIP {0} in acct {1} {2}"
.format(eip_address, acct['id'], region))
eip_association_id = "none"
if 'AllocationId' in eip:
eip_id = eip['AllocationId']
else:
logger.info(
"No AllocadtionId found for EIP {0} in acct {1} {2}"
.format(eip_address, acct['id'], region))
eip_id = "none"
logger.info(
'Discovered EIP in use: {0} with ID: {1}'.format(
eip_address, eip_id))
response = eip_table.put_item(
Item={
'id': eip_address,
'AllocationId': eip_id,
'AccountID': acct_id,
'AssociationId': eip_association_id,
'Region': region
},
ConditionExpression='attribute_not_exists(eip_id)')
logger.info("Dynamodb response: {}".format(response))
def import_eips(event, context):
dynamodb = boto3.resource('dynamodb')
client = boto3.client('ec2')
eip_table = dynamodb.Table(os.environ['DYNAMODB_TABLE_EIP'])
account = event['account']
region = event['region']
# ttl time to expire items in DynamoDB table, default 48 hours
# ttl provided in seconds
ttl_expire_time = (
int(time.time()) + os.environ.get('TTL_EXPIRE_TIME', 172800))
ACCESS_KEY, SECRET_KEY, SESSION_TOKEN = establish_role(account)
client = boto3.client('ec2',
aws_access_key_id=ACCESS_KEY,
aws_secret_access_key=SECRET_KEY,
aws_session_token=SESSION_TOKEN,
region_name=region
)
eips = client.describe_addresses()
if not eips['Addresses']:
logger.info("No allocated EIPs for account: {0} in region {1}"
.format(account, region))
else:
for eip in eips['Addresses']:
eip_address = eip['PublicIp']
if 'AssociationId' in eip:
eip_association_id = eip['AssociationId']
else:
logger.info(
"No AllocadtionId found for EIP {0} in account {1} {2}"
.format(eip_address, account, region))
eip_association_id = "none"
if 'AllocationId' in eip:
eip_id = eip['AllocationId']
else:
logger.info(
"No AllocadtionId found for EIP {0} in account {1} {2}"
.format(eip_address, account, region))
eip_id = "none"
logger.info('Discovered EIP in use: {0} with ID: {1}'
.format(eip_address, eip_id))
response = eip_table.put_item(
Item={
'id': eip_address,
'AllocationId': eip_id,
'AccountID': account,
'AssociationId': eip_association_id,
'Region': region,
'ttl': ttl_expire_time
}
)
logger.info("Dynamodb response: {}".format(response))
def available_ips(event, context):
"""Record AvailableIpAddressCount for all subnets in all regions
"""
dynamodb = boto3.resource('dynamodb')
client = boto3.client('ec2')
available_ips_table = dynamodb.Table(
os.environ['DYNAMODB_TABLE_AVAILABLE_IPS'])
accounts_table = dynamodb.Table(os.environ['DYNAMODB_TABLE_ACCOUNTS'])
accounts = accounts_table.scan()['Items']
regions = ([
region['RegionName'] for region in client.describe_regions()['Regions']
])
for region in regions:
for acct in accounts:
ACCESS_KEY, SECRET_KEY, SESSION_TOKEN = establish_role(acct)
client = boto3.client('ec2',
aws_access_key_id=ACCESS_KEY,
aws_secret_access_key=SECRET_KEY,
aws_session_token=SESSION_TOKEN,
region_name=region)
subnets = client.describe_subnets()
if not subnets['Subnets']:
logger.info(
"No allocated subnets for acct: {0} in region {1}".format(
acct['id'], region))
else:
for subnet in subnets['Subnets']:
vpc_id = subnet['VpcId']
available_ips = str(subnet['AvailableIpAddressCount'])
acct_id = acct['id']
subnet_id = subnet['SubnetId']
subnet = subnet['CidrBlock']
unique_id = f'{acct_id}{vpc_id}{subnet_id}{subnet}'
# ttl set to 48 hours
ttl_expire_time = int(time.time()) + 172800
response = available_ips_table.put_item(
Item={
'id': unique_id,
'VpcId': vpc_id,
'AccountID': acct_id,
'SubnetId': subnet_id,
'Subnet': subnet,
'Region': region,
'AvailableIpAddressCount': available_ips,
'ttl': ttl_expire_time
})
logger.info("Dynamodb response: {}".format(response))
def available_ips(event, context):
"""
Take in event data which should contain AWS account and region and
record AvailableIpAddressCount for all subnets in all regions
"""
dynamodb = boto3.resource('dynamodb')
client = boto3.client('ec2')
available_ips_table = dynamodb.Table(
os.environ['DYNAMODB_TABLE_AVAILABLE_IPS'])
account = event['account']
region = event['region']
# ttl time to expire items in DynamoDB table, default 48 hours
# ttl provided in seconds
ttl_expire_time = (
int(time.time()) + os.environ.get('TTL_EXPIRE_TIME', 172800))
ACCESS_KEY, SECRET_KEY, SESSION_TOKEN = establish_role(account)
client = boto3.client('ec2',
aws_access_key_id=ACCESS_KEY,
aws_secret_access_key=SECRET_KEY,
aws_session_token=SESSION_TOKEN,
region_name=region
)
subnets = client.describe_subnets()
if not subnets['Subnets']:
logger.info("No allocated subnets for account: {0} in region {1}"
.format(account, region))
else:
for subnet in subnets['Subnets']:
vpc_id = subnet['VpcId']
available_ips = str(subnet['AvailableIpAddressCount'])
subnet_id = subnet['SubnetId']
subnet = subnet['CidrBlock']
unique_id = f'{account}{vpc_id}{subnet_id}{subnet}'
response = available_ips_table.put_item(
Item={
'id': unique_id,
'VpcId': vpc_id,
'AccountID': account,
'SubnetId': subnet_id,
'Subnet': subnet,
'Region': region,
'AvailableIpAddressCount': available_ips,
'ttl': ttl_expire_time
}
)
logger.info("Dynamodb response: {}".format(response))
def import_nat_gateways(event, context):
dynamodb = boto3.resource('dynamodb')
client = boto3.client('ec2')
nat_table = dynamodb.Table(os.environ['DYNAMODB_TABLE_NAT_GATEWAYS'])
cidr_table = dynamodb.Table(os.environ['DYNAMODB_TABLE_CIDRS'])
accounts_table = dynamodb.Table(os.environ['DYNAMODB_TABLE_ACCOUNTS'])
accounts = accounts_table.scan()['Items']
cidrs = cidr_table.scan()['Items']
regions = [
region['RegionName'] for region in client.describe_regions()['Regions']
]
for region in regions:
for acct in accounts:
ACCESS_KEY, SECRET_KEY, SESSION_TOKEN = establish_role(acct)
client = boto3.client('ec2',
aws_access_key_id=ACCESS_KEY,
aws_secret_access_key=SECRET_KEY,
aws_session_token=SESSION_TOKEN,
region_name=region)
nats = client.describe_nat_gateways()
for nat in nats['NatGateways']:
public_ip = nat['NatGatewayAddresses'][0]['PublicIp']
acct_id = acct['id']
nat_id = nat['NatGatewayId']
nat_vpc_id = nat['VpcId']
logger.info('Logging NAT Gateway: {0} for account {1}'.format(
public_ip, acct))
response = nat_table.put_item(
Item={
'id': nat_id,
'PublicIp': public_ip,
'AccountID': acct_id,
'VpcId': nat_vpc_id,
'Region': region
},
ConditionExpression='attribute_not_exists(nat_id)')
def import_nat_gateways(event, context):
dynamodb = boto3.resource('dynamodb')
client = boto3.client('ec2')
nat_table = dynamodb.Table(os.environ['DYNAMODB_TABLE_NAT_GATEWAYS'])
cidr_table = dynamodb.Table(os.environ['DYNAMODB_TABLE_CIDRS'])
cidrs = cidr_table.scan()['Items']
account = event['account']
region = event['region']
# ttl time to expire items in DynamoDB table, default 48 hours
# ttl provided in seconds
ttl_expire_time = (int(time.time()) +
os.environ.get('TTL_EXPIRE_TIME', 172800))
ACCESS_KEY, SECRET_KEY, SESSION_TOKEN = establish_role(account)
client = boto3.client('ec2',
aws_access_key_id=ACCESS_KEY,
aws_secret_access_key=SECRET_KEY,
aws_session_token=SESSION_TOKEN,
region_name=region)
nats = client.describe_nat_gateways()
for nat in nats['NatGateways']:
public_ip = nat['NatGatewayAddresses'][0]['PublicIp']
nat_id = nat['NatGatewayId']
nat_vpc_id = nat['VpcId']
logger.info('Logging NAT Gateway: {0} for account {1}'.format(
public_ip, account))
response = nat_table.put_item(
Item={
'id': nat_id,
'PublicIp': public_ip,
'AccountID': account,
'VpcId': nat_vpc_id,
'Region': region,
'ttl': ttl_expire_time
})
def import_endpoint_services(event, context):
dynamodb = boto3.resource('dynamodb')
client = boto3.client('ec2')
endpoint_service_table = dynamodb.Table(
os.environ['DYNAMODB_TABLE_ENDPOINT_SERVICES'])
# ttl time to expire items in DynamoDB table, default 48 hours
# ttl provided in seconds
ttl_expire_time = (int(time.time()) +
os.environ.get('TTL_EXPIRE_TIME', 172800))
account = event['account']
region = event['region']
endpoint_services = endpoint_service_table.scan()['Items']
ACCESS_KEY, SECRET_KEY, SESSION_TOKEN = establish_role(account)
client = boto3.client('ec2',
aws_access_key_id=ACCESS_KEY,
aws_secret_access_key=SECRET_KEY,
aws_session_token=SESSION_TOKEN,
region_name=region)
elbv2_client = boto3.client('elbv2',
aws_access_key_id=ACCESS_KEY,
aws_secret_access_key=SECRET_KEY,
aws_session_token=SESSION_TOKEN,
region_name=region)
logger.info(
'Looking up endpoint details on account {} in region {}'.format(
account, region))
# Some Regions don't support this service yet,
# capture and log these exceptions
try:
endpoint_services = client.describe_vpc_endpoint_service_configurations(
)
except client.exceptions.ClientError as e:
if e.response['Error'][
'Message'] == endpoint_service_api_not_available:
logger.error('Error: {}'.format(e))
# Bail out here if AWS doesn't support Endpoint Services in region
return logger.info(
'VPC Endpoint Service is not available in {}'.format(region))
else:
return logger.error('Unknown error: {}'.format(
e.response['Error']['Message']))
for endpoint_srv in endpoint_services['ServiceConfigurations']:
nlb_arns = {}
service_id = endpoint_srv['ServiceId']
service_name = endpoint_srv['ServiceName']
service_state = endpoint_srv['ServiceState']
acceptance_required = endpoint_srv['AcceptanceRequired']
endpoint_service_nlb_arns = endpoint_srv['NetworkLoadBalancerArns']
# Fetch tags of NLBs and map into a dictionary
for nlb in endpoint_service_nlb_arns:
nlb_tags_response = elbv2_client.describe_tags(ResourceArns=[nlb])
nlb_tags = nlb_tags_response['TagDescriptions'][0]['Tags']
nlb_arns.update({nlb: [nlb_tags]})
logger.info(
'Recording Endpoint Service: {0} to nlbs {1} for account {2}'.
format(service_name, nlb_arns, account))
response = endpoint_service_table.put_item(
Item={
'id': service_id,
'ServiceName': service_name,
'AccountID': account,
'ServiceState': service_state,
'AcceptanceRequired': acceptance_required,
'NetworkLoadBalancerArns': endpoint_service_nlb_arns,
'Region': region,
'NLBTags': nlb_arns,
'ttl': ttl_expire_time
})
def import_cidrs(event, context):
"""
Take event data which should include AWS account and region and import
CIDR blocks that are in use
"""
dynamodb = boto3.resource('dynamodb')
client = boto3.client('ec2')
cidr_table = dynamodb.Table(os.environ['DYNAMODB_TABLE_CIDRS'])
cidrs = cidr_table.scan()['Items']
account = event['account']
region = event['region']
# ttl time to expire items in DynamoDB table, default 48 hours
# ttl provided in seconds
ttl_expire_time = (int(time.time()) +
os.environ.get('TTL_EXPIRE_TIME', 172800))
ACCESS_KEY, SECRET_KEY, SESSION_TOKEN = establish_role(account)
client = boto3.client('ec2',
aws_access_key_id=ACCESS_KEY,
aws_secret_access_key=SECRET_KEY,
aws_session_token=SESSION_TOKEN,
region_name=region)
vpcs = client.describe_vpcs()
for vpc in vpcs['Vpcs']:
vpc_cidr = vpc['CidrBlock']
vpc_id = vpc['VpcId']
unique_id = "{0}{1}{2}{3}".format(account, vpc_cidr, region, vpc_id)
logger.info("Found vpc-id: {0} and cidr: {1} ({2}) from "
"account {3} in Dyanmodb".format(vpc['VpcId'],
vpc['CidrBlock'], region,
account))
response = cidr_table.put_item(
Item={
'id': unique_id,
'cidr': vpc_cidr,
'AccountID': account,
'Region': region,
'VpcId': vpc_id,
},
ConditionExpression='attribute_not_exists(unique_id)')
logger.info("Dynamodb response: {}".format(response))
if 'CidrBlockAssociationSet' in vpc:
for cidr_associaton in vpc['CidrBlockAssociationSet']:
cidr_state = cidr_associaton['CidrBlockState']['State']
if cidr_state == "associated":
unique_id = ("{0}{1}{2}{3}".format(
account, cidr_associaton['CidrBlock'], region, vpc_id))
response = cidr_table.put_item(
Item={
'id': unique_id,
'cidr': cidr_associaton['CidrBlock'],
'AccountID': account,
'Region': region,
'VpcId': vpc_id,
'ttl': ttl_expire_time
})
logger.info("Dynamodb response: {}".format(response))
else:
logger.info("CIDR: {} not associated".format(
cidr_associaton['CidrBlock']))
def prune_tables(event, context):
recorded_eips = {}
recorded_cidrs = {}
recorded_nats = {}
dynamodb = boto3.resource('dynamodb')
client = boto3.client('ec2')
nats_table = dynamodb.Table(os.environ['DYNAMODB_TABLE_NAT_GATEWAYS'])
cidrs_table = dynamodb.Table(os.environ['DYNAMODB_TABLE_CIDRS'])
eips_table = dynamodb.Table(os.environ['DYNAMODB_TABLE_EIP'])
accounts_table = dynamodb.Table(os.environ['DYNAMODB_TABLE_ACCOUNTS'])
accounts = accounts_table.scan()['Items']
cidrs = cidrs_table.scan()['Items']
scan_cidrs_table = cidrs_table.scan(
AttributesToGet=['id', 'AccountID', 'Region', 'VpcId', 'cidr'
])['Items']
scan_nats_table = nats_table.scan(
AttributesToGet=['id', 'AccountID', 'PublicIp', 'VpcId', 'Region'
])['Items']
scan_eips_table = eips_table.scan(
AttributesToGet=['id', 'AccountID', 'Region'])['Items']
for i in scan_cidrs_table:
cidr_id = i['id']
recorded_cidrs[cidr_id] = [
i['VpcId'], i['cidr'], i['Region'], i['AccountID']
]
logger.info("CIDRS Found: Account: {0} Region: {1} Cidr: {2}".format(
i['AccountID'], i['Region'], i['cidr']))
for j in scan_nats_table:
nats_id = j['id']
recorded_nats[nats_id] = [
j['AccountID'], j['PublicIp'], j['VpcId'], j['Region']
]
logger.info("Nat Found: Account: {0} PublicIp: {1} VpcId: {2}".format(
j['AccountID'], j['PublicIp'], j['VpcId']))
for e in scan_eips_table:
eips_id = e['id']
recorded_eips[eips_id] = [e['AccountID'], e['Region']]
logger.info("EIP Found: Account: {0} PublicIp: {1} Region: {2}".format(
e['AccountID'], e['id'], e['Region']))
regions = ([
region['RegionName'] for region in client.describe_regions()['Regions']
])
for region in regions:
for acct in accounts:
ACCESS_KEY, SECRET_KEY, SESSION_TOKEN = establish_role(acct)
client = boto3.client('ec2',
aws_access_key_id=ACCESS_KEY,
aws_secret_access_key=SECRET_KEY,
aws_session_token=SESSION_TOKEN,
region_name=region)
nats = client.describe_nat_gateways()
vpcs = client.describe_vpcs()
eips = client.describe_addresses()
active_cidrs = ([
"{0}{1}{2}{3}".format(acct['id'], vpc['CidrBlock'], region,
vpc['VpcId']) for vpc in vpcs['Vpcs']
])
active_cidr_associations = ([
"{0}{1}{2}{3}".format(acct['id'], cidr_associaton['CidrBlock'],
region, vpc['VpcId'])
for vpc in vpcs['Vpcs']
for cidr_associaton in vpc['CidrBlockAssociationSet']
if 'CidrBlockAssociationSet' in vpc
and cidr_associaton['CidrBlockState']['State'] == "associated"
])
active_nats = ([
"{0}".format(nat['NatGatewayId'])
for nat in nats['NatGateways']
])
active_eips = ([
"{0}".format(eip['PublicIp']) for eip in eips['Addresses']
])
cidrs_keys_list = ([
k for k, v in recorded_cidrs.items()
if (v[3] == str(acct['id']) and v[2] == str(region))
])
nats_keys_list = ([
k for k, v in recorded_nats.items()
if (v[0] == str(acct['id']) and v[3] == str(region))
])
eips_keys_list = ([
k for k, v in recorded_eips.items()
if (v[0] == str(acct['id']) and v[1] == str(region))
])
logger.info("Current active_cidrs: {}".format(active_cidrs))
logger.info("Current active_cidr_associations: {}".format(
active_cidr_associations))
logger.info("Current recorded cidrs: {}".format(cidrs_keys_list))
logger.info("Current active_nats: {}".format(active_nats))
logger.info("Current recorded nats: {}".format(nats_keys_list))
logger.info("Current active_eips: {}".format(active_eips))
logger.info("Current recorded eips: {}".format(eips_keys_list))
all_cidrs = set(list(active_cidrs + active_cidr_associations))
cidrs_to_prune = list(set(cidrs_keys_list) - set(all_cidrs))
nats_to_prune = list(set(nats_keys_list) - set(active_nats))
eips_to_prune = list(set(eips_keys_list) - set(active_eips))
logger.info("cidrs_to_prune: {}".format(cidrs_to_prune))
logger.info("nats_to_prune: {}".format(nats_to_prune))
logger.info("eips_to_prune: {}".format(eips_to_prune))
for cidr in cidrs_to_prune:
logger.info("Found unused cidr: {} - removing".format(cidr))
cidrs_table.delete_item(Key={'id': cidr})
for nat in nats_to_prune:
logger.info("Found unused nat: {} - removing".format(nat))
nats_table.delete_item(Key={'id': nat})
for eip in eips_to_prune:
logger.info("Found unused eip: {} - removing".format(eip))
eips_table.delete_item(Key={'id': eip})
def import_cidrs(event, context):
dynamodb = boto3.resource('dynamodb')
client = boto3.client('ec2')
cidr_table = dynamodb.Table(os.environ['DYNAMODB_TABLE_CIDRS'])
accounts_table = dynamodb.Table(os.environ['DYNAMODB_TABLE_ACCOUNTS'])
accounts = accounts_table.scan()['Items']
cidrs = cidr_table.scan()['Items']
regions = [
region['RegionName'] for region in client.describe_regions()['Regions']
]
for region in regions:
for acct in accounts:
ACCESS_KEY, SECRET_KEY, SESSION_TOKEN = establish_role(acct)
client = boto3.client('ec2',
aws_access_key_id=ACCESS_KEY,
aws_secret_access_key=SECRET_KEY,
aws_session_token=SESSION_TOKEN,
region_name=region)
vpcs = client.describe_vpcs()
for vpc in vpcs['Vpcs']:
acct_id = acct['id']
vpc_cidr = vpc['CidrBlock']
vpc_id = vpc['VpcId']
unique_id = "{0}{1}{2}{3}".format(acct_id, vpc_cidr, region,
vpc_id)
logger.info("Found vpc-id: {0} and cidr: {1} ({2}) from "
"account {3} in Dyanmodb".format(
vpc['VpcId'], vpc['CidrBlock'], region,
acct['id']))
response = cidr_table.put_item(
Item={
'id': unique_id,
'cidr': vpc_cidr,
'AccountID': acct_id,
'Region': region,
'VpcId': vpc_id,
},
ConditionExpression='attribute_not_exists(unique_id)')
logger.info("Dynamodb response: {}".format(response))
if 'CidrBlockAssociationSet' in vpc:
for cidr_associaton in vpc['CidrBlockAssociationSet']:
cidr_state = cidr_associaton['CidrBlockState']['State']
if cidr_state == "associated":
unique_id = ("{0}{1}{2}{3}".format(
acct_id, cidr_associaton['CidrBlock'], region,
vpc_id))
response = cidr_table.put_item(
Item={
'id': unique_id,
'cidr': cidr_associaton['CidrBlock'],
'AccountID': acct_id,
'Region': region,
'VpcId': vpc_id,
},
ConditionExpression=(
'attribute_not_exists(unique_id)'))
logger.info(
"Dynamodb response: {}".format(response))
else:
logger.info("CIDR: {} not associated".format(
cidr_associaton['CidrBlock']))