def slurp(self):
"""
:returns: item_list - list of endpoints.
:returns: exception_map - A dict where the keys are a tuple containing the
location of the exception and the value is the actual exception
"""
self.prep_for_slurp()
item_list = []
exception_map = {}
from security_monkey.common.sts_connect import connect
for account in self.accounts:
for region in regions():
app.logger.debug(
"Checking {}/{}/{}".format(self.index, account, region.name))
try:
conn = connect(account, 'boto3.ec2.client', region=region)
all_vpc_endpoints_resp = self.wrap_aws_rate_limited_call(
conn.describe_vpc_endpoints
)
all_vpc_endpoints = all_vpc_endpoints_resp.get(
'VpcEndpoints', [])
except Exception as e:
if region.name not in TROUBLE_REGIONS:
exc = BotoConnectionIssue(
str(e), self.index, account, region.name)
self.slurp_exception(
(self.index, account, region.name), exc, exception_map)
continue
app.logger.debug("Found {} {}".format(
len(all_vpc_endpoints), self.i_am_plural))
for endpoint in all_vpc_endpoints:
endpoint_name = endpoint.get(u'VpcEndpointId')
if self.check_ignore_list(endpoint_name):
continue
service = endpoint.get('ServiceName', '').split('.')[-1]
config = {
"id": endpoint.get('VpcEndpointId'),
"policy_document": endpoint.get('PolicyDocument', {}),
"service_name": endpoint.get('ServiceName'),
"service": service,
"route_table_ids": endpoint.get('RouteTableIds', []),
"creation_time_stamp": str(endpoint.get('CreationTimestamp')),
"state": endpoint.get('State'),
"vpc_id": endpoint.get('VpcId'),
}
item = EndpointItem(
region=region.name, account=account, name=endpoint_name, config=config, source_watcher=self)
item_list.append(item)
return item_list, exception_map
def slurp(self):
"""
:returns: item_list - list of subnets.
:returns: exception_map - A dict where the keys are a tuple containing the
location of the exception and the value is the actual exception
"""
self.prep_for_slurp()
item_list = []
exception_map = {}
from security_monkey.common.sts_connect import connect
for account in self.accounts:
for region in regions():
app.logger.debug("Checking {}/{}/{}".format(self.index, account, region.name))
try:
conn = connect(account, 'vpc', region=region)
all_subnets = self.wrap_aws_rate_limited_call(
conn.get_all_subnets
)
except Exception as e:
if region.name not in TROUBLE_REGIONS:
exc = BotoConnectionIssue(str(e), self.index, account, region.name)
self.slurp_exception((self.index, account, region.name), exc, exception_map)
continue
app.logger.debug("Found {} {}".format(len(all_subnets), self.i_am_plural))
for subnet in all_subnets:
subnet_name = subnet.tags.get(u'Name', None)
if subnet_name:
subnet_name = "{0} ({1})".format(subnet_name, subnet.id)
else:
subnet_name = subnet.id
if self.check_ignore_list(subnet_name):
continue
config = {
"name": subnet.tags.get(u'Name', None),
"id": subnet.id,
"cidr_block": subnet.cidr_block,
"availability_zone": subnet.availability_zone,
# TODO:
# available_ip_address_count is likely to change often
# and should be in the upcoming ephemeral section.
# "available_ip_address_count": subnet.available_ip_address_count,
"defaultForAz": subnet.defaultForAz,
"mapPublicIpOnLaunch": subnet.mapPublicIpOnLaunch,
"state": subnet.state,
"tags": dict(subnet.tags),
"vpc_id": subnet.vpc_id
}
item = SubnetItem(region=region.name, account=account, name=subnet_name, config=config)
item_list.append(item)
return item_list, exception_map
def slurp(self):
"""
:returns: item_list - list of endpoints.
:returns: exception_map - A dict where the keys are a tuple containing the
location of the exception and the value is the actual exception
"""
self.prep_for_slurp()
item_list = []
exception_map = {}
from security_monkey.common.sts_connect import connect
for account in self.accounts:
for region in regions():
app.logger.debug("Checking {}/{}/{}".format(
self.index, account, region.name))
try:
conn = connect(account, 'boto3.ec2.client', region=region)
all_vpc_endpoints_resp = self.wrap_aws_rate_limited_call(
conn.describe_vpc_endpoints)
all_vpc_endpoints = all_vpc_endpoints_resp.get(
'VpcEndpoints', [])
except Exception as e:
if region.name not in TROUBLE_REGIONS:
exc = BotoConnectionIssue(str(e), self.index, account,
region.name)
self.slurp_exception(
(self.index, account, region.name), exc,
exception_map)
continue
app.logger.debug("Found {} {}".format(len(all_vpc_endpoints),
self.i_am_plural))
for endpoint in all_vpc_endpoints:
endpoint_name = endpoint.get('VpcEndpointId')
if self.check_ignore_list(endpoint_name):
continue
service = endpoint.get('ServiceName', '').split('.')[-1]
config = {
"id":
endpoint.get('VpcEndpointId'),
"policy_document":
endpoint.get('PolicyDocument', {}),
"service_name":
endpoint.get('ServiceName'),
"service":
service,
"route_table_ids":
endpoint.get('RouteTableIds', []),
"creation_time_stamp":
str(endpoint.get('CreationTimestamp')),
"state":
endpoint.get('State'),
"vpc_id":
endpoint.get('VpcId'),
}
item = EndpointItem(region=region.name,
account=account,
name=endpoint_name,
config=config,
source_watcher=self)
item_list.append(item)
return item_list, exception_map
def slurp(self):
"""
:returns: item_list - list of Flow Logs in use by account
:returns: exception_map - A dict where the keys are a tuple containing the
location of the exception and the value is the actual exception
"""
self.prep_for_slurp()
from security_monkey.common.sts_connect import connect
item_list = []
exception_map = {}
for account in self.accounts:
for region in regions():
app.logger.debug("Checking {}/{}/{}".format(
self.index, account, region.name))
response_items = []
try:
conn = connect(account, 'boto3.ec2.client', region=region)
next_token = None
while True:
if next_token:
response = self.wrap_aws_rate_limited_call(
conn.describe_flow_logs, NextToken=next_token)
else:
response = self.wrap_aws_rate_limited_call(
conn.describe_flow_logs)
response_items.extend(response.get('FlowLogs'))
if response.get('NextToken'):
next_token = response.get('NextToken')
else:
break
except Exception as e:
if region.name not in TROUBLE_REGIONS:
exc = BotoConnectionIssue(str(e), self.index, account,
region.name)
self.slurp_exception(
(self.index, account, region.name), exc,
exception_map)
continue
app.logger.debug("Found {} {}".format(len(response_items),
self.i_am_plural))
for response_item in response_items:
name = "{} ({})".format(response_item.get('LogGroupName'),
response_item.get('FlowLogId'))
if self.check_ignore_list(name):
continue
config = {
'creation_time':
str(response_item.get('CreationTime')),
'deliver_logs_permission_arn':
response_item.get('DeliverLogsPermissionArn'),
'flow_log_id':
response_item.get('FlowLogId'),
'flow_log_status':
response_item.get('FlowLogStatus'),
'log_group_name':
response_item.get('LogGroupName'),
'resource_id':
response_item.get('ResourceId'),
'traffic_type':
response_item.get('TrafficType')
}
item = FlowLogItem(region=region.name,
account=account,
name=name,
config=dict(config),
source_watcher=self)
item_list.append(item)
return item_list, exception_map
def slurp(self):
"""
:returns: item_list - list of subnets.
:returns: exception_map - A dict where the keys are a tuple containing the
location of the exception and the value is the actual exception
"""
self.prep_for_slurp()
item_list = []
exception_map = {}
from security_monkey.common.sts_connect import connect
for account in self.accounts:
for region in regions():
app.logger.debug("Checking {}/{}/{}".format(
self.index, account, region.name))
try:
conn = connect(account, 'vpc', region=region)
all_subnets = self.wrap_aws_rate_limited_call(
conn.get_all_subnets)
except Exception as e:
if region.name not in TROUBLE_REGIONS:
exc = BotoConnectionIssue(str(e), self.index, account,
region.name)
self.slurp_exception(
(self.index, account, region.name), exc,
exception_map)
continue
app.logger.debug("Found {} {}".format(len(all_subnets),
self.i_am_plural))
for subnet in all_subnets:
subnet_name = subnet.tags.get(u'Name', None)
if subnet_name:
subnet_name = "{0} ({1})".format(
subnet_name, subnet.id)
else:
subnet_name = subnet.id
if self.check_ignore_list(subnet_name):
continue
config = {
"name": subnet.tags.get(u'Name', None),
"id": subnet.id,
"cidr_block": subnet.cidr_block,
"availability_zone": subnet.availability_zone,
# TODO:
# available_ip_address_count is likely to change often
# and should be in the upcoming ephemeral section.
# "available_ip_address_count": subnet.available_ip_address_count,
"defaultForAz": subnet.defaultForAz,
"mapPublicIpOnLaunch": subnet.mapPublicIpOnLaunch,
"state": subnet.state,
"tags": dict(subnet.tags),
"vpc_id": subnet.vpc_id
}
item = SubnetItem(region=region.name,
account=account,
name=subnet_name,
config=config)
item_list.append(item)
return item_list, exception_map
def slurp(self):
"""
:returns: item_list - list of VPCs.
:returns: exception_map - A dict where the keys are a tuple containing the
location of the exception and the value is the actual exception
"""
self.prep_for_slurp()
item_list = []
exception_map = {}
from security_monkey.common.sts_connect import connect
for account in self.accounts:
account_db = Account.query.filter(Account.name == account).first()
account_number = account_db.number
for region in regions():
app.logger.debug("Checking {}/{}/{}".format(self.index, account, region.name))
try:
conn = connect(account, 'vpc', region=region)
all_vpcs = self.wrap_aws_rate_limited_call(
conn.get_all_vpcs
)
all_dhcp_options = self.wrap_aws_rate_limited_call(
conn.get_all_dhcp_options
)
all_internet_gateways = self.wrap_aws_rate_limited_call(
conn.get_all_internet_gateways
)
except Exception as e:
if region.name not in TROUBLE_REGIONS:
exc = BotoConnectionIssue(str(e), 'vpc', account, region.name)
self.slurp_exception((self.index, account, region.name), exc, exception_map,
source="{}-watcher".format(self.index))
continue
app.logger.debug("Found {} {}".format(len(all_vpcs), self.i_am_plural))
dhcp_options = {dhcp_option.id: dhcp_option.options for dhcp_option in all_dhcp_options}
internet_gateways = {}
for internet_gateway in all_internet_gateways:
for attachment in internet_gateway.attachments:
internet_gateways[attachment.vpc_id] = {
"id": internet_gateway.id,
"state": attachment.state
}
for vpc in all_vpcs:
vpc_name = vpc.tags.get(u'Name', None)
vpc_name = "{0} ({1})".format(vpc_name, vpc.id)
if self.check_ignore_list(vpc_name):
continue
dhcp_options.get(vpc.dhcp_options_id, {}).update(
{"id": vpc.dhcp_options_id}
)
arn = 'arn:aws:ec2:{region}:{account_number}:vpc/{vpc_id}'.format(
region=region.name,
account_number=account_number,
vpc_id=vpc.id)
config = {
"name": vpc.tags.get(u'Name', None),
"arn": arn,
"id": vpc.id,
"cidr_block": vpc.cidr_block,
"instance_tenancy": vpc.instance_tenancy,
"is_default": vpc.is_default,
"state": vpc.state,
"tags": dict(vpc.tags),
"classic_link_enabled": vpc.classic_link_enabled,
"dhcp_options": deep_dict(dhcp_options.get(vpc.dhcp_options_id, {})),
"internet_gateway": internet_gateways.get(vpc.id, None)
}
item = VPCItem(region=region.name, account=account, name=vpc_name, arn=arn, config=config)
item_list.append(item)
return item_list, exception_map
def slurp(self):
"""
:returns: item_list - list of VPCs.
:returns: exception_map - A dict where the keys are a tuple containing the
location of the exception and the value is the actual exception
"""
self.prep_for_slurp()
item_list = []
exception_map = {}
from security_monkey.common.sts_connect import connect
for account in self.accounts:
for region in regions():
app.logger.debug("Checking {}/{}/{}".format(
self.index, account, region.name))
try:
conn = connect(account, 'vpc', region=region)
all_vpcs = self.wrap_aws_rate_limited_call(
conn.get_all_vpcs)
all_dhcp_options = self.wrap_aws_rate_limited_call(
conn.get_all_dhcp_options)
all_internet_gateways = self.wrap_aws_rate_limited_call(
conn.get_all_internet_gateways)
except Exception as e:
if region.name not in TROUBLE_REGIONS:
exc = BotoConnectionIssue(str(e), 'vpc', account,
region.name)
self.slurp_exception(
(self.index, account, region.name), exc,
exception_map)
continue
app.logger.debug("Found {} {}".format(len(all_vpcs),
self.i_am_plural))
dhcp_options = {
dhcp_option.id: dhcp_option.options
for dhcp_option in all_dhcp_options
}
internet_gateways = {}
for internet_gateway in all_internet_gateways:
for attachment in internet_gateway.attachments:
internet_gateways[attachment.vpc_id] = {
"id": internet_gateway.id,
"state": attachment.state
}
for vpc in all_vpcs:
vpc_name = vpc.tags.get(u'Name', None)
vpc_name = "{0} ({1})".format(vpc_name, vpc.id)
if self.check_ignore_list(vpc_name):
continue
dhcp_options.get(vpc.dhcp_options_id,
{}).update({"id": vpc.dhcp_options_id})
config = {
"name":
vpc.tags.get(u'Name', None),
"id":
vpc.id,
"cidr_block":
vpc.cidr_block,
"instance_tenancy":
vpc.instance_tenancy,
"is_default":
vpc.is_default,
"state":
vpc.state,
"tags":
dict(vpc.tags),
"classic_link_enabled":
vpc.classic_link_enabled,
"dhcp_options":
deep_dict(dhcp_options.get(vpc.dhcp_options_id, {})),
"internet_gateway":
internet_gateways.get(vpc.id, None)
}
item = VPCItem(region=region.name,
account=account,
name=vpc_name,
config=config)
item_list.append(item)
return item_list, exception_map
def slurp(self):
"""
:returns: item_list - list of nat_gateways.
:returns: exception_map - A dict where the keys are a tuple containing the
location of the exception and the value is the actual exception
"""
self.prep_for_slurp()
item_list = []
exception_map = {}
from security_monkey.common.sts_connect import connect
for account in self.accounts:
for region in regions():
app.logger.debug("Checking {}/{}/{}".format(
self.index, account, region.name))
try:
conn = connect(account, 'boto3.ec2.client', region=region)
all_nat_gateways_resp = self.wrap_aws_rate_limited_call(
conn.describe_nat_gateways)
all_nat_gateways = all_nat_gateways_resp.get(
'NatGateways', [])
except Exception as e:
if region.name not in TROUBLE_REGIONS:
exc = BotoConnectionIssue(str(e), self.index, account,
region.name)
self.slurp_exception(
(self.index, account, region.name), exc,
exception_map)
continue
app.logger.debug("Found {} {}".format(len(all_nat_gateways),
self.i_am_plural))
for nat_gateway in all_nat_gateways:
nat_gateway_name = nat_gateway.get(u'NatGatewayId')
if self.check_ignore_list(nat_gateway_name):
continue
natGatewayAddresses = []
for address in nat_gateway.get('NatGatewayAddresses'):
next_addr = {
'public_ip':
address.get('PublicIp', None),
'allocation_id':
address.get('AllocationId', None),
'private_ip':
address.get('PrivateIp', None),
'network_interface_id':
address.get('NetworkInterfaceId', None)
}
natGatewayAddresses.append(next_addr)
config = {
'id': nat_gateway.get('NatGatewayId'),
'subnet_id': nat_gateway.get('SubnetId'),
'vpc_id': nat_gateway.get('VpcId'),
'create_time': str(nat_gateway.get('CreateTime')),
'delete_time': str(nat_gateway.get('DeleteTime')),
'nat_gateway_addresses': natGatewayAddresses,
'state': nat_gateway.get('State'),
'failure_code': nat_gateway.get('FailureCode'),
'failure_message': nat_gateway.get('FailureMessage')
}
item = NATGatewayItem(region=region.name,
account=account,
name=nat_gateway_name,
config=config)
item_list.append(item)
return item_list, exception_map
def slurp(self):
"""
:returns: item_list - list of nat_gateways.
:returns: exception_map - A dict where the keys are a tuple containing the
location of the exception and the value is the actual exception
"""
self.prep_for_slurp()
item_list = []
exception_map = {}
from security_monkey.common.sts_connect import connect
for account in self.accounts:
for region in regions():
app.logger.debug(
"Checking {}/{}/{}".format(self.index, account, region.name))
try:
conn = connect(account, 'boto3.ec2.client', region=region)
all_nat_gateways_resp = self.wrap_aws_rate_limited_call(
conn.describe_nat_gateways
)
all_nat_gateways = all_nat_gateways_resp.get(
'NatGateways', [])
except Exception as e:
if region.name not in TROUBLE_REGIONS:
exc = BotoConnectionIssue(
str(e), self.index, account, region.name)
self.slurp_exception(
(self.index, account, region.name), exc, exception_map)
continue
app.logger.debug("Found {} {}".format(
len(all_nat_gateways), self.i_am_plural))
for nat_gateway in all_nat_gateways:
nat_gateway_name = nat_gateway.get(u'NatGatewayId')
if self.check_ignore_list(nat_gateway_name):
continue
natGatewayAddresses = []
for address in nat_gateway.get('NatGatewayAddresses'):
next_addr = {
'public_ip': address.get('PublicIp', None),
'allocation_id': address.get('AllocationId', None),
'private_ip': address.get('PrivateIp', None),
'network_interface_id': address.get('NetworkInterfaceId', None)
}
natGatewayAddresses.append(next_addr)
config = {
'id': nat_gateway.get('NatGatewayId'),
'subnet_id': nat_gateway.get('SubnetId'),
'vpc_id': nat_gateway.get('VpcId'),
'create_time': str(nat_gateway.get('CreateTime')),
'delete_time': str(nat_gateway.get('DeleteTime')),
'nat_gateway_addresses': natGatewayAddresses,
'state': nat_gateway.get('State'),
'failure_code': nat_gateway.get('FailureCode'),
'failure_message': nat_gateway.get('FailureMessage')
}
item = NATGatewayItem(
region=region.name, account=account, name=nat_gateway_name, config=config)
item_list.append(item)
return item_list, exception_map
def slurp(self):
"""
:returns: item_list - list of route tables.
:returns: exception_map - A dict where the keys are a tuple containing the
location of the exception and the value is the actual exception
"""
self.prep_for_slurp()
item_list = []
exception_map = {}
from security_monkey.common.sts_connect import connect
for account in self.accounts:
account_db = Account.query.filter(Account.name == account).first()
account_number = account_db.number
for region in regions():
app.logger.debug("Checking {}/{}/{}".format(self.index, account, region.name))
try:
conn = connect(account, 'vpc', region=region)
all_route_tables = self.wrap_aws_rate_limited_call(
conn.get_all_route_tables
)
except Exception as e:
if region.name not in TROUBLE_REGIONS:
exc = BotoConnectionIssue(str(e), self.index, account, region.name)
self.slurp_exception((self.index, account, region.name), exc, exception_map,
source="{}-watcher".format(self.index))
continue
app.logger.debug("Found {} {}".format(len(all_route_tables), self.i_am_plural))
for route_table in all_route_tables:
subnet_name = route_table.tags.get(u'Name', None)
if subnet_name:
subnet_name = "{0} ({1})".format(subnet_name, route_table.id)
else:
subnet_name = route_table.id
if self.check_ignore_list(subnet_name):
continue
routes = []
for boto_route in route_table.routes:
routes.append({
"destination_cidr_block": boto_route.destination_cidr_block,
"gateway_id": boto_route.gateway_id,
"instance_id": boto_route.instance_id,
"interface_id": boto_route.interface_id,
"state": boto_route.state,
"vpc_peering_connection_id": boto_route.vpc_peering_connection_id
})
associations = []
for boto_association in route_table.associations:
associations.append({
"id": boto_association.id,
"main": boto_association.main,
"subnet_id": boto_association.subnet_id
})
arn = 'arn:aws:ec2:{region}:{account_number}:route-table/{route_table_id}'.format(
region=region.name,
account_number=account_number,
route_table_id=route_table.id)
config = {
"name": route_table.tags.get(u'Name', None),
"arn": arn,
"id": route_table.id,
"routes": routes,
"tags": dict(route_table.tags),
"vpc_id": route_table.vpc_id,
"associations": associations
}
item = RouteTableItem(region=region.name, account=account, name=subnet_name, arn=arn, config=config)
item_list.append(item)
return item_list, exception_map
def slurp(self):
"""
:returns: item_list - list of route tables.
:returns: exception_map - A dict where the keys are a tuple containing the
location of the exception and the value is the actual exception
"""
self.prep_for_slurp()
item_list = []
exception_map = {}
from security_monkey.common.sts_connect import connect
for account in self.accounts:
for region in regions():
app.logger.debug("Checking {}/{}/{}".format(self.index, account, region.name))
try:
conn = connect(account, 'vpc', region=region)
all_route_tables = self.wrap_aws_rate_limited_call(
conn.get_all_route_tables
)
except Exception as e:
if region.name not in TROUBLE_REGIONS:
exc = BotoConnectionIssue(str(e), self.index, account, region.name)
self.slurp_exception((self.index, account, region.name), exc, exception_map)
continue
app.logger.debug("Found {} {}".format(len(all_route_tables), self.i_am_plural))
for route_table in all_route_tables:
subnet_name = route_table.tags.get(u'Name', None)
if subnet_name:
subnet_name = "{0} ({1})".format(subnet_name, route_table.id)
else:
subnet_name = route_table.id
if self.check_ignore_list(subnet_name):
continue
routes = []
for boto_route in route_table.routes:
routes.append({
"destination_cidr_block": boto_route.destination_cidr_block,
"gateway_id": boto_route.gateway_id,
"instance_id": boto_route.instance_id,
"interface_id": boto_route.interface_id,
"state": boto_route.state,
"vpc_peering_connection_id": boto_route.vpc_peering_connection_id
})
associations = []
for boto_association in route_table.associations:
associations.append({
"id": boto_association.id,
"main": boto_association.main,
"subnet_id": boto_association.subnet_id
})
config = {
"name": route_table.tags.get(u'Name', None),
"id": route_table.id,
"routes": routes,
"tags": dict(route_table.tags),
"vpc_id": route_table.vpc_id,
"associations": associations
}
item = RouteTableItem(region=region.name, account=account, name=subnet_name, config=config)
item_list.append(item)
return item_list, exception_map
