def slurp(self):
"""
:returns: item_list - list of IAM SSH Keypairs.
:returns: exception_map - A dict where the keys are a tuple containing the
location of the exception and the value is the actual exception
"""
item_list = []
exception_map = {}
from security_monkey.common.sts_connect import connect
for account in self.accounts:
try:
ec2 = connect(account, 'ec2')
regions = ec2.get_all_regions()
except Exception as e: # EC2ResponseError
# Some Accounts don't subscribe to EC2 and will throw an exception here.
exc = BotoConnectionIssue(str(e), 'keypair', account, None)
self.slurp_exception((self.index, account), exc, exception_map)
continue
for region in regions:
app.logger.debug("Checking {}/{}/{}".format(
Keypair.index, account, region.name))
try:
rec2 = connect(account, 'ec2', region=region)
kps = self.wrap_aws_rate_limited_call(
rec2.get_all_key_pairs)
except Exception as e:
if region.name not in TROUBLE_REGIONS:
exc = BotoConnectionIssue(str(e), 'keypair', account,
region.name)
self.slurp_exception(
(self.index, account, region.name), exc,
exception_map)
continue
app.logger.debug("Found {} {}".format(len(kps),
Keypair.i_am_plural))
for kp in kps:
### Check if this Keypair is on the Ignore List ###
ignore_item = False
for ignore_item_name in IGNORE_PREFIX[self.index]:
if kp.name.lower().startswith(
ignore_item_name.lower()):
ignore_item = True
break
if ignore_item:
continue
item_list.append(
KeypairItem(region=region.name,
account=account,
name=kp.name,
config={'fingerprint': kp.fingerprint}))
return item_list, exception_map
def slurp(self):
"""
:returns: item_list - list of IAM SSH Keypairs.
:returns: exception_map - A dict where the keys are a tuple containing the
location of the exception and the value is the actual exception
"""
self.prep_for_slurp()
item_list = []
exception_map = {}
from security_monkey.common.sts_connect import connect
for account in self.accounts:
try:
account_db = Account.query.filter(Account.name == account).first()
account_number = account_db.number
ec2 = connect(account, 'ec2')
regions = ec2.get_all_regions()
except Exception as e: # EC2ResponseError
# Some Accounts don't subscribe to EC2 and will throw an exception here.
exc = BotoConnectionIssue(str(e), 'keypair', account, None)
self.slurp_exception((self.index, account), exc, exception_map, source="{}-watcher".format(self.index))
continue
for region in regions:
app.logger.debug("Checking {}/{}/{}".format(Keypair.index, account, region.name))
try:
rec2 = connect(account, 'ec2', region=region)
kps = self.wrap_aws_rate_limited_call(
rec2.get_all_key_pairs
)
except Exception as e:
if region.name not in TROUBLE_REGIONS:
exc = BotoConnectionIssue(str(e), 'keypair', account, region.name)
self.slurp_exception((self.index, account, region.name), exc, exception_map,
source="{}-watcher".format(self.index))
continue
app.logger.debug("Found {} {}".format(len(kps), Keypair.i_am_plural))
for kp in kps:
if self.check_ignore_list(kp.name):
continue
arn = 'arn:aws:ec2:{region}:{account_number}:key-pair/{name}'.format(
region=region.name,
account_number=account_number,
name=kp.name)
item_list.append(KeypairItem(region=region.name, account=account, name=kp.name, arn=arn,
config={
'fingerprint': kp.fingerprint,
'arn': arn,
'name': kp.name
}))
return item_list, exception_map
def slurp(self):
"""
:returns: item_list - list of IAM SSH Keypairs.
:returns: exception_map - A dict where the keys are a tuple containing the
location of the exception and the value is the actual exception
"""
self.prep_for_slurp()
item_list = []
exception_map = {}
from security_monkey.common.sts_connect import connect
for account in self.accounts:
try:
account_db = Account.query.filter(Account.name == account).first()
account_number = account_db.identifier
ec2 = connect(account, 'ec2')
regions = ec2.get_all_regions()
except Exception as e: # EC2ResponseError
# Some Accounts don't subscribe to EC2 and will throw an exception here.
exc = BotoConnectionIssue(str(e), 'keypair', account, None)
self.slurp_exception((self.index, account), exc, exception_map, source="{}-watcher".format(self.index))
continue
for region in regions:
app.logger.debug("Checking {}/{}/{}".format(Keypair.index, account, region.name))
try:
rec2 = connect(account, 'boto3.ec2.client', region=region)
kps = self.wrap_aws_rate_limited_call(
rec2.describe_key_pairs
)
except Exception as e:
if region.name not in TROUBLE_REGIONS:
exc = BotoConnectionIssue(str(e), 'keypair', account, region.name)
self.slurp_exception((self.index, account, region.name), exc, exception_map,
source="{}-watcher".format(self.index))
continue
app.logger.debug("Found {} {}".format(len(kps), Keypair.i_am_plural))
for kp in kps['KeyPairs']:
if self.check_ignore_list(kp['KeyName']):
continue
arn = ARN_PREFIX + ':ec2:{region}:{account_number}:key-pair/{name}'.format(
region=region.name,
account_number=account_number,
name=kp["KeyName"])
item_list.append(KeypairItem(region=region.name, account=account, name=kp["KeyName"], arn=arn,
config={
'fingerprint': kp["KeyFingerprint"],
'arn': arn,
'name': kp["KeyName"]
}, source_watcher=self))
return item_list, exception_map
def slurp(self):
"""
:returns: item_list - list of S3 Buckets.
:returns: exception_map - A dict where the keys are a tuple containing the
location of the exception and the value is the actual exception
"""
item_list = []
exception_map = {}
from security_monkey.common.sts_connect import connect
for account in self.accounts:
try:
s3conn = connect(account, 's3', calling_format=OrdinaryCallingFormat())
all_buckets = s3conn.get_all_buckets()
except Exception as e:
exc = BotoConnectionIssue(str(e), 's3', account, None)
self.slurp_exception((self.index, account), exc, exception_map)
continue
for bucket in all_buckets:
app.logger.debug("Slurping %s (%s) from %s" % (self.i_am_singular, bucket.name, account))
### Check if this bucket is on the Ignore List ###
ignore_item = False
for ignore_item_name in IGNORE_PREFIX[self.index]:
if bucket.name.lower().startswith(ignore_item_name.lower()):
ignore_item = True
break
if ignore_item:
continue
try:
region = self.translate_location_to_region(bucket.get_location())
if region == '':
s3regionconn = connect(account, 's3', calling_format=OrdinaryCallingFormat())
region = 'us-east-1'
else:
s3regionconn = connect(account, 's3', region=region, calling_format=OrdinaryCallingFormat())
bhandle = s3regionconn.get_bucket(bucket)
s3regionconn.close()
except Exception as e:
exc = S3PermissionsIssue(bucket.name)
# Unfortunately, we can't get the region, so the entire account
# will be skipped in find_changes, not just the bad bucket.
self.slurp_exception((self.index, account), exc, exception_map)
continue
app.logger.debug("Slurping %s (%s) from %s/%s" % (self.i_am_singular, bucket.name, account, region))
bucket_dict = self.conv_bucket_to_dict(bhandle, account, region, bucket.name, exception_map)
item = S3Item(account=account, region=region, name=bucket.name, config=bucket_dict)
item_list.append(item)
return item_list, exception_map
def slurp(self):
"""
:returns: item_list - list of IAM SSH Keypairs.
:returns: exception_map - A dict where the keys are a tuple containing the
location of the exception and the value is the actual exception
"""
item_list = []
exception_map = {}
from security_monkey.common.sts_connect import connect
for account in self.accounts:
try:
ec2 = connect(account, 'ec2')
regions = ec2.get_all_regions()
except Exception as e: # EC2ResponseError
# Some Accounts don't subscribe to EC2 and will throw an exception here.
exc = BotoConnectionIssue(str(e), 'keypair', account, None)
self.slurp_exception((self.index, account), exc, exception_map)
continue
for region in regions:
app.logger.debug("Checking {}/{}/{}".format(Keypair.index, account, region.name))
try:
rec2 = connect(account, 'ec2', region=region)
kps = self.wrap_aws_rate_limited_call(
rec2.get_all_key_pairs
)
except Exception as e:
if region.name not in TROUBLE_REGIONS:
exc = BotoConnectionIssue(str(e), 'keypair', account, region.name)
self.slurp_exception((self.index, account, region.name), exc, exception_map)
continue
app.logger.debug("Found {} {}".format(len(kps), Keypair.i_am_plural))
for kp in kps:
### Check if this Keypair is on the Ignore List ###
ignore_item = False
for ignore_item_name in IGNORE_PREFIX[self.index]:
if kp.name.lower().startswith(ignore_item_name.lower()):
ignore_item = True
break
if ignore_item:
continue
item_list.append(KeypairItem(region=region.name, account=account, name=kp.name,
config={
'fingerprint': kp.fingerprint
}))
return item_list, exception_map
def get_all_subnets(self, **kwargs):
from security_monkey.common.sts_connect import connect
conn = connect(kwargs['account_name'], 'boto3.ec2.client', region=kwargs['region'],
assumed_role=kwargs['assumed_role'])
all_subnets = self.wrap_aws_rate_limited_call(conn.describe_subnets)
return all_subnets.get('Subnets')
def slurp(self):
"""
:returns: item_list - list of Redshift Policies.
:returns: exception_map - A dict where the keys are a tuple containing the
location of the exception and the value is the actual exception
"""
self.prep_for_slurp()
from security_monkey.common.sts_connect import connect
item_list = []
exception_map = {}
for account in self.accounts:
for region in regions():
app.logger.debug("Checking {}/{}/{}".format(self.index, account, region.name))
try:
redshift = connect(account, 'redshift', region=region)
response = self.wrap_aws_rate_limited_call(
redshift.describe_clusters
)
all_clusters = response['DescribeClustersResponse']['DescribeClustersResult']['Clusters']
except Exception as e:
if region.name not in TROUBLE_REGIONS:
exc = BotoConnectionIssue(str(e), 'redshift', account, region.name)
self.slurp_exception((self.index, account, region.name), exc, exception_map)
continue
app.logger.debug("Found {} {}".format(len(all_clusters), Redshift.i_am_plural))
for cluster in all_clusters:
cluster_id = cluster['ClusterIdentifier']
if self.check_ignore_list(cluster_id):
continue
item = RedshiftCluster(region=region.name, account=account, name=cluster_id, config=dict(cluster))
item_list.append(item)
return item_list, exception_map
def get_all_certs_in_region(self, account, region, exception_map):
from security_monkey.common.sts_connect import connect
import traceback
all_certs = []
app.logger.debug("Checking {}/{}/{}".format(self.index, account, region))
try:
iamconn = connect(account, 'iam', region=region)
marker = None
while True:
certs = self.wrap_aws_rate_limited_call(
iamconn.list_server_certs,
marker=marker
)
all_certs.extend(certs.server_certificate_metadata_list)
if certs.is_truncated == u'true':
marker = certs.marker
else:
break
except Exception as e:
app.logger.warn(traceback.format_exc())
if region not in TROUBLE_REGIONS:
exc = BotoConnectionIssue(str(e), self.index, account, region)
self.slurp_exception((self.index, account, region), exc, exception_map)
app.logger.info("Found {} {} from {}/{}".format(len(all_certs), self.i_am_plural, account, region))
return all_certs
def get_all_certs_in_region(self, account, region, exception_map):
from security_monkey.common.sts_connect import connect
import traceback
all_certs = []
app.logger.debug("Checking {}/{}/{}".format(self.index, account,
region))
try:
iamconn = connect(account, 'iam', region=region)
marker = None
while True:
certs = self.wrap_aws_rate_limited_call(
iamconn.list_server_certs, marker=marker)
all_certs.extend(certs.server_certificate_metadata_list)
if certs.is_truncated == u'true':
marker = certs.marker
else:
break
except Exception as e:
app.logger.warn(traceback.format_exc())
if region not in TROUBLE_REGIONS:
exc = BotoConnectionIssue(str(e), self.index, account, region)
self.slurp_exception((self.index, account, region), exc,
exception_map)
app.logger.info("Found {} {} from {}/{}".format(
len(all_certs), self.i_am_plural, account, region))
return all_certs
def describe_db_subnet_groups(self, **kwargs):
from security_monkey.common.sts_connect import connect
db_sub_groups = []
rds = connect(kwargs['account_name'],
'boto3.rds.client',
region=kwargs['region'],
assumed_role=kwargs['assumed_role'])
marker = None
while True:
if marker:
response = self.wrap_aws_rate_limited_call(
rds.describe_db_subnet_groups, Marker=marker)
else:
response = self.wrap_aws_rate_limited_call(
rds.describe_db_subnet_groups)
db_sub_groups.extend(response.get('DBSubnetGroups'))
if response.get('Marker'):
marker = response.get('Marker')
else:
break
return db_sub_groups
def slurp(self):
"""
:returns: item_dict - list of IAM Account attributes.
:returns: exception_map - A dict where the keys are a tuple containing the
location of the exception and the value is the actual exception
"""
self.prep_for_slurp()
item_list = []
exception_map = {}
from security_monkey.common.sts_connect import connect
for account in self.accounts:
try:
iam = connect(account, 'iam')
account_summary = self.wrap_aws_rate_limited_call(
iam.get_account_summary)
except Exception as e:
exc = BotoConnectionIssue(str(e), 'iamaccount', account, None)
self.slurp_exception((self.index, account, 'universal'), exc,
exception_map)
continue
item_list.append(
IAMAccountItem(account=account,
name=account,
config=dict(account_summary)))
return item_list, exception_map
def get_all_subnets(self, **kwargs):
from security_monkey.common.sts_connect import connect
conn = connect(kwargs['account_name'], 'vpc', region=kwargs['region'],
assumed_role=kwargs['assumed_role'])
all_subnets = self.wrap_aws_rate_limited_call(conn.get_all_subnets)
return all_subnets
def slurp(self):
"""
:returns: item_list - list of connections
:returns: exception_map - A dict where the keys are a tuple containing the
location of the exception and the value is the actual exception
"""
self.prep_for_slurp()
from security_monkey.common.sts_connect import connect
item_list = []
exception_map = {}
for account in self.accounts:
for region in regions():
app.logger.debug("Checking {}/{}/{}".format(
self.index, account, region.name))
try:
dc = connect(account,
'boto3.directconnect.client',
region=region)
response = self.wrap_aws_rate_limited_call(
dc.describe_connections)
connections = response.get('connections')
except Exception as e:
if region.name not in TROUBLE_REGIONS:
exc = BotoConnectionIssue(str(e), self.index, account,
region.name)
self.slurp_exception(
(self.index, account, region.name), exc,
exception_map)
continue
app.logger.debug("Found {} {}.".format(len(connections),
self.i_am_plural))
for connection in connections:
name = connection['connectionName']
if self.check_ignore_list(name):
continue
config = {
'name': name,
'owner_account': connection.get('ownerAccount'),
'connection_id': connection.get('connectionId'),
'connection_name': connection.get('connectionName'),
'connection_state': connection.get('connectionState'),
'region': connection.get('region'),
'location': connection.get('location'),
'bandwidth': connection.get('bandwidth'),
'vlan': connection.get('vlan'),
'partner_name': connection.get('partnerName'),
}
item = ConnectionItem(region=region.name,
account=account,
name=name,
config=dict(config),
source_watcher=self)
item_list.append(item)
return item_list, exception_map
def slurp(self):
"""
:returns: item_list - list of endpoints.
:returns: exception_map - A dict where the keys are a tuple containing the
location of the exception and the value is the actual exception
"""
self.prep_for_slurp()
item_list = []
exception_map = {}
from security_monkey.common.sts_connect import connect
for account in self.accounts:
for region in regions():
app.logger.debug(
"Checking {}/{}/{}".format(self.index, account, region.name))
try:
conn = connect(account, 'boto3.ec2.client', region=region)
all_vpc_endpoints_resp = self.wrap_aws_rate_limited_call(
conn.describe_vpc_endpoints
)
all_vpc_endpoints = all_vpc_endpoints_resp.get(
'VpcEndpoints', [])
except Exception as e:
if region.name not in TROUBLE_REGIONS:
exc = BotoConnectionIssue(
str(e), self.index, account, region.name)
self.slurp_exception(
(self.index, account, region.name), exc, exception_map)
continue
app.logger.debug("Found {} {}".format(
len(all_vpc_endpoints), self.i_am_plural))
for endpoint in all_vpc_endpoints:
endpoint_name = endpoint.get(u'VpcEndpointId')
if self.check_ignore_list(endpoint_name):
continue
service = endpoint.get('ServiceName', '').split('.')[-1]
config = {
"id": endpoint.get('VpcEndpointId'),
"policy_document": endpoint.get('PolicyDocument', {}),
"service_name": endpoint.get('ServiceName'),
"service": service,
"route_table_ids": endpoint.get('RouteTableIds', []),
"creation_time_stamp": str(endpoint.get('CreationTimestamp')),
"state": endpoint.get('State'),
"vpc_id": endpoint.get('VpcId'),
}
item = EndpointItem(
region=region.name, account=account, name=endpoint_name, config=config, source_watcher=self)
item_list.append(item)
return item_list, exception_map
def slurp(self):
"""
:returns: item_list - list of configs.
:returns: exception_map - A dict where the keys are a tuple containing the
location of the exception and the value is the actual exception
"""
self.prep_for_slurp()
item_list = []
exception_map = {}
from security_monkey.common.sts_connect import connect
for account in self.accounts:
for region in regions():
app.logger.debug(
"Checking {}/{}/{}".format(self.index, account, region.name))
if region.name not in AVAILABLE_REGIONS:
continue
try:
configService = connect(
account, 'boto3.config.client', region=region)
response = self.wrap_aws_rate_limited_call(
configService.describe_config_rules
)
config_rules = response.get('ConfigRules', [])
except Exception as e:
app.logger.debug("Exception found: {}".format(e))
if region.name not in TROUBLE_REGIONS:
exc = BotoConnectionIssue(
str(e), self.index, account, region.name)
self.slurp_exception(
(self.index, account, region.name), exc, exception_map)
continue
app.logger.debug("Found {} {}.".format(
len(config_rules), self.i_am_plural))
for config_rule in config_rules:
name = config_rule.get('ConfigRuleName')
if self.check_ignore_list(name):
continue
item_config = {
'config_rule': name,
'config_rule_arn': config_rule.get('ConfigRuleArn'),
'config_rule_id': config_rule.get('ConfigRuleId'),
'scope': config_rule.get('Scope', {}),
'source': config_rule.get('Source', {}),
'imput_parameters': config_rule.get('InputParameters'),
'maximum_execution_frequency': config_rule.get('MaximumExecutionFrequency'),
'config_rule_state': config_rule.get('ConfigRuleState'),
}
item = ConfigItem(
region=region.name, account=account, name=name,
arn=config_rule.get('ConfigRuleArn'), config=item_config)
item_list.append(item)
return item_list, exception_map
def describe_vpc_peering_connections(self, **kwargs):
from security_monkey.common.sts_connect import connect
conn = connect(kwargs['account_name'], 'boto3.ec2.client', region=kwargs['region'],
assumed_role=kwargs['assumed_role'])
peering_info = self.wrap_aws_rate_limited_call(
conn.describe_vpc_peering_connections)
return peering_info
def get_all_es_domains_in_region(self, account, region):
from security_monkey.common.sts_connect import connect
client = connect(account, "boto3.es.client", region=region)
app.logger.debug("Checking {}/{}/{}".format(ElasticSearchService.index, account, region.name))
# No need to paginate according to: client.can_paginate("list_domain_names")
domains = self.wrap_aws_rate_limited_call(client.list_domain_names)["DomainNames"]
return client, domains
def describe_vpc_peering_connections(self, **kwargs):
from security_monkey.common.sts_connect import connect
conn = connect(kwargs['account_name'], 'boto3.ec2.client', region=kwargs['region'],
assumed_role=kwargs['assumed_role'])
peering_info = self.wrap_aws_rate_limited_call(
conn.describe_vpc_peering_connections)
return peering_info
def describe_route_tables(self, **kwargs):
from security_monkey.common.sts_connect import connect
conn = connect(kwargs['account_name'], 'boto3.ec2.client',
region=kwargs['region'], assumed_role=kwargs['assumed_role'])
response = self.wrap_aws_rate_limited_call(conn.describe_route_tables)
all_route_tables = response.get('RouteTables', [])
return all_route_tables
def describe_db_instances(self, **kwargs):
from security_monkey.common.sts_connect import connect
rds = connect(kwargs['account_name'], 'boto3.rds.client', region=kwargs['region'],
assumed_role=kwargs['assumed_role'])
response = self.wrap_aws_rate_limited_call(rds.describe_db_instances)
rds_db_instances = response.get('DBInstances')
return rds_db_instances
def describe_db_instances(self, **kwargs):
from security_monkey.common.sts_connect import connect
rds = connect(kwargs['account_name'], 'boto3.rds.client', region=kwargs['region'],
assumed_role=kwargs['assumed_role'])
response = self.wrap_aws_rate_limited_call(rds.describe_db_instances)
rds_db_instances = response.get('DBInstances')
return rds_db_instances
def describe_configuration_recorders(self, **kwargs):
from security_monkey.common.sts_connect import connect
config_service = connect(kwargs["account_name"], "boto3.config.client", region=kwargs["region"])
response = self.wrap_aws_rate_limited_call(config_service.describe_configuration_recorders)
config_recorders = response.get("ConfigurationRecorders", [])
return [recorder for recorder in config_recorders if not self.check_ignore_list(recorder.get("name"))]
def describe_route_tables(self, **kwargs):
from security_monkey.common.sts_connect import connect
conn = connect(kwargs['account_name'], 'boto3.ec2.client',
region=kwargs['region'], assumed_role=kwargs['assumed_role'])
response = self.wrap_aws_rate_limited_call(conn.describe_route_tables)
all_route_tables = response.get('RouteTables', [])
return all_route_tables
def slurp(self):
"""
:returns: item_list - list of subnets.
:returns: exception_map - A dict where the keys are a tuple containing the
location of the exception and the value is the actual exception
"""
self.prep_for_slurp()
item_list = []
exception_map = {}
from security_monkey.common.sts_connect import connect
for account in self.accounts:
for region in regions():
app.logger.debug("Checking {}/{}/{}".format(self.index, account, region.name))
try:
conn = connect(account, 'vpc', region=region)
all_subnets = self.wrap_aws_rate_limited_call(
conn.get_all_subnets
)
except Exception as e:
if region.name not in TROUBLE_REGIONS:
exc = BotoConnectionIssue(str(e), self.index, account, region.name)
self.slurp_exception((self.index, account, region.name), exc, exception_map)
continue
app.logger.debug("Found {} {}".format(len(all_subnets), self.i_am_plural))
for subnet in all_subnets:
subnet_name = subnet.tags.get(u'Name', None)
if subnet_name:
subnet_name = "{0} ({1})".format(subnet_name, subnet.id)
else:
subnet_name = subnet.id
if self.check_ignore_list(subnet_name):
continue
config = {
"name": subnet.tags.get(u'Name', None),
"id": subnet.id,
"cidr_block": subnet.cidr_block,
"availability_zone": subnet.availability_zone,
# TODO:
# available_ip_address_count is likely to change often
# and should be in the upcoming ephemeral section.
# "available_ip_address_count": subnet.available_ip_address_count,
"defaultForAz": subnet.defaultForAz,
"mapPublicIpOnLaunch": subnet.mapPublicIpOnLaunch,
"state": subnet.state,
"tags": dict(subnet.tags),
"vpc_id": subnet.vpc_id
}
item = SubnetItem(region=region.name, account=account, name=subnet_name, config=config)
item_list.append(item)
return item_list, exception_map
def describe_network_acls(self, **kwargs):
from security_monkey.common.sts_connect import connect
conn = connect(kwargs['account_name'], 'boto3.ec2.client', region=kwargs['region'],
assumed_role=kwargs['assumed_role'])
networkacls_resp = self.wrap_aws_rate_limited_call(
conn.describe_network_acls)
networkacls = networkacls_resp.get('NetworkAcls', [])
return networkacls
def describe_dhcp_options(self, **kwargs):
from security_monkey.common.sts_connect import connect
conn = connect(kwargs['account_name'], 'boto3.ec2.client', region=kwargs['region'],
assumed_role=kwargs['assumed_role'])
dhcp_option_sets_resp = self.wrap_aws_rate_limited_call(
conn.describe_dhcp_options)
dhcp_option_sets = dhcp_option_sets_resp.get('DhcpOptions', [])
return dhcp_option_sets
def slurp(self):
"""
:returns: item_list - list of IAM Groups.
:returns: exception_map - A dict where the keys are a tuple containing the
location of the exception and the value is the actual exception
"""
self.prep_for_slurp()
item_list = []
exception_map = {}
from security_monkey.common.sts_connect import connect
for account in self.accounts:
try:
iam = connect(account, 'iam')
groups = self.get_all_groups(iam)
except Exception as e:
exc = BotoConnectionIssue(str(e), 'iamgroup', account, None)
self.slurp_exception((self.index, account, 'universal'), exc, exception_map)
continue
for group in groups:
app.logger.debug("Slurping %s (%s) from %s" % (self.i_am_singular, group.group_name, account))
if self.check_ignore_list(group.group_name):
continue
item_config = {
'group': dict(group),
'grouppolicies': {},
'users': {}
}
### GROUP POLICIES ###
group_policies = self.get_all_group_policies(iam, group.group_name)
for policy_name in group_policies:
policy = self.wrap_aws_rate_limited_call(iam.get_group_policy, group.group_name, policy_name)
policy = policy.policy_document
policy = urllib.unquote(policy)
try:
policydict = json.loads(policy)
except:
exc = InvalidAWSJSON(policy)
self.slurp_exception((self.index, account, 'universal', group.group_name), exc, exception_map)
item_config['grouppolicies'][policy_name] = dict(policydict)
### GROUP USERS ###
group_users = self.get_all_group_users(iam, group['group_name'])
for user in group_users:
item_config['users'][user.arn] = user.user_name
item = IAMGroupItem(account=account, name=group.group_name, config=item_config)
item_list.append(item)
return item_list, exception_map
def describe_images(self, **kwargs):
from security_monkey.common.sts_connect import connect
ec2 = connect(kwargs['account_name'], 'boto3.ec2.client', region=kwargs['region'],
assumed_role=kwargs['assumed_role'])
response = self.wrap_aws_rate_limited_call(ec2.describe_images,
Owners=['self'])
images = response.get('Images')
return images
def slurp(self):
"""
:returns: item_list - list of Redshift Policies.
:returns: exception_map - A dict where the keys are a tuple containing the
location of the exception and the value is the actual exception
"""
self.prep_for_slurp()
from security_monkey.common.sts_connect import connect
item_list = []
exception_map = {}
for account in self.accounts:
account_db = Account.query.filter(Account.name == account).first()
account_number = account_db.identifier
for region in regions():
app.logger.debug("Checking {}/{}/{}".format(self.index, account, region.name))
try:
redshift = connect(account, 'redshift', region=region)
all_clusters = []
marker = None
while True:
response = self.wrap_aws_rate_limited_call(
redshift.describe_clusters,
marker=marker
)
all_clusters.extend(response['DescribeClustersResponse']['DescribeClustersResult']['Clusters'])
if response['DescribeClustersResponse']['DescribeClustersResult']['Marker'] is not None:
marker = response['DescribeClustersResponse']['DescribeClustersResult']['Marker']
else:
break
except Exception as e:
if region.name not in TROUBLE_REGIONS:
exc = BotoConnectionIssue(str(e), 'redshift', account, region.name)
self.slurp_exception((self.index, account, region.name), exc, exception_map,
source="{}-watcher".format(self.index))
continue
app.logger.debug("Found {} {}".format(len(all_clusters), Redshift.i_am_plural))
for cluster in all_clusters:
cluster_id = cluster['ClusterIdentifier']
if self.check_ignore_list(cluster_id):
continue
arn = ARN_PREFIX + ':redshift:{region}:{account_number}:cluster:{name}'.format(
region=region.name,
account_number=account_number,
name=cluster_id)
cluster['arn'] = arn
item = RedshiftCluster(region=region.name, account=account, name=cluster_id, arn=arn,
config=dict(cluster), source_watcher=self)
item_list.append(item)
return item_list, exception_map
def describe_instances(self, **kwargs):
from security_monkey.common.sts_connect import connect
ec2 = connect(kwargs['account_name'],
'boto3.ec2.client',
region=kwargs['region'],
assumed_role=kwargs['assumed_role'])
response = self.wrap_aws_rate_limited_call(ec2.describe_instances)
reservations = response.get('Reservations')
return reservations
def describe_instances(self, **kwargs):
from security_monkey.common.sts_connect import connect
ec2 = connect(kwargs['account_name'], 'boto3.ec2.client', region=kwargs['region'],
assumed_role=kwargs['assumed_role'])
response = self.wrap_aws_rate_limited_call(
ec2.describe_instances
)
reservations = response.get('Reservations')
return reservations
def describe_volumes(self, **kwargs):
from security_monkey.common.sts_connect import connect
ec2 = connect(
kwargs["account_name"], "boto3.ec2.client", region=kwargs["region"], assumed_role=kwargs["assumed_role"]
)
response = self.wrap_aws_rate_limited_call(ec2.describe_volumes)
volumes = response.get("Volumes")
return [volume for volume in volumes if not self.check_ignore_list(volume_name(volume))]
def slurp(self):
"""
:returns: item_list - list of SQS Policies.
:returns: exception_map - A dict where the keys are a tuple containing the
location of the exception and the value is the actual exception
"""
item_list = []
exception_map = {}
from security_monkey.common.sts_connect import connect
for account in self.accounts:
for region in regions():
app.logger.debug("Checking {}/{}/{}".format(SQS.index, account, region.name))
try:
sqs = connect(account, 'sqs', region=region)
all_queues = self.wrap_aws_rate_limited_call(
sqs.get_all_queues
)
except Exception as e:
if region.name not in TROUBLE_REGIONS:
exc = BotoConnectionIssue(str(e), 'sqs', account, region.name)
self.slurp_exception((self.index, account, region.name), exc, exception_map)
continue
app.logger.debug("Found {} {}".format(len(all_queues), SQS.i_am_plural))
for q in all_queues:
### Check if this Queue is on the Ignore List ###
ignore_item = False
for ignore_item_name in IGNORE_PREFIX[self.index]:
if q.name.lower().startswith(ignore_item_name.lower()):
ignore_item = True
break
if ignore_item:
continue
try:
policy = self.wrap_aws_rate_limited_call(
q.get_attributes,
attributes='Policy'
)
if 'Policy' in policy:
try:
json_str = policy['Policy']
policy = json.loads(json_str)
item = SQSItem(region=region.name, account=account, name=q.name,
config=policy)
item_list.append(item)
except:
self.slurp_exception((self.index, account, region, q.name), InvalidAWSJSON(json_str), exception_map)
except boto.exception.SQSError:
# A number of Queues are so ephemeral that they may be gone by the time
# the code reaches here. Just ignore them and move on.
pass
return item_list, exception_map
def get_all_certs_in_region(self, account, region, exception_map):
from security_monkey.common.sts_connect import connect
import traceback
all_certs = []
app.logger.debug("Checking {}/{}/{}".format(self.index, account,
region))
try:
iamconn = connect(account, 'iam', region=region)
marker = None
while True:
certs = self.wrap_aws_rate_limited_call(
iamconn.list_server_certs, marker=marker)
all_certs.extend(certs.server_certificate_metadata_list)
if certs.is_truncated == u'true':
marker = certs.marker
else:
break
for cert in all_certs:
try:
iam_cert = self.wrap_aws_rate_limited_call(
iamconn.get_server_certificate,
cert_name=cert.server_certificate_name)
cert['body'] = iam_cert.certificate_body
cert['chain'] = None
if hasattr(iam_cert, 'certificate_chain'):
cert['chain'] = iam_cert.certificate_chain
cert_info = get_cert_info(cert['body'])
for key in cert_info.iterkeys():
cert[key] = cert_info[key]
except Exception as e:
app.logger.warn(traceback.format_exc())
app.logger.error("Invalid certificate {}!".format(
cert.server_certificate_id))
self.slurp_exception(
(self.index, account, 'universal',
cert.server_certificate_name),
e,
exception_map,
source="{}-watcher".format(self.index))
except Exception as e:
app.logger.warn(traceback.format_exc())
if region not in TROUBLE_REGIONS:
exc = BotoConnectionIssue(str(e), self.index, account,
'universal')
self.slurp_exception((self.index, account, 'universal'),
exc,
exception_map,
source="{}-watcher".format(self.index))
app.logger.info("Found {} {} from {}/{}".format(
len(all_certs), self.i_am_plural, account, 'universal'))
return all_certs
def slurp(self):
"""
:returns: item_list - list of EC2 instances.
:returns: exception_map - A dict where the keys are a tuple containing the
location of the exception and the value is the actual exception
"""
self.prep_for_slurp()
item_list = []
exception_map = {}
from security_monkey.common.sts_connect import connect
for account in self.accounts:
for region in regions():
app.logger.debug("Checking {}/{}/{}".format(EC2.index, account, region.name))
try:
ec2 = connect(account, 'ec2', region=region)
all_instances = self.wrap_aws_rate_limited_call(
ec2.get_only_instances
)
except Exception as e:
if region.name not in TROUBLE_REGIONS:
exc = BotoConnectionIssue(str(e), 'ec2', account, region.name)
self.slurp_exception((self.index, account, region.name), exc, exception_map)
continue
app.logger.debug("Found {} {}".format(len(all_instances), EC2.i_am_plural))
for instance in all_instances:
# A list of properties, in order of priority, to try and use as the name
names = [instance.tags.get('Name'),
instance.private_dns_name,
instance.id]
for name in names:
if name:
break
if self.check_ignore_list(name):
continue
# Dont try to track pending, rebooting, terminated, etc, instances as they are missing attributes
if instance.state not in ['running', 'stopped']:
continue
groups = [{'id': group.id, 'name': group.name} for group in instance.groups]
instance_info = {'tags': dict(instance.tags),
'type': instance.instance_type,
'vpc_id': instance.vpc_id,
'subnet_id': instance.subnet_id,
'security_groups': groups,
'id': instance.id,
'dns_name': instance.private_dns_name}
item = EC2Item(region=region.name, account=account, name=name,
config=instance_info)
item_list.append(item)
return item_list, exception_map
def describe_images(self, **kwargs):
from security_monkey.common.sts_connect import connect
ec2 = connect(kwargs['account_name'],
'boto3.ec2.client',
region=kwargs['region'],
assumed_role=kwargs['assumed_role'])
response = self.wrap_aws_rate_limited_call(ec2.describe_images,
Owners=['self'])
images = response.get('Images')
return images
def describe_dhcp_options(self, **kwargs):
from security_monkey.common.sts_connect import connect
conn = connect(kwargs['account_name'],
'boto3.ec2.client',
region=kwargs['region'],
assumed_role=kwargs['assumed_role'])
dhcp_option_sets_resp = self.wrap_aws_rate_limited_call(
conn.describe_dhcp_options)
dhcp_option_sets = dhcp_option_sets_resp.get('DhcpOptions', [])
return dhcp_option_sets
def describe_network_acls(self, **kwargs):
from security_monkey.common.sts_connect import connect
conn = connect(kwargs['account_name'],
'boto3.ec2.client',
region=kwargs['region'],
assumed_role=kwargs['assumed_role'])
networkacls_resp = self.wrap_aws_rate_limited_call(
conn.describe_network_acls)
networkacls = networkacls_resp.get('NetworkAcls', [])
return networkacls
def describe_snapshots(self, **kwargs):
from security_monkey.common.sts_connect import connect
ec2 = connect(kwargs['account_name'], 'boto3.ec2.client', region=kwargs['region'],
assumed_role=kwargs['assumed_role'])
response = self.wrap_aws_rate_limited_call(
ec2.describe_snapshots,
OwnerIds=['self']
)
snapshots = response.get('Snapshots')
return [snapshot for snapshot in snapshots if not self.check_ignore_list(snapshot_name(snapshot))]
def slurp(self):
"""
:returns: item_list - list of Managed Policies.
:returns: exception_map - A dict where the keys are a tuple containing the
location of the exception and the value is the actual exception
"""
self.prep_for_slurp()
item_list = []
exception_map = {}
from security_monkey.common.sts_connect import connect
for account in self.accounts:
all_policies = []
try:
iam = connect(account, 'iam_boto3')
for policy in iam.policies.all():
all_policies.append(policy)
except Exception as e:
exc = BotoConnectionIssue(str(e), 'iamuser', account, None)
self.slurp_exception((self.index, account, 'universal'), exc, exception_map)
continue
for policy in all_policies:
if self.check_ignore_list(policy.policy_name):
continue
if self.check_ignore_list(policy.arn):
continue
item_config = {
'name': policy.policy_name,
'arn': policy.arn,
'create_date': str(policy.create_date),
'update_date': str(policy.update_date),
'default_version_id': policy.default_version_id,
'attachment_count': policy.attachment_count,
'attached_users': [a.arn for a in policy.attached_users.all()],
'attached_groups': [a.arn for a in policy.attached_groups.all()],
'attached_roles': [a.arn for a in policy.attached_roles.all()],
'policy': policy.default_version.document
}
app.logger.debug("Slurping %s (%s) from %s" % (self.i_am_singular, policy.policy_name, account))
item_list.append(
ManagedPolicyItem(account=account, name=policy.policy_name, config=item_config)
)
return item_list, exception_map
def slurp(self):
"""
:returns: item_list - list of Route53 records.
:returns: exception_map - A dict where the keys are a tuple containing the
location of the exception and the value is the actual exception
"""
self.prep_for_slurp()
item_list = []
exception_map = {}
from security_monkey.common.sts_connect import connect
for account in self.accounts:
app.logger.debug("Checking {}/{}".format(self.index, account))
try:
route53conn = connect(account, 'route53')
# Note : This fails if you have over 100 hosted zones
# maybe create paged_wrap_aws_rate_limited_call
route53_response = self.wrap_aws_rate_limited_call(
route53conn.get_all_hosted_zones
)
zones = route53_response['ListHostedZonesResponse']['HostedZones']
for zone in zones:
zone_id = zone['Id'][12:] # Trim leading '/hostedzone'
record_sets = self.wrap_aws_rate_limited_call(
route53conn.get_all_rrsets,
hosted_zone_id=zone_id
)
for record in record_sets:
if (record.type == 'CNAME'
and any([x.endswith(
tuple(self.third_party_domains))
for x in record.resource_records])):
# This CNAME contains a record which points to a
# third party domain
item = Route53Item(
account=account,
name=record.name,
config=record)
item_list.append(item)
except Exception as e:
exc = BotoConnectionIssue(str(e), self.index, account, None)
self.slurp_exception((self.index, account, 'universal'), exc, exception_map)
continue
app.logger.debug(
"Found {} {}".format(len(item_list), self.i_am_plural))
return item_list, exception_map
def describe_configuration_recorders(self, **kwargs):
from security_monkey.common.sts_connect import connect
config_service = connect(kwargs['account_name'],
'boto3.config.client',
region=kwargs['region'])
response = self.wrap_aws_rate_limited_call(
config_service.describe_configuration_recorders)
config_recorders = response.get('ConfigurationRecorders', [])
return [
recorder for recorder in config_recorders
if not self.check_ignore_list(recorder.get('name'))
]
def describe_volumes(self, **kwargs):
from security_monkey.common.sts_connect import connect
ec2 = connect(kwargs['account_name'],
'boto3.ec2.client',
region=kwargs['region'],
assumed_role=kwargs['assumed_role'])
response = self.wrap_aws_rate_limited_call(ec2.describe_volumes)
volumes = response.get('Volumes')
return [
volume for volume in volumes
if not self.check_ignore_list(volume_name(volume))
]
def list_functions(self, **kwargs):
from security_monkey.common.sts_connect import connect
app.logger.debug("Checking {}/{}/{}".format(self.index,
kwargs['account_name'], kwargs['region']))
lambda_client = connect(kwargs['account_name'], 'boto3.lambda.client',
region=kwargs['region'],
assumed_role=kwargs['assumed_role'])
response = self.wrap_aws_rate_limited_call(
lambda_client.list_functions
)
lambda_functions = response.get('Functions')
return lambda_functions
def describe_snapshots(self, **kwargs):
from security_monkey.common.sts_connect import connect
ec2 = connect(kwargs['account_name'],
'boto3.ec2.client',
region=kwargs['region'],
assumed_role=kwargs['assumed_role'])
response = self.wrap_aws_rate_limited_call(ec2.describe_snapshots,
OwnerIds=['self'])
snapshots = response.get('Snapshots')
return [
snapshot for snapshot in snapshots
if not self.check_ignore_list(snapshot_name(snapshot))
]
def list_functions(self, **kwargs):
from security_monkey.common.sts_connect import connect
app.logger.debug("Checking {}/{}/{}".format(self.index,
kwargs['account_name'],
kwargs['region']))
lambda_client = connect(kwargs['account_name'],
'boto3.lambda.client',
region=kwargs['region'],
assumed_role=kwargs['assumed_role'])
response = self.wrap_aws_rate_limited_call(
lambda_client.list_functions)
lambda_functions = response.get('Functions')
return lambda_functions
def get_all_certs_in_region(self, account, region, exception_map):
from security_monkey.common.sts_connect import connect
import traceback
all_certs = []
app.logger.debug("Checking {}/{}/{}".format(self.index, account, region))
try:
iamconn = connect(account, 'iam', region=region)
marker = None
while True:
certs = self.wrap_aws_rate_limited_call(
iamconn.list_server_certs,
marker=marker
)
all_certs.extend(certs.server_certificate_metadata_list)
if certs.is_truncated == u'true':
marker = certs.marker
else:
break
for cert in all_certs:
try:
iam_cert = self.wrap_aws_rate_limited_call(
iamconn.get_server_certificate,
cert_name=cert.server_certificate_name
)
cert['body'] = iam_cert.certificate_body
cert['chain'] = None
if hasattr(iam_cert, 'certificate_chain'):
cert['chain'] = iam_cert.certificate_chain
cert_info = get_cert_info(cert['body'])
for key in cert_info.iterkeys():
cert[key] = cert_info[key]
except Exception as e:
app.logger.warn(traceback.format_exc())
app.logger.error("Invalid certificate {}!".format(cert.server_certificate_id))
self.slurp_exception(
(self.index, account, 'universal', cert.server_certificate_name),
e, exception_map, source="{}-watcher".format(self.index))
except Exception as e:
app.logger.warn(traceback.format_exc())
if region not in TROUBLE_REGIONS:
exc = BotoConnectionIssue(str(e), self.index, account, 'universal')
self.slurp_exception((self.index, account, 'universal'), exc, exception_map,
source="{}-watcher".format(self.index))
app.logger.info("Found {} {} from {}/{}".format(len(all_certs), self.i_am_plural, account, 'universal'))
return all_certs
def slurp(self):
"""
:returns: item_list - list of virtual gateways
:returns: exception_map - A dict where the keys are a tuple containing the
location of the exception and the value is the actual exception
"""
self.prep_for_slurp()
from security_monkey.common.sts_connect import connect
item_list = []
exception_map = {}
for account in self.accounts:
for region in regions():
app.logger.debug(
"Checking {}/{}/{}".format(self.index, account, region.name))
try:
dc = connect(account, 'boto3.ec2.client', region=region)
response = self.wrap_aws_rate_limited_call(
dc.describe_vpn_gateways
)
gateways = response.get('VpnGateways')
except Exception as e:
if region.name not in TROUBLE_REGIONS:
exc = BotoConnectionIssue(
str(e), self.index, account, region.name)
self.slurp_exception(
(self.index, account, region.name), exc, exception_map)
continue
app.logger.debug("Found {} {}.".format(
len(gateways), self.i_am_plural))
for gateway in gateways:
name = gateway['VpnGatewayId']
if self.check_ignore_list(name):
continue
config = {
'name': name,
'state': gateway.get('State'),
'type': gateway.get('Type'),
'vpcAttachments': gateway.get('VpcAttachments'),
'virtual_gateway_state': gateway.get('VirtualGatewayState')
}
item = VirtualGatewayItem(
region=region.name, account=account, name=name, config=dict(config))
item_list.append(item)
return item_list, exception_map
def slurp(self):
"""
:returns: item_list - list of SES Identities.
:returns: exception_map - A dict where the keys are a tuple containing the
location of the exception and the value is the actual exception
"""
self.prep_for_slurp()
from security_monkey.common.sts_connect import connect
item_list = []
exception_map = {}
for account in self.accounts:
for region in regions():
if region.name == 'eu-central-1':
# as of boto 2.34.0, boto cannot connect to ses in eu-central-1
# TODO: Remove this if-block when boto can handle ses in eu-central-1
continue
app.logger.debug("Checking {}/{}/{}".format(self.index, account, region.name))
try:
ses = connect(account, 'ses', region=region.name)
response = self.wrap_aws_rate_limited_call(
ses.list_identities
)
identities = response.Identities
response = self.wrap_aws_rate_limited_call(
ses.list_verified_email_addresses
)
verified_identities = response.VerifiedEmailAddresses
except Exception as e:
if region.name not in TROUBLE_REGIONS:
exc = BotoConnectionIssue(str(e), self.index, account, region.name)
self.slurp_exception((self.index, account, region.name), exc, exception_map)
continue
app.logger.debug("Found {} {}. {} are verified.".format(len(identities), self.i_am_plural, len(verified_identities)))
for identity in identities:
if self.check_ignore_list(identity):
continue
config = {
'name': identity,
'verified': identity in verified_identities
}
item = SESItem(region=region.name, account=account, name=identity, config=dict(config))
item_list.append(item)
return item_list, exception_map
def get_all_topics_in_region(self, account, region):
from security_monkey.common.sts_connect import connect
sns = connect(account, 'sns', region=region)
app.logger.debug("Checking {}/{}/{}".format(SNS.index, account, region.name))
topics = []
marker = None
while True:
topics_response = sns.get_all_topics(next_token=marker)
current_page_topics = topics_response['ListTopicsResponse']['ListTopicsResult']['Topics']
topics.extend(current_page_topics)
if topics_response[u'ListTopicsResponse'][u'ListTopicsResult'][u'NextToken']:
marker = topics_response[u'ListTopicsResponse'][u'ListTopicsResult'][u'NextToken']
else:
break
return (sns, topics)
def s3connect(account, bucket):
""" s3connect will attempt to connect to an s3 bucket resource.
If the resource does not exist it will attempt to create it
:param account: string the aws account you are connecting to
:param bucket: string the name of the bucket you wish to connect to
:returns: Boolean of connection Status
"""
conn = connect(account, 's3')
if conn.lookup(bucket) is None:
bucket = conn.create_bucket(bucket)
else:
bucket = conn.get_bucket(bucket)
key = Key(bucket)
return conn, bucket, key
def s3connect(self, account, bucket):
""" s3connect will attempt to connect to an s3 bucket resource.
If the resource does not exist it will attempt to create it
:param account: string the aws account you are connecting to
:param bucket: string the name of the bucket you wish to connect to
:returns: Boolean of connection Status
"""
self.conn = connect(account, 's3')
if self.conn.lookup(bucket) is None:
app.logger.debug("Bucket Does not exist. Creating one")
self.bucket = self.conn.create_bucket(bucket)
else:
self.bucket = self.conn.get_bucket(bucket)
self.key = Key(self.bucket)
return True
def get_all_topics_in_region(self, account, region):
from security_monkey.common.sts_connect import connect
sns = connect(account, 'sns', region=region)
app.logger.debug("Checking {}/{}/{}".format(SNS.index, account, region.name))
topics = []
marker = None
while True:
topics_response = self.wrap_aws_rate_limited_call(
sns.get_all_topics,
next_token=marker
)
current_page_topics = topics_response['ListTopicsResponse']['ListTopicsResult']['Topics']
topics.extend(current_page_topics)
if topics_response[u'ListTopicsResponse'][u'ListTopicsResult'][u'NextToken']:
marker = topics_response[u'ListTopicsResponse'][u'ListTopicsResult'][u'NextToken']
else:
break
return sns, topics
def get_all_dbsecurity_groups(self, **kwargs):
from security_monkey.common.sts_connect import connect
sgs = []
rds = connect(kwargs['account_name'],
'rds',
region=kwargs['region'],
assumed_role=kwargs['assumed_role'])
marker = None
while True:
response = self.wrap_aws_rate_limited_call(
rds.get_all_dbsecurity_groups, marker=marker)
sgs.extend(response)
if response.marker:
marker = response.marker
else:
break
return sgs