def slurp(self): """ :returns: item_list - list of IAM SSH Keypairs. :returns: exception_map - A dict where the keys are a tuple containing the location of the exception and the value is the actual exception """ item_list = [] exception_map = {} from security_monkey.common.sts_connect import connect for account in self.accounts: try: ec2 = connect(account, 'ec2') regions = ec2.get_all_regions() except Exception as e: # EC2ResponseError # Some Accounts don't subscribe to EC2 and will throw an exception here. exc = BotoConnectionIssue(str(e), 'keypair', account, None) self.slurp_exception((self.index, account), exc, exception_map) continue for region in regions: app.logger.debug("Checking {}/{}/{}".format( Keypair.index, account, region.name)) try: rec2 = connect(account, 'ec2', region=region) kps = self.wrap_aws_rate_limited_call( rec2.get_all_key_pairs) except Exception as e: if region.name not in TROUBLE_REGIONS: exc = BotoConnectionIssue(str(e), 'keypair', account, region.name) self.slurp_exception( (self.index, account, region.name), exc, exception_map) continue app.logger.debug("Found {} {}".format(len(kps), Keypair.i_am_plural)) for kp in kps: ### Check if this Keypair is on the Ignore List ### ignore_item = False for ignore_item_name in IGNORE_PREFIX[self.index]: if kp.name.lower().startswith( ignore_item_name.lower()): ignore_item = True break if ignore_item: continue item_list.append( KeypairItem(region=region.name, account=account, name=kp.name, config={'fingerprint': kp.fingerprint})) return item_list, exception_map
def slurp(self): """ :returns: item_list - list of IAM SSH Keypairs. :returns: exception_map - A dict where the keys are a tuple containing the location of the exception and the value is the actual exception """ self.prep_for_slurp() item_list = [] exception_map = {} from security_monkey.common.sts_connect import connect for account in self.accounts: try: account_db = Account.query.filter(Account.name == account).first() account_number = account_db.number ec2 = connect(account, 'ec2') regions = ec2.get_all_regions() except Exception as e: # EC2ResponseError # Some Accounts don't subscribe to EC2 and will throw an exception here. exc = BotoConnectionIssue(str(e), 'keypair', account, None) self.slurp_exception((self.index, account), exc, exception_map, source="{}-watcher".format(self.index)) continue for region in regions: app.logger.debug("Checking {}/{}/{}".format(Keypair.index, account, region.name)) try: rec2 = connect(account, 'ec2', region=region) kps = self.wrap_aws_rate_limited_call( rec2.get_all_key_pairs ) except Exception as e: if region.name not in TROUBLE_REGIONS: exc = BotoConnectionIssue(str(e), 'keypair', account, region.name) self.slurp_exception((self.index, account, region.name), exc, exception_map, source="{}-watcher".format(self.index)) continue app.logger.debug("Found {} {}".format(len(kps), Keypair.i_am_plural)) for kp in kps: if self.check_ignore_list(kp.name): continue arn = 'arn:aws:ec2:{region}:{account_number}:key-pair/{name}'.format( region=region.name, account_number=account_number, name=kp.name) item_list.append(KeypairItem(region=region.name, account=account, name=kp.name, arn=arn, config={ 'fingerprint': kp.fingerprint, 'arn': arn, 'name': kp.name })) return item_list, exception_map
def slurp(self): """ :returns: item_list - list of IAM SSH Keypairs. :returns: exception_map - A dict where the keys are a tuple containing the location of the exception and the value is the actual exception """ self.prep_for_slurp() item_list = [] exception_map = {} from security_monkey.common.sts_connect import connect for account in self.accounts: try: account_db = Account.query.filter(Account.name == account).first() account_number = account_db.identifier ec2 = connect(account, 'ec2') regions = ec2.get_all_regions() except Exception as e: # EC2ResponseError # Some Accounts don't subscribe to EC2 and will throw an exception here. exc = BotoConnectionIssue(str(e), 'keypair', account, None) self.slurp_exception((self.index, account), exc, exception_map, source="{}-watcher".format(self.index)) continue for region in regions: app.logger.debug("Checking {}/{}/{}".format(Keypair.index, account, region.name)) try: rec2 = connect(account, 'boto3.ec2.client', region=region) kps = self.wrap_aws_rate_limited_call( rec2.describe_key_pairs ) except Exception as e: if region.name not in TROUBLE_REGIONS: exc = BotoConnectionIssue(str(e), 'keypair', account, region.name) self.slurp_exception((self.index, account, region.name), exc, exception_map, source="{}-watcher".format(self.index)) continue app.logger.debug("Found {} {}".format(len(kps), Keypair.i_am_plural)) for kp in kps['KeyPairs']: if self.check_ignore_list(kp['KeyName']): continue arn = ARN_PREFIX + ':ec2:{region}:{account_number}:key-pair/{name}'.format( region=region.name, account_number=account_number, name=kp["KeyName"]) item_list.append(KeypairItem(region=region.name, account=account, name=kp["KeyName"], arn=arn, config={ 'fingerprint': kp["KeyFingerprint"], 'arn': arn, 'name': kp["KeyName"] }, source_watcher=self)) return item_list, exception_map
def slurp(self): """ :returns: item_list - list of S3 Buckets. :returns: exception_map - A dict where the keys are a tuple containing the location of the exception and the value is the actual exception """ item_list = [] exception_map = {} from security_monkey.common.sts_connect import connect for account in self.accounts: try: s3conn = connect(account, 's3', calling_format=OrdinaryCallingFormat()) all_buckets = s3conn.get_all_buckets() except Exception as e: exc = BotoConnectionIssue(str(e), 's3', account, None) self.slurp_exception((self.index, account), exc, exception_map) continue for bucket in all_buckets: app.logger.debug("Slurping %s (%s) from %s" % (self.i_am_singular, bucket.name, account)) ### Check if this bucket is on the Ignore List ### ignore_item = False for ignore_item_name in IGNORE_PREFIX[self.index]: if bucket.name.lower().startswith(ignore_item_name.lower()): ignore_item = True break if ignore_item: continue try: region = self.translate_location_to_region(bucket.get_location()) if region == '': s3regionconn = connect(account, 's3', calling_format=OrdinaryCallingFormat()) region = 'us-east-1' else: s3regionconn = connect(account, 's3', region=region, calling_format=OrdinaryCallingFormat()) bhandle = s3regionconn.get_bucket(bucket) s3regionconn.close() except Exception as e: exc = S3PermissionsIssue(bucket.name) # Unfortunately, we can't get the region, so the entire account # will be skipped in find_changes, not just the bad bucket. self.slurp_exception((self.index, account), exc, exception_map) continue app.logger.debug("Slurping %s (%s) from %s/%s" % (self.i_am_singular, bucket.name, account, region)) bucket_dict = self.conv_bucket_to_dict(bhandle, account, region, bucket.name, exception_map) item = S3Item(account=account, region=region, name=bucket.name, config=bucket_dict) item_list.append(item) return item_list, exception_map
def slurp(self): """ :returns: item_list - list of IAM SSH Keypairs. :returns: exception_map - A dict where the keys are a tuple containing the location of the exception and the value is the actual exception """ item_list = [] exception_map = {} from security_monkey.common.sts_connect import connect for account in self.accounts: try: ec2 = connect(account, 'ec2') regions = ec2.get_all_regions() except Exception as e: # EC2ResponseError # Some Accounts don't subscribe to EC2 and will throw an exception here. exc = BotoConnectionIssue(str(e), 'keypair', account, None) self.slurp_exception((self.index, account), exc, exception_map) continue for region in regions: app.logger.debug("Checking {}/{}/{}".format(Keypair.index, account, region.name)) try: rec2 = connect(account, 'ec2', region=region) kps = self.wrap_aws_rate_limited_call( rec2.get_all_key_pairs ) except Exception as e: if region.name not in TROUBLE_REGIONS: exc = BotoConnectionIssue(str(e), 'keypair', account, region.name) self.slurp_exception((self.index, account, region.name), exc, exception_map) continue app.logger.debug("Found {} {}".format(len(kps), Keypair.i_am_plural)) for kp in kps: ### Check if this Keypair is on the Ignore List ### ignore_item = False for ignore_item_name in IGNORE_PREFIX[self.index]: if kp.name.lower().startswith(ignore_item_name.lower()): ignore_item = True break if ignore_item: continue item_list.append(KeypairItem(region=region.name, account=account, name=kp.name, config={ 'fingerprint': kp.fingerprint })) return item_list, exception_map
def get_all_subnets(self, **kwargs): from security_monkey.common.sts_connect import connect conn = connect(kwargs['account_name'], 'boto3.ec2.client', region=kwargs['region'], assumed_role=kwargs['assumed_role']) all_subnets = self.wrap_aws_rate_limited_call(conn.describe_subnets) return all_subnets.get('Subnets')
def slurp(self): """ :returns: item_list - list of Redshift Policies. :returns: exception_map - A dict where the keys are a tuple containing the location of the exception and the value is the actual exception """ self.prep_for_slurp() from security_monkey.common.sts_connect import connect item_list = [] exception_map = {} for account in self.accounts: for region in regions(): app.logger.debug("Checking {}/{}/{}".format(self.index, account, region.name)) try: redshift = connect(account, 'redshift', region=region) response = self.wrap_aws_rate_limited_call( redshift.describe_clusters ) all_clusters = response['DescribeClustersResponse']['DescribeClustersResult']['Clusters'] except Exception as e: if region.name not in TROUBLE_REGIONS: exc = BotoConnectionIssue(str(e), 'redshift', account, region.name) self.slurp_exception((self.index, account, region.name), exc, exception_map) continue app.logger.debug("Found {} {}".format(len(all_clusters), Redshift.i_am_plural)) for cluster in all_clusters: cluster_id = cluster['ClusterIdentifier'] if self.check_ignore_list(cluster_id): continue item = RedshiftCluster(region=region.name, account=account, name=cluster_id, config=dict(cluster)) item_list.append(item) return item_list, exception_map
def get_all_certs_in_region(self, account, region, exception_map): from security_monkey.common.sts_connect import connect import traceback all_certs = [] app.logger.debug("Checking {}/{}/{}".format(self.index, account, region)) try: iamconn = connect(account, 'iam', region=region) marker = None while True: certs = self.wrap_aws_rate_limited_call( iamconn.list_server_certs, marker=marker ) all_certs.extend(certs.server_certificate_metadata_list) if certs.is_truncated == u'true': marker = certs.marker else: break except Exception as e: app.logger.warn(traceback.format_exc()) if region not in TROUBLE_REGIONS: exc = BotoConnectionIssue(str(e), self.index, account, region) self.slurp_exception((self.index, account, region), exc, exception_map) app.logger.info("Found {} {} from {}/{}".format(len(all_certs), self.i_am_plural, account, region)) return all_certs
def get_all_certs_in_region(self, account, region, exception_map): from security_monkey.common.sts_connect import connect import traceback all_certs = [] app.logger.debug("Checking {}/{}/{}".format(self.index, account, region)) try: iamconn = connect(account, 'iam', region=region) marker = None while True: certs = self.wrap_aws_rate_limited_call( iamconn.list_server_certs, marker=marker) all_certs.extend(certs.server_certificate_metadata_list) if certs.is_truncated == u'true': marker = certs.marker else: break except Exception as e: app.logger.warn(traceback.format_exc()) if region not in TROUBLE_REGIONS: exc = BotoConnectionIssue(str(e), self.index, account, region) self.slurp_exception((self.index, account, region), exc, exception_map) app.logger.info("Found {} {} from {}/{}".format( len(all_certs), self.i_am_plural, account, region)) return all_certs
def describe_db_subnet_groups(self, **kwargs): from security_monkey.common.sts_connect import connect db_sub_groups = [] rds = connect(kwargs['account_name'], 'boto3.rds.client', region=kwargs['region'], assumed_role=kwargs['assumed_role']) marker = None while True: if marker: response = self.wrap_aws_rate_limited_call( rds.describe_db_subnet_groups, Marker=marker) else: response = self.wrap_aws_rate_limited_call( rds.describe_db_subnet_groups) db_sub_groups.extend(response.get('DBSubnetGroups')) if response.get('Marker'): marker = response.get('Marker') else: break return db_sub_groups
def slurp(self): """ :returns: item_dict - list of IAM Account attributes. :returns: exception_map - A dict where the keys are a tuple containing the location of the exception and the value is the actual exception """ self.prep_for_slurp() item_list = [] exception_map = {} from security_monkey.common.sts_connect import connect for account in self.accounts: try: iam = connect(account, 'iam') account_summary = self.wrap_aws_rate_limited_call( iam.get_account_summary) except Exception as e: exc = BotoConnectionIssue(str(e), 'iamaccount', account, None) self.slurp_exception((self.index, account, 'universal'), exc, exception_map) continue item_list.append( IAMAccountItem(account=account, name=account, config=dict(account_summary))) return item_list, exception_map
def get_all_subnets(self, **kwargs): from security_monkey.common.sts_connect import connect conn = connect(kwargs['account_name'], 'vpc', region=kwargs['region'], assumed_role=kwargs['assumed_role']) all_subnets = self.wrap_aws_rate_limited_call(conn.get_all_subnets) return all_subnets
def slurp(self): """ :returns: item_list - list of connections :returns: exception_map - A dict where the keys are a tuple containing the location of the exception and the value is the actual exception """ self.prep_for_slurp() from security_monkey.common.sts_connect import connect item_list = [] exception_map = {} for account in self.accounts: for region in regions(): app.logger.debug("Checking {}/{}/{}".format( self.index, account, region.name)) try: dc = connect(account, 'boto3.directconnect.client', region=region) response = self.wrap_aws_rate_limited_call( dc.describe_connections) connections = response.get('connections') except Exception as e: if region.name not in TROUBLE_REGIONS: exc = BotoConnectionIssue(str(e), self.index, account, region.name) self.slurp_exception( (self.index, account, region.name), exc, exception_map) continue app.logger.debug("Found {} {}.".format(len(connections), self.i_am_plural)) for connection in connections: name = connection['connectionName'] if self.check_ignore_list(name): continue config = { 'name': name, 'owner_account': connection.get('ownerAccount'), 'connection_id': connection.get('connectionId'), 'connection_name': connection.get('connectionName'), 'connection_state': connection.get('connectionState'), 'region': connection.get('region'), 'location': connection.get('location'), 'bandwidth': connection.get('bandwidth'), 'vlan': connection.get('vlan'), 'partner_name': connection.get('partnerName'), } item = ConnectionItem(region=region.name, account=account, name=name, config=dict(config), source_watcher=self) item_list.append(item) return item_list, exception_map
def slurp(self): """ :returns: item_list - list of endpoints. :returns: exception_map - A dict where the keys are a tuple containing the location of the exception and the value is the actual exception """ self.prep_for_slurp() item_list = [] exception_map = {} from security_monkey.common.sts_connect import connect for account in self.accounts: for region in regions(): app.logger.debug( "Checking {}/{}/{}".format(self.index, account, region.name)) try: conn = connect(account, 'boto3.ec2.client', region=region) all_vpc_endpoints_resp = self.wrap_aws_rate_limited_call( conn.describe_vpc_endpoints ) all_vpc_endpoints = all_vpc_endpoints_resp.get( 'VpcEndpoints', []) except Exception as e: if region.name not in TROUBLE_REGIONS: exc = BotoConnectionIssue( str(e), self.index, account, region.name) self.slurp_exception( (self.index, account, region.name), exc, exception_map) continue app.logger.debug("Found {} {}".format( len(all_vpc_endpoints), self.i_am_plural)) for endpoint in all_vpc_endpoints: endpoint_name = endpoint.get(u'VpcEndpointId') if self.check_ignore_list(endpoint_name): continue service = endpoint.get('ServiceName', '').split('.')[-1] config = { "id": endpoint.get('VpcEndpointId'), "policy_document": endpoint.get('PolicyDocument', {}), "service_name": endpoint.get('ServiceName'), "service": service, "route_table_ids": endpoint.get('RouteTableIds', []), "creation_time_stamp": str(endpoint.get('CreationTimestamp')), "state": endpoint.get('State'), "vpc_id": endpoint.get('VpcId'), } item = EndpointItem( region=region.name, account=account, name=endpoint_name, config=config, source_watcher=self) item_list.append(item) return item_list, exception_map
def slurp(self): """ :returns: item_list - list of configs. :returns: exception_map - A dict where the keys are a tuple containing the location of the exception and the value is the actual exception """ self.prep_for_slurp() item_list = [] exception_map = {} from security_monkey.common.sts_connect import connect for account in self.accounts: for region in regions(): app.logger.debug( "Checking {}/{}/{}".format(self.index, account, region.name)) if region.name not in AVAILABLE_REGIONS: continue try: configService = connect( account, 'boto3.config.client', region=region) response = self.wrap_aws_rate_limited_call( configService.describe_config_rules ) config_rules = response.get('ConfigRules', []) except Exception as e: app.logger.debug("Exception found: {}".format(e)) if region.name not in TROUBLE_REGIONS: exc = BotoConnectionIssue( str(e), self.index, account, region.name) self.slurp_exception( (self.index, account, region.name), exc, exception_map) continue app.logger.debug("Found {} {}.".format( len(config_rules), self.i_am_plural)) for config_rule in config_rules: name = config_rule.get('ConfigRuleName') if self.check_ignore_list(name): continue item_config = { 'config_rule': name, 'config_rule_arn': config_rule.get('ConfigRuleArn'), 'config_rule_id': config_rule.get('ConfigRuleId'), 'scope': config_rule.get('Scope', {}), 'source': config_rule.get('Source', {}), 'imput_parameters': config_rule.get('InputParameters'), 'maximum_execution_frequency': config_rule.get('MaximumExecutionFrequency'), 'config_rule_state': config_rule.get('ConfigRuleState'), } item = ConfigItem( region=region.name, account=account, name=name, arn=config_rule.get('ConfigRuleArn'), config=item_config) item_list.append(item) return item_list, exception_map
def describe_vpc_peering_connections(self, **kwargs): from security_monkey.common.sts_connect import connect conn = connect(kwargs['account_name'], 'boto3.ec2.client', region=kwargs['region'], assumed_role=kwargs['assumed_role']) peering_info = self.wrap_aws_rate_limited_call( conn.describe_vpc_peering_connections) return peering_info
def get_all_es_domains_in_region(self, account, region): from security_monkey.common.sts_connect import connect client = connect(account, "boto3.es.client", region=region) app.logger.debug("Checking {}/{}/{}".format(ElasticSearchService.index, account, region.name)) # No need to paginate according to: client.can_paginate("list_domain_names") domains = self.wrap_aws_rate_limited_call(client.list_domain_names)["DomainNames"] return client, domains
def describe_route_tables(self, **kwargs): from security_monkey.common.sts_connect import connect conn = connect(kwargs['account_name'], 'boto3.ec2.client', region=kwargs['region'], assumed_role=kwargs['assumed_role']) response = self.wrap_aws_rate_limited_call(conn.describe_route_tables) all_route_tables = response.get('RouteTables', []) return all_route_tables
def describe_db_instances(self, **kwargs): from security_monkey.common.sts_connect import connect rds = connect(kwargs['account_name'], 'boto3.rds.client', region=kwargs['region'], assumed_role=kwargs['assumed_role']) response = self.wrap_aws_rate_limited_call(rds.describe_db_instances) rds_db_instances = response.get('DBInstances') return rds_db_instances
def describe_configuration_recorders(self, **kwargs): from security_monkey.common.sts_connect import connect config_service = connect(kwargs["account_name"], "boto3.config.client", region=kwargs["region"]) response = self.wrap_aws_rate_limited_call(config_service.describe_configuration_recorders) config_recorders = response.get("ConfigurationRecorders", []) return [recorder for recorder in config_recorders if not self.check_ignore_list(recorder.get("name"))]
def slurp(self): """ :returns: item_list - list of subnets. :returns: exception_map - A dict where the keys are a tuple containing the location of the exception and the value is the actual exception """ self.prep_for_slurp() item_list = [] exception_map = {} from security_monkey.common.sts_connect import connect for account in self.accounts: for region in regions(): app.logger.debug("Checking {}/{}/{}".format(self.index, account, region.name)) try: conn = connect(account, 'vpc', region=region) all_subnets = self.wrap_aws_rate_limited_call( conn.get_all_subnets ) except Exception as e: if region.name not in TROUBLE_REGIONS: exc = BotoConnectionIssue(str(e), self.index, account, region.name) self.slurp_exception((self.index, account, region.name), exc, exception_map) continue app.logger.debug("Found {} {}".format(len(all_subnets), self.i_am_plural)) for subnet in all_subnets: subnet_name = subnet.tags.get(u'Name', None) if subnet_name: subnet_name = "{0} ({1})".format(subnet_name, subnet.id) else: subnet_name = subnet.id if self.check_ignore_list(subnet_name): continue config = { "name": subnet.tags.get(u'Name', None), "id": subnet.id, "cidr_block": subnet.cidr_block, "availability_zone": subnet.availability_zone, # TODO: # available_ip_address_count is likely to change often # and should be in the upcoming ephemeral section. # "available_ip_address_count": subnet.available_ip_address_count, "defaultForAz": subnet.defaultForAz, "mapPublicIpOnLaunch": subnet.mapPublicIpOnLaunch, "state": subnet.state, "tags": dict(subnet.tags), "vpc_id": subnet.vpc_id } item = SubnetItem(region=region.name, account=account, name=subnet_name, config=config) item_list.append(item) return item_list, exception_map
def describe_network_acls(self, **kwargs): from security_monkey.common.sts_connect import connect conn = connect(kwargs['account_name'], 'boto3.ec2.client', region=kwargs['region'], assumed_role=kwargs['assumed_role']) networkacls_resp = self.wrap_aws_rate_limited_call( conn.describe_network_acls) networkacls = networkacls_resp.get('NetworkAcls', []) return networkacls
def describe_dhcp_options(self, **kwargs): from security_monkey.common.sts_connect import connect conn = connect(kwargs['account_name'], 'boto3.ec2.client', region=kwargs['region'], assumed_role=kwargs['assumed_role']) dhcp_option_sets_resp = self.wrap_aws_rate_limited_call( conn.describe_dhcp_options) dhcp_option_sets = dhcp_option_sets_resp.get('DhcpOptions', []) return dhcp_option_sets
def slurp(self): """ :returns: item_list - list of IAM Groups. :returns: exception_map - A dict where the keys are a tuple containing the location of the exception and the value is the actual exception """ self.prep_for_slurp() item_list = [] exception_map = {} from security_monkey.common.sts_connect import connect for account in self.accounts: try: iam = connect(account, 'iam') groups = self.get_all_groups(iam) except Exception as e: exc = BotoConnectionIssue(str(e), 'iamgroup', account, None) self.slurp_exception((self.index, account, 'universal'), exc, exception_map) continue for group in groups: app.logger.debug("Slurping %s (%s) from %s" % (self.i_am_singular, group.group_name, account)) if self.check_ignore_list(group.group_name): continue item_config = { 'group': dict(group), 'grouppolicies': {}, 'users': {} } ### GROUP POLICIES ### group_policies = self.get_all_group_policies(iam, group.group_name) for policy_name in group_policies: policy = self.wrap_aws_rate_limited_call(iam.get_group_policy, group.group_name, policy_name) policy = policy.policy_document policy = urllib.unquote(policy) try: policydict = json.loads(policy) except: exc = InvalidAWSJSON(policy) self.slurp_exception((self.index, account, 'universal', group.group_name), exc, exception_map) item_config['grouppolicies'][policy_name] = dict(policydict) ### GROUP USERS ### group_users = self.get_all_group_users(iam, group['group_name']) for user in group_users: item_config['users'][user.arn] = user.user_name item = IAMGroupItem(account=account, name=group.group_name, config=item_config) item_list.append(item) return item_list, exception_map
def describe_images(self, **kwargs): from security_monkey.common.sts_connect import connect ec2 = connect(kwargs['account_name'], 'boto3.ec2.client', region=kwargs['region'], assumed_role=kwargs['assumed_role']) response = self.wrap_aws_rate_limited_call(ec2.describe_images, Owners=['self']) images = response.get('Images') return images
def slurp(self): """ :returns: item_list - list of Redshift Policies. :returns: exception_map - A dict where the keys are a tuple containing the location of the exception and the value is the actual exception """ self.prep_for_slurp() from security_monkey.common.sts_connect import connect item_list = [] exception_map = {} for account in self.accounts: account_db = Account.query.filter(Account.name == account).first() account_number = account_db.identifier for region in regions(): app.logger.debug("Checking {}/{}/{}".format(self.index, account, region.name)) try: redshift = connect(account, 'redshift', region=region) all_clusters = [] marker = None while True: response = self.wrap_aws_rate_limited_call( redshift.describe_clusters, marker=marker ) all_clusters.extend(response['DescribeClustersResponse']['DescribeClustersResult']['Clusters']) if response['DescribeClustersResponse']['DescribeClustersResult']['Marker'] is not None: marker = response['DescribeClustersResponse']['DescribeClustersResult']['Marker'] else: break except Exception as e: if region.name not in TROUBLE_REGIONS: exc = BotoConnectionIssue(str(e), 'redshift', account, region.name) self.slurp_exception((self.index, account, region.name), exc, exception_map, source="{}-watcher".format(self.index)) continue app.logger.debug("Found {} {}".format(len(all_clusters), Redshift.i_am_plural)) for cluster in all_clusters: cluster_id = cluster['ClusterIdentifier'] if self.check_ignore_list(cluster_id): continue arn = ARN_PREFIX + ':redshift:{region}:{account_number}:cluster:{name}'.format( region=region.name, account_number=account_number, name=cluster_id) cluster['arn'] = arn item = RedshiftCluster(region=region.name, account=account, name=cluster_id, arn=arn, config=dict(cluster), source_watcher=self) item_list.append(item) return item_list, exception_map
def describe_instances(self, **kwargs): from security_monkey.common.sts_connect import connect ec2 = connect(kwargs['account_name'], 'boto3.ec2.client', region=kwargs['region'], assumed_role=kwargs['assumed_role']) response = self.wrap_aws_rate_limited_call(ec2.describe_instances) reservations = response.get('Reservations') return reservations
def describe_instances(self, **kwargs): from security_monkey.common.sts_connect import connect ec2 = connect(kwargs['account_name'], 'boto3.ec2.client', region=kwargs['region'], assumed_role=kwargs['assumed_role']) response = self.wrap_aws_rate_limited_call( ec2.describe_instances ) reservations = response.get('Reservations') return reservations
def describe_volumes(self, **kwargs): from security_monkey.common.sts_connect import connect ec2 = connect( kwargs["account_name"], "boto3.ec2.client", region=kwargs["region"], assumed_role=kwargs["assumed_role"] ) response = self.wrap_aws_rate_limited_call(ec2.describe_volumes) volumes = response.get("Volumes") return [volume for volume in volumes if not self.check_ignore_list(volume_name(volume))]
def slurp(self): """ :returns: item_list - list of SQS Policies. :returns: exception_map - A dict where the keys are a tuple containing the location of the exception and the value is the actual exception """ item_list = [] exception_map = {} from security_monkey.common.sts_connect import connect for account in self.accounts: for region in regions(): app.logger.debug("Checking {}/{}/{}".format(SQS.index, account, region.name)) try: sqs = connect(account, 'sqs', region=region) all_queues = self.wrap_aws_rate_limited_call( sqs.get_all_queues ) except Exception as e: if region.name not in TROUBLE_REGIONS: exc = BotoConnectionIssue(str(e), 'sqs', account, region.name) self.slurp_exception((self.index, account, region.name), exc, exception_map) continue app.logger.debug("Found {} {}".format(len(all_queues), SQS.i_am_plural)) for q in all_queues: ### Check if this Queue is on the Ignore List ### ignore_item = False for ignore_item_name in IGNORE_PREFIX[self.index]: if q.name.lower().startswith(ignore_item_name.lower()): ignore_item = True break if ignore_item: continue try: policy = self.wrap_aws_rate_limited_call( q.get_attributes, attributes='Policy' ) if 'Policy' in policy: try: json_str = policy['Policy'] policy = json.loads(json_str) item = SQSItem(region=region.name, account=account, name=q.name, config=policy) item_list.append(item) except: self.slurp_exception((self.index, account, region, q.name), InvalidAWSJSON(json_str), exception_map) except boto.exception.SQSError: # A number of Queues are so ephemeral that they may be gone by the time # the code reaches here. Just ignore them and move on. pass return item_list, exception_map
def get_all_certs_in_region(self, account, region, exception_map): from security_monkey.common.sts_connect import connect import traceback all_certs = [] app.logger.debug("Checking {}/{}/{}".format(self.index, account, region)) try: iamconn = connect(account, 'iam', region=region) marker = None while True: certs = self.wrap_aws_rate_limited_call( iamconn.list_server_certs, marker=marker) all_certs.extend(certs.server_certificate_metadata_list) if certs.is_truncated == u'true': marker = certs.marker else: break for cert in all_certs: try: iam_cert = self.wrap_aws_rate_limited_call( iamconn.get_server_certificate, cert_name=cert.server_certificate_name) cert['body'] = iam_cert.certificate_body cert['chain'] = None if hasattr(iam_cert, 'certificate_chain'): cert['chain'] = iam_cert.certificate_chain cert_info = get_cert_info(cert['body']) for key in cert_info.iterkeys(): cert[key] = cert_info[key] except Exception as e: app.logger.warn(traceback.format_exc()) app.logger.error("Invalid certificate {}!".format( cert.server_certificate_id)) self.slurp_exception( (self.index, account, 'universal', cert.server_certificate_name), e, exception_map, source="{}-watcher".format(self.index)) except Exception as e: app.logger.warn(traceback.format_exc()) if region not in TROUBLE_REGIONS: exc = BotoConnectionIssue(str(e), self.index, account, 'universal') self.slurp_exception((self.index, account, 'universal'), exc, exception_map, source="{}-watcher".format(self.index)) app.logger.info("Found {} {} from {}/{}".format( len(all_certs), self.i_am_plural, account, 'universal')) return all_certs
def slurp(self): """ :returns: item_list - list of EC2 instances. :returns: exception_map - A dict where the keys are a tuple containing the location of the exception and the value is the actual exception """ self.prep_for_slurp() item_list = [] exception_map = {} from security_monkey.common.sts_connect import connect for account in self.accounts: for region in regions(): app.logger.debug("Checking {}/{}/{}".format(EC2.index, account, region.name)) try: ec2 = connect(account, 'ec2', region=region) all_instances = self.wrap_aws_rate_limited_call( ec2.get_only_instances ) except Exception as e: if region.name not in TROUBLE_REGIONS: exc = BotoConnectionIssue(str(e), 'ec2', account, region.name) self.slurp_exception((self.index, account, region.name), exc, exception_map) continue app.logger.debug("Found {} {}".format(len(all_instances), EC2.i_am_plural)) for instance in all_instances: # A list of properties, in order of priority, to try and use as the name names = [instance.tags.get('Name'), instance.private_dns_name, instance.id] for name in names: if name: break if self.check_ignore_list(name): continue # Dont try to track pending, rebooting, terminated, etc, instances as they are missing attributes if instance.state not in ['running', 'stopped']: continue groups = [{'id': group.id, 'name': group.name} for group in instance.groups] instance_info = {'tags': dict(instance.tags), 'type': instance.instance_type, 'vpc_id': instance.vpc_id, 'subnet_id': instance.subnet_id, 'security_groups': groups, 'id': instance.id, 'dns_name': instance.private_dns_name} item = EC2Item(region=region.name, account=account, name=name, config=instance_info) item_list.append(item) return item_list, exception_map
def describe_snapshots(self, **kwargs): from security_monkey.common.sts_connect import connect ec2 = connect(kwargs['account_name'], 'boto3.ec2.client', region=kwargs['region'], assumed_role=kwargs['assumed_role']) response = self.wrap_aws_rate_limited_call( ec2.describe_snapshots, OwnerIds=['self'] ) snapshots = response.get('Snapshots') return [snapshot for snapshot in snapshots if not self.check_ignore_list(snapshot_name(snapshot))]
def slurp(self): """ :returns: item_list - list of Managed Policies. :returns: exception_map - A dict where the keys are a tuple containing the location of the exception and the value is the actual exception """ self.prep_for_slurp() item_list = [] exception_map = {} from security_monkey.common.sts_connect import connect for account in self.accounts: all_policies = [] try: iam = connect(account, 'iam_boto3') for policy in iam.policies.all(): all_policies.append(policy) except Exception as e: exc = BotoConnectionIssue(str(e), 'iamuser', account, None) self.slurp_exception((self.index, account, 'universal'), exc, exception_map) continue for policy in all_policies: if self.check_ignore_list(policy.policy_name): continue if self.check_ignore_list(policy.arn): continue item_config = { 'name': policy.policy_name, 'arn': policy.arn, 'create_date': str(policy.create_date), 'update_date': str(policy.update_date), 'default_version_id': policy.default_version_id, 'attachment_count': policy.attachment_count, 'attached_users': [a.arn for a in policy.attached_users.all()], 'attached_groups': [a.arn for a in policy.attached_groups.all()], 'attached_roles': [a.arn for a in policy.attached_roles.all()], 'policy': policy.default_version.document } app.logger.debug("Slurping %s (%s) from %s" % (self.i_am_singular, policy.policy_name, account)) item_list.append( ManagedPolicyItem(account=account, name=policy.policy_name, config=item_config) ) return item_list, exception_map
def slurp(self): """ :returns: item_list - list of Route53 records. :returns: exception_map - A dict where the keys are a tuple containing the location of the exception and the value is the actual exception """ self.prep_for_slurp() item_list = [] exception_map = {} from security_monkey.common.sts_connect import connect for account in self.accounts: app.logger.debug("Checking {}/{}".format(self.index, account)) try: route53conn = connect(account, 'route53') # Note : This fails if you have over 100 hosted zones # maybe create paged_wrap_aws_rate_limited_call route53_response = self.wrap_aws_rate_limited_call( route53conn.get_all_hosted_zones ) zones = route53_response['ListHostedZonesResponse']['HostedZones'] for zone in zones: zone_id = zone['Id'][12:] # Trim leading '/hostedzone' record_sets = self.wrap_aws_rate_limited_call( route53conn.get_all_rrsets, hosted_zone_id=zone_id ) for record in record_sets: if (record.type == 'CNAME' and any([x.endswith( tuple(self.third_party_domains)) for x in record.resource_records])): # This CNAME contains a record which points to a # third party domain item = Route53Item( account=account, name=record.name, config=record) item_list.append(item) except Exception as e: exc = BotoConnectionIssue(str(e), self.index, account, None) self.slurp_exception((self.index, account, 'universal'), exc, exception_map) continue app.logger.debug( "Found {} {}".format(len(item_list), self.i_am_plural)) return item_list, exception_map
def describe_configuration_recorders(self, **kwargs): from security_monkey.common.sts_connect import connect config_service = connect(kwargs['account_name'], 'boto3.config.client', region=kwargs['region']) response = self.wrap_aws_rate_limited_call( config_service.describe_configuration_recorders) config_recorders = response.get('ConfigurationRecorders', []) return [ recorder for recorder in config_recorders if not self.check_ignore_list(recorder.get('name')) ]
def describe_volumes(self, **kwargs): from security_monkey.common.sts_connect import connect ec2 = connect(kwargs['account_name'], 'boto3.ec2.client', region=kwargs['region'], assumed_role=kwargs['assumed_role']) response = self.wrap_aws_rate_limited_call(ec2.describe_volumes) volumes = response.get('Volumes') return [ volume for volume in volumes if not self.check_ignore_list(volume_name(volume)) ]
def list_functions(self, **kwargs): from security_monkey.common.sts_connect import connect app.logger.debug("Checking {}/{}/{}".format(self.index, kwargs['account_name'], kwargs['region'])) lambda_client = connect(kwargs['account_name'], 'boto3.lambda.client', region=kwargs['region'], assumed_role=kwargs['assumed_role']) response = self.wrap_aws_rate_limited_call( lambda_client.list_functions ) lambda_functions = response.get('Functions') return lambda_functions
def describe_snapshots(self, **kwargs): from security_monkey.common.sts_connect import connect ec2 = connect(kwargs['account_name'], 'boto3.ec2.client', region=kwargs['region'], assumed_role=kwargs['assumed_role']) response = self.wrap_aws_rate_limited_call(ec2.describe_snapshots, OwnerIds=['self']) snapshots = response.get('Snapshots') return [ snapshot for snapshot in snapshots if not self.check_ignore_list(snapshot_name(snapshot)) ]
def list_functions(self, **kwargs): from security_monkey.common.sts_connect import connect app.logger.debug("Checking {}/{}/{}".format(self.index, kwargs['account_name'], kwargs['region'])) lambda_client = connect(kwargs['account_name'], 'boto3.lambda.client', region=kwargs['region'], assumed_role=kwargs['assumed_role']) response = self.wrap_aws_rate_limited_call( lambda_client.list_functions) lambda_functions = response.get('Functions') return lambda_functions
def get_all_certs_in_region(self, account, region, exception_map): from security_monkey.common.sts_connect import connect import traceback all_certs = [] app.logger.debug("Checking {}/{}/{}".format(self.index, account, region)) try: iamconn = connect(account, 'iam', region=region) marker = None while True: certs = self.wrap_aws_rate_limited_call( iamconn.list_server_certs, marker=marker ) all_certs.extend(certs.server_certificate_metadata_list) if certs.is_truncated == u'true': marker = certs.marker else: break for cert in all_certs: try: iam_cert = self.wrap_aws_rate_limited_call( iamconn.get_server_certificate, cert_name=cert.server_certificate_name ) cert['body'] = iam_cert.certificate_body cert['chain'] = None if hasattr(iam_cert, 'certificate_chain'): cert['chain'] = iam_cert.certificate_chain cert_info = get_cert_info(cert['body']) for key in cert_info.iterkeys(): cert[key] = cert_info[key] except Exception as e: app.logger.warn(traceback.format_exc()) app.logger.error("Invalid certificate {}!".format(cert.server_certificate_id)) self.slurp_exception( (self.index, account, 'universal', cert.server_certificate_name), e, exception_map, source="{}-watcher".format(self.index)) except Exception as e: app.logger.warn(traceback.format_exc()) if region not in TROUBLE_REGIONS: exc = BotoConnectionIssue(str(e), self.index, account, 'universal') self.slurp_exception((self.index, account, 'universal'), exc, exception_map, source="{}-watcher".format(self.index)) app.logger.info("Found {} {} from {}/{}".format(len(all_certs), self.i_am_plural, account, 'universal')) return all_certs
def slurp(self): """ :returns: item_list - list of virtual gateways :returns: exception_map - A dict where the keys are a tuple containing the location of the exception and the value is the actual exception """ self.prep_for_slurp() from security_monkey.common.sts_connect import connect item_list = [] exception_map = {} for account in self.accounts: for region in regions(): app.logger.debug( "Checking {}/{}/{}".format(self.index, account, region.name)) try: dc = connect(account, 'boto3.ec2.client', region=region) response = self.wrap_aws_rate_limited_call( dc.describe_vpn_gateways ) gateways = response.get('VpnGateways') except Exception as e: if region.name not in TROUBLE_REGIONS: exc = BotoConnectionIssue( str(e), self.index, account, region.name) self.slurp_exception( (self.index, account, region.name), exc, exception_map) continue app.logger.debug("Found {} {}.".format( len(gateways), self.i_am_plural)) for gateway in gateways: name = gateway['VpnGatewayId'] if self.check_ignore_list(name): continue config = { 'name': name, 'state': gateway.get('State'), 'type': gateway.get('Type'), 'vpcAttachments': gateway.get('VpcAttachments'), 'virtual_gateway_state': gateway.get('VirtualGatewayState') } item = VirtualGatewayItem( region=region.name, account=account, name=name, config=dict(config)) item_list.append(item) return item_list, exception_map
def slurp(self): """ :returns: item_list - list of SES Identities. :returns: exception_map - A dict where the keys are a tuple containing the location of the exception and the value is the actual exception """ self.prep_for_slurp() from security_monkey.common.sts_connect import connect item_list = [] exception_map = {} for account in self.accounts: for region in regions(): if region.name == 'eu-central-1': # as of boto 2.34.0, boto cannot connect to ses in eu-central-1 # TODO: Remove this if-block when boto can handle ses in eu-central-1 continue app.logger.debug("Checking {}/{}/{}".format(self.index, account, region.name)) try: ses = connect(account, 'ses', region=region.name) response = self.wrap_aws_rate_limited_call( ses.list_identities ) identities = response.Identities response = self.wrap_aws_rate_limited_call( ses.list_verified_email_addresses ) verified_identities = response.VerifiedEmailAddresses except Exception as e: if region.name not in TROUBLE_REGIONS: exc = BotoConnectionIssue(str(e), self.index, account, region.name) self.slurp_exception((self.index, account, region.name), exc, exception_map) continue app.logger.debug("Found {} {}. {} are verified.".format(len(identities), self.i_am_plural, len(verified_identities))) for identity in identities: if self.check_ignore_list(identity): continue config = { 'name': identity, 'verified': identity in verified_identities } item = SESItem(region=region.name, account=account, name=identity, config=dict(config)) item_list.append(item) return item_list, exception_map
def get_all_topics_in_region(self, account, region): from security_monkey.common.sts_connect import connect sns = connect(account, 'sns', region=region) app.logger.debug("Checking {}/{}/{}".format(SNS.index, account, region.name)) topics = [] marker = None while True: topics_response = sns.get_all_topics(next_token=marker) current_page_topics = topics_response['ListTopicsResponse']['ListTopicsResult']['Topics'] topics.extend(current_page_topics) if topics_response[u'ListTopicsResponse'][u'ListTopicsResult'][u'NextToken']: marker = topics_response[u'ListTopicsResponse'][u'ListTopicsResult'][u'NextToken'] else: break return (sns, topics)
def s3connect(account, bucket): """ s3connect will attempt to connect to an s3 bucket resource. If the resource does not exist it will attempt to create it :param account: string the aws account you are connecting to :param bucket: string the name of the bucket you wish to connect to :returns: Boolean of connection Status """ conn = connect(account, 's3') if conn.lookup(bucket) is None: bucket = conn.create_bucket(bucket) else: bucket = conn.get_bucket(bucket) key = Key(bucket) return conn, bucket, key
def s3connect(self, account, bucket): """ s3connect will attempt to connect to an s3 bucket resource. If the resource does not exist it will attempt to create it :param account: string the aws account you are connecting to :param bucket: string the name of the bucket you wish to connect to :returns: Boolean of connection Status """ self.conn = connect(account, 's3') if self.conn.lookup(bucket) is None: app.logger.debug("Bucket Does not exist. Creating one") self.bucket = self.conn.create_bucket(bucket) else: self.bucket = self.conn.get_bucket(bucket) self.key = Key(self.bucket) return True
def get_all_topics_in_region(self, account, region): from security_monkey.common.sts_connect import connect sns = connect(account, 'sns', region=region) app.logger.debug("Checking {}/{}/{}".format(SNS.index, account, region.name)) topics = [] marker = None while True: topics_response = self.wrap_aws_rate_limited_call( sns.get_all_topics, next_token=marker ) current_page_topics = topics_response['ListTopicsResponse']['ListTopicsResult']['Topics'] topics.extend(current_page_topics) if topics_response[u'ListTopicsResponse'][u'ListTopicsResult'][u'NextToken']: marker = topics_response[u'ListTopicsResponse'][u'ListTopicsResult'][u'NextToken'] else: break return sns, topics
def get_all_dbsecurity_groups(self, **kwargs): from security_monkey.common.sts_connect import connect sgs = [] rds = connect(kwargs['account_name'], 'rds', region=kwargs['region'], assumed_role=kwargs['assumed_role']) marker = None while True: response = self.wrap_aws_rate_limited_call( rds.get_all_dbsecurity_groups, marker=marker) sgs.extend(response) if response.marker: marker = response.marker else: break return sgs