def slurp(self): """ :returns: item_list - list of IAM SSH Keypairs. :returns: exception_map - A dict where the keys are a tuple containing the location of the exception and the value is the actual exception """ item_list = [] exception_map = {} from security_monkey.common.sts_connect import connect for account in self.accounts: try: ec2 = connect(account, 'ec2') regions = ec2.get_all_regions() except Exception as e: # EC2ResponseError # Some Accounts don't subscribe to EC2 and will throw an exception here. exc = BotoConnectionIssue(str(e), 'keypair', account, None) self.slurp_exception((self.index, account), exc, exception_map) continue for region in regions: app.logger.debug("Checking {}/{}/{}".format( Keypair.index, account, region.name)) try: rec2 = connect(account, 'ec2', region=region) kps = self.wrap_aws_rate_limited_call( rec2.get_all_key_pairs) except Exception as e: if region.name not in TROUBLE_REGIONS: exc = BotoConnectionIssue(str(e), 'keypair', account, region.name) self.slurp_exception( (self.index, account, region.name), exc, exception_map) continue app.logger.debug("Found {} {}".format(len(kps), Keypair.i_am_plural)) for kp in kps: ### Check if this Keypair is on the Ignore List ### ignore_item = False for ignore_item_name in IGNORE_PREFIX[self.index]: if kp.name.lower().startswith( ignore_item_name.lower()): ignore_item = True break if ignore_item: continue item_list.append( KeypairItem(region=region.name, account=account, name=kp.name, config={'fingerprint': kp.fingerprint})) return item_list, exception_map
def slurp(self): """ :returns: item_list - list of IAM SSH Keypairs. :returns: exception_map - A dict where the keys are a tuple containing the location of the exception and the value is the actual exception """ self.prep_for_slurp() item_list = [] exception_map = {} from security_monkey.common.sts_connect import connect for account in self.accounts: try: account_db = Account.query.filter(Account.name == account).first() account_number = account_db.identifier ec2 = connect(account, 'ec2') regions = ec2.get_all_regions() except Exception as e: # EC2ResponseError # Some Accounts don't subscribe to EC2 and will throw an exception here. exc = BotoConnectionIssue(str(e), 'keypair', account, None) self.slurp_exception((self.index, account), exc, exception_map, source="{}-watcher".format(self.index)) continue for region in regions: app.logger.debug("Checking {}/{}/{}".format(Keypair.index, account, region.name)) try: rec2 = connect(account, 'boto3.ec2.client', region=region) kps = self.wrap_aws_rate_limited_call( rec2.describe_key_pairs ) except Exception as e: if region.name not in TROUBLE_REGIONS: exc = BotoConnectionIssue(str(e), 'keypair', account, region.name) self.slurp_exception((self.index, account, region.name), exc, exception_map, source="{}-watcher".format(self.index)) continue app.logger.debug("Found {} {}".format(len(kps), Keypair.i_am_plural)) for kp in kps['KeyPairs']: if self.check_ignore_list(kp['KeyName']): continue arn = ARN_PREFIX + ':ec2:{region}:{account_number}:key-pair/{name}'.format( region=region.name, account_number=account_number, name=kp["KeyName"]) item_list.append(KeypairItem(region=region.name, account=account, name=kp["KeyName"], arn=arn, config={ 'fingerprint': kp["KeyFingerprint"], 'arn': arn, 'name': kp["KeyName"] }, source_watcher=self)) return item_list, exception_map
def slurp(self): """ :returns: item_list - list of Redshift Policies. :returns: exception_map - A dict where the keys are a tuple containing the location of the exception and the value is the actual exception """ self.prep_for_slurp() from security_monkey.common.sts_connect import connect item_list = [] exception_map = {} for account in self.accounts: for region in regions(): app.logger.debug("Checking {}/{}/{}".format(self.index, account, region.name)) try: redshift = connect(account, 'redshift', region=region) response = self.wrap_aws_rate_limited_call( redshift.describe_clusters ) all_clusters = response['DescribeClustersResponse']['DescribeClustersResult']['Clusters'] except Exception as e: if region.name not in TROUBLE_REGIONS: exc = BotoConnectionIssue(str(e), 'redshift', account, region.name) self.slurp_exception((self.index, account, region.name), exc, exception_map) continue app.logger.debug("Found {} {}".format(len(all_clusters), Redshift.i_am_plural)) for cluster in all_clusters: cluster_id = cluster['ClusterIdentifier'] if self.check_ignore_list(cluster_id): continue item = RedshiftCluster(region=region.name, account=account, name=cluster_id, config=dict(cluster)) item_list.append(item) return item_list, exception_map
def decorated_function(*args, **kwargs): try: return f(*args, **kwargs) except Exception as e: index = kwargs.get('index') account = kwargs.get('account_name') # Allow the recording region to be overridden for universal tech like IAM region = kwargs.get('exception_record_region') or kwargs.get( 'region') name = kwargs.get('name') exception_map = kwargs.get('exception_map') exc = BotoConnectionIssue(str(e), index, account, name) if name: location = (index, account, region, name) elif region: location = (index, account, region) elif account: location = (index, account) else: location = (index, ) exception_map[location] = exc # Store the exception (the original one passed in, not exc): store_exception(source=source, location=location, exception=e)
def get_all_certs_in_region(self, account, region, exception_map): from security_monkey.common.sts_connect import connect import traceback all_certs = [] app.logger.debug("Checking {}/{}/{}".format(self.index, account, region)) try: iamconn = connect(account, 'iam', region=region) marker = None while True: certs = self.wrap_aws_rate_limited_call( iamconn.list_server_certs, marker=marker) all_certs.extend(certs.server_certificate_metadata_list) if certs.is_truncated == u'true': marker = certs.marker else: break except Exception as e: app.logger.warn(traceback.format_exc()) if region not in TROUBLE_REGIONS: exc = BotoConnectionIssue(str(e), self.index, account, region) self.slurp_exception((self.index, account, region), exc, exception_map) app.logger.info("Found {} {} from {}/{}".format( len(all_certs), self.i_am_plural, account, region)) return all_certs
def decorated_function(*args, **kwargs): # prevent these from being passed to the wrapped function: m = kwargs.pop if pop_exception_fields else kwargs.get exception_values = { 'index': m('index'), 'account': m('account_name', None), 'exception_record_region': m('exception_record_region', None), 'name': m('name', None), 'exception_map': m('exception_map') } try: return f(*args, **kwargs) except Exception as e: index = exception_values['index'] account = exception_values['account'] # Allow the recording region to be overridden for universal tech like IAM region = exception_values[ 'exception_record_region'] or kwargs.get('region') name = exception_values['name'] exception_map = exception_values['exception_map'] exc = BotoConnectionIssue(str(e), index, account, name) if name: location = (index, account, region, name) elif region: location = (index, account, region) elif account: location = (index, account) else: location = (index, ) exception_map[location] = exc # Store the exception (the original one passed in, not exc): store_exception(source=source, location=location, exception=e)
def slurp(self): """ :returns: item_dict - list of IAM Account attributes. :returns: exception_map - A dict where the keys are a tuple containing the location of the exception and the value is the actual exception """ self.prep_for_slurp() item_list = [] exception_map = {} from security_monkey.common.sts_connect import connect for account in self.accounts: try: iam = connect(account, 'iam') account_summary = self.wrap_aws_rate_limited_call( iam.get_account_summary) except Exception as e: exc = BotoConnectionIssue(str(e), 'iamaccount', account, None) self.slurp_exception((self.index, account, 'universal'), exc, exception_map) continue item_list.append( IAMAccountItem(account=account, name=account, config=dict(account_summary))) return item_list, exception_map
def decorated_function(*args, **kwargs): item_list = [] exception_map = {} for account_name in accounts: account = Account.query.filter(Account.name == account_name).first() if not account: app.logger.error("Couldn't find account with name", account_name) return try: (role, regions) = get_regions(account, service_name) except Exception as e: exc = BotoConnectionIssue(str(e), index, account.name, None) exception_map[(index, account)] = exc return item_list, exception_map for region in regions: kwargs['index'] = index kwargs['account_name'] = account.name kwargs['account_number'] = account.number kwargs['region'] = region kwargs['assume_role'] = account.role_name or 'SecurityMonkey' if role: kwargs['assumed_role'] = role or 'SecurityMonkey' kwargs['exception_map'] = {} if exception_record_region: kwargs['exception_record_region'] = exception_record_region itm, exc = f(*args, **kwargs) item_list.extend(itm) exception_map.update(exc) return item_list, exception_map
def slurp(self): """ :returns: item_list - list of connections :returns: exception_map - A dict where the keys are a tuple containing the location of the exception and the value is the actual exception """ self.prep_for_slurp() from security_monkey.common.sts_connect import connect item_list = [] exception_map = {} for account in self.accounts: for region in regions(): app.logger.debug("Checking {}/{}/{}".format( self.index, account, region.name)) try: dc = connect(account, 'boto3.directconnect.client', region=region) response = self.wrap_aws_rate_limited_call( dc.describe_connections) connections = response.get('connections') except Exception as e: if region.name not in TROUBLE_REGIONS: exc = BotoConnectionIssue(str(e), self.index, account, region.name) self.slurp_exception( (self.index, account, region.name), exc, exception_map) continue app.logger.debug("Found {} {}.".format(len(connections), self.i_am_plural)) for connection in connections: name = connection['connectionName'] if self.check_ignore_list(name): continue config = { 'name': name, 'owner_account': connection.get('ownerAccount'), 'connection_id': connection.get('connectionId'), 'connection_name': connection.get('connectionName'), 'connection_state': connection.get('connectionState'), 'region': connection.get('region'), 'location': connection.get('location'), 'bandwidth': connection.get('bandwidth'), 'vlan': connection.get('vlan'), 'partner_name': connection.get('partnerName'), } item = ConnectionItem(region=region.name, account=account, name=name, config=dict(config), source_watcher=self) item_list.append(item) return item_list, exception_map
def slurp(self): """ :returns: item_list - list of Redshift Policies. :returns: exception_map - A dict where the keys are a tuple containing the location of the exception and the value is the actual exception """ self.prep_for_slurp() from security_monkey.common.sts_connect import connect item_list = [] exception_map = {} for account in self.accounts: account_db = Account.query.filter(Account.name == account).first() account_number = account_db.identifier for region in regions(): app.logger.debug("Checking {}/{}/{}".format(self.index, account, region.name)) try: redshift = connect(account, 'redshift', region=region) all_clusters = [] marker = None while True: response = self.wrap_aws_rate_limited_call( redshift.describe_clusters, marker=marker ) all_clusters.extend(response['DescribeClustersResponse']['DescribeClustersResult']['Clusters']) if response['DescribeClustersResponse']['DescribeClustersResult']['Marker'] is not None: marker = response['DescribeClustersResponse']['DescribeClustersResult']['Marker'] else: break except Exception as e: if region.name not in TROUBLE_REGIONS: exc = BotoConnectionIssue(str(e), 'redshift', account, region.name) self.slurp_exception((self.index, account, region.name), exc, exception_map, source="{}-watcher".format(self.index)) continue app.logger.debug("Found {} {}".format(len(all_clusters), Redshift.i_am_plural)) for cluster in all_clusters: cluster_id = cluster['ClusterIdentifier'] if self.check_ignore_list(cluster_id): continue arn = ARN_PREFIX + ':redshift:{region}:{account_number}:cluster:{name}'.format( region=region.name, account_number=account_number, name=cluster_id) cluster['arn'] = arn item = RedshiftCluster(region=region.name, account=account, name=cluster_id, arn=arn, config=dict(cluster), source_watcher=self) item_list.append(item) return item_list, exception_map
def get_all_certs_in_region(self, account, region, exception_map): from security_monkey.common.sts_connect import connect import traceback all_certs = [] app.logger.debug("Checking {}/{}/{}".format(self.index, account, region)) try: iamconn = connect(account, 'iam', region=region) marker = None while True: certs = self.wrap_aws_rate_limited_call( iamconn.list_server_certs, marker=marker) all_certs.extend(certs.server_certificate_metadata_list) if certs.is_truncated == u'true': marker = certs.marker else: break for cert in all_certs: try: iam_cert = self.wrap_aws_rate_limited_call( iamconn.get_server_certificate, cert_name=cert.server_certificate_name) cert['body'] = iam_cert.certificate_body cert['chain'] = None if hasattr(iam_cert, 'certificate_chain'): cert['chain'] = iam_cert.certificate_chain cert_info = get_cert_info(cert['body']) for key in cert_info.iterkeys(): cert[key] = cert_info[key] except Exception as e: app.logger.warn(traceback.format_exc()) app.logger.error("Invalid certificate {}!".format( cert.server_certificate_id)) self.slurp_exception( (self.index, account, 'universal', cert.server_certificate_name), e, exception_map, source="{}-watcher".format(self.index)) except Exception as e: app.logger.warn(traceback.format_exc()) if region not in TROUBLE_REGIONS: exc = BotoConnectionIssue(str(e), self.index, account, 'universal') self.slurp_exception((self.index, account, 'universal'), exc, exception_map, source="{}-watcher".format(self.index)) app.logger.info("Found {} {} from {}/{}".format( len(all_certs), self.i_am_plural, account, 'universal')) return all_certs
def slurp(self): """ :returns: item_list - list of SQS Policies. :returns: exception_map - A dict where the keys are a tuple containing the location of the exception and the value is the actual exception """ item_list = [] exception_map = {} from security_monkey.common.sts_connect import connect for account in self.accounts: for region in regions(): app.logger.debug("Checking {}/{}/{}".format(SQS.index, account, region.name)) try: sqs = connect(account, 'sqs', region=region) all_queues = self.wrap_aws_rate_limited_call( sqs.get_all_queues ) except Exception as e: if region.name not in TROUBLE_REGIONS: exc = BotoConnectionIssue(str(e), 'sqs', account, region.name) self.slurp_exception((self.index, account, region.name), exc, exception_map) continue app.logger.debug("Found {} {}".format(len(all_queues), SQS.i_am_plural)) for q in all_queues: ### Check if this Queue is on the Ignore List ### ignore_item = False for ignore_item_name in IGNORE_PREFIX[self.index]: if q.name.lower().startswith(ignore_item_name.lower()): ignore_item = True break if ignore_item: continue try: policy = self.wrap_aws_rate_limited_call( q.get_attributes, attributes='Policy' ) if 'Policy' in policy: try: json_str = policy['Policy'] policy = json.loads(json_str) item = SQSItem(region=region.name, account=account, name=q.name, config=policy) item_list.append(item) except: self.slurp_exception((self.index, account, region, q.name), InvalidAWSJSON(json_str), exception_map) except boto.exception.SQSError: # A number of Queues are so ephemeral that they may be gone by the time # the code reaches here. Just ignore them and move on. pass return item_list, exception_map
def slurp(self): """ :returns: item_list - list of Route53 records. :returns: exception_map - A dict where the keys are a tuple containing the location of the exception and the value is the actual exception """ self.prep_for_slurp() item_list = [] exception_map = {} from security_monkey.common.sts_connect import connect for account in self.accounts: app.logger.debug("Checking {}/{}".format(self.index, account)) try: route53conn = connect(account, 'route53') # Note : This fails if you have over 100 hosted zones # maybe create paged_wrap_aws_rate_limited_call route53_response = self.wrap_aws_rate_limited_call( route53conn.get_all_hosted_zones ) zones = route53_response['ListHostedZonesResponse']['HostedZones'] for zone in zones: zone_id = zone['Id'][12:] # Trim leading '/hostedzone' record_sets = self.wrap_aws_rate_limited_call( route53conn.get_all_rrsets, hosted_zone_id=zone_id ) for record in record_sets: if (record.type == 'CNAME' and any([x.endswith( tuple(self.third_party_domains)) for x in record.resource_records])): # This CNAME contains a record which points to a # third party domain item = Route53Item( account=account, name=record.name, config=record) item_list.append(item) except Exception as e: exc = BotoConnectionIssue(str(e), self.index, account, None) self.slurp_exception((self.index, account, 'universal'), exc, exception_map) continue app.logger.debug( "Found {} {}".format(len(item_list), self.i_am_plural)) return item_list, exception_map
def slurp(self): """ :returns: item_list - list of virtual gateways :returns: exception_map - A dict where the keys are a tuple containing the location of the exception and the value is the actual exception """ self.prep_for_slurp() from security_monkey.common.sts_connect import connect item_list = [] exception_map = {} for account in self.accounts: for region in regions(): app.logger.debug( "Checking {}/{}/{}".format(self.index, account, region.name)) try: dc = connect(account, 'boto3.ec2.client', region=region) response = self.wrap_aws_rate_limited_call( dc.describe_vpn_gateways ) gateways = response.get('VpnGateways') except Exception as e: if region.name not in TROUBLE_REGIONS: exc = BotoConnectionIssue( str(e), self.index, account, region.name) self.slurp_exception( (self.index, account, region.name), exc, exception_map) continue app.logger.debug("Found {} {}.".format( len(gateways), self.i_am_plural)) for gateway in gateways: name = gateway['VpnGatewayId'] if self.check_ignore_list(name): continue config = { 'name': name, 'state': gateway.get('State'), 'type': gateway.get('Type'), 'vpcAttachments': gateway.get('VpcAttachments'), 'virtual_gateway_state': gateway.get('VirtualGatewayState') } item = VirtualGatewayItem( region=region.name, account=account, name=name, config=dict(config)) item_list.append(item) return item_list, exception_map
def slurp(self): """ :returns: item_list - list of SES Identities. :returns: exception_map - A dict where the keys are a tuple containing the location of the exception and the value is the actual exception """ self.prep_for_slurp() from security_monkey.common.sts_connect import connect item_list = [] exception_map = {} for account in self.accounts: for region in regions(): if region.name == 'eu-central-1': # as of boto 2.34.0, boto cannot connect to ses in eu-central-1 # TODO: Remove this if-block when boto can handle ses in eu-central-1 continue app.logger.debug("Checking {}/{}/{}".format(self.index, account, region.name)) try: ses = connect(account, 'ses', region=region.name) response = self.wrap_aws_rate_limited_call( ses.list_identities ) identities = response.Identities response = self.wrap_aws_rate_limited_call( ses.list_verified_email_addresses ) verified_identities = response.VerifiedEmailAddresses except Exception as e: if region.name not in TROUBLE_REGIONS: exc = BotoConnectionIssue(str(e), self.index, account, region.name) self.slurp_exception((self.index, account, region.name), exc, exception_map) continue app.logger.debug("Found {} {}. {} are verified.".format(len(identities), self.i_am_plural, len(verified_identities))) for identity in identities: if self.check_ignore_list(identity): continue config = { 'name': identity, 'verified': identity in verified_identities } item = SESItem(region=region.name, account=account, name=identity, config=dict(config)) item_list.append(item) return item_list, exception_map
def slurp(self): """ :returns: item_list - list of SNSItem's. :returns: exception_map - A dict where the keys are a tuple containing the location of the exception and the value is the actual exception """ item_list = [] exception_map = {} for account in self.accounts: for region in regions(): try: (sns, topics) = self.get_all_topics_in_region(account, region) except Exception as e: if region.name not in TROUBLE_REGIONS: exc = BotoConnectionIssue(str(e), 'sns', account, region.name) self.slurp_exception( (self.index, account, region.name), exc, exception_map) continue app.logger.debug("Found {} {}".format(len(topics), SNS.i_am_plural)) for topic in topics: arn = topic['TopicArn'] ### Check if this SNS Topic is on the Ignore List ### ignore_item = False for ignore_item_name in IGNORE_PREFIX[self.index]: if arn.lower().startswith(ignore_item_name.lower()): ignore_item = True break if ignore_item: continue attrs = self.wrap_aws_rate_limited_call( sns.get_topic_attributes, arn) item = self.build_item(arn=arn, attrs=attrs, region=region.name, account=account, exception_map=exception_map) if item: item_list.append(item) return item_list, exception_map
def slurp(self): """ :returns: item_list - list of SNSItem's. :returns: exception_map - A dict where the keys are a tuple containing the location of the exception and the value is the actual exception """ self.prep_for_slurp() item_list = [] exception_map = {} for account in self.accounts: for region in regions(): try: (sns, topics) = self.get_all_topics_in_region(account, region) except Exception as e: if region.name not in TROUBLE_REGIONS: exc = BotoConnectionIssue(str(e), 'sns', account, region.name) self.slurp_exception( (self.index, account, region.name), exc, exception_map, source="{}-watcher".format(self.index)) continue app.logger.debug("Found {} {}".format(len(topics), SNS.i_am_plural)) for topic in topics: arn = topic['TopicArn'] if self.check_ignore_list(arn.split(':')[5]): continue item = self.build_item(arn=arn, conn=sns, region=region.name, account=account, exception_map=exception_map) if item: item_list.append(item) return item_list, exception_map
def decorated_function(*args, **kwargs): try: return f(*args, **kwargs) except Exception as e: index = kwargs.get('index') account = kwargs.get('account_name') # Allow the recording region to be overridden for universal tech like IAM region = kwargs.get('exception_record_region') or kwargs.get( 'region') name = kwargs.get('name') exception_map = kwargs.get('exception_map') exc = BotoConnectionIssue(str(e), index, account, name) if name: exception_map[(index, account, region, name)] = exc elif region: exception_map[(index, account, region)] = exc elif account: exception_map[(index, account)] = exc else: exception_map[(index, )] = exc
def slurp(self): """ :returns: item_list - list of ElasticSearchService Items :return: exception_map - A dict where the keys are a tuple containing the location of the exception and the value is the actual exception """ self.prep_for_slurp() item_list = [] exception_map = {} for account in self.accounts: for region in regions(): try: if region.name in TROUBLE_REGIONS: continue (client, domains) = self.get_all_es_domains_in_region( account, region) except Exception as e: if region.name not in TROUBLE_REGIONS: exc = BotoConnectionIssue(str(e), 'es', account, region.name) self.slurp_exception( (self.index, account, region.name), exc, exception_map) continue app.logger.debug("Found {} {}".format( len(domains), ElasticSearchService.i_am_plural)) for domain in domains: if self.check_ignore_list(domain["DomainName"]): continue # Fetch the policy: item = self.build_item(domain["DomainName"], client, region.name, account, exception_map) if item: item_list.append(item) return item_list, exception_map
def slurp(self): """ :returns: item_list - list of Security Groups. :returns: exception_map - A dict where the keys are a tuple containing the location of the exception and the value is the actual exception """ self.prep_for_slurp() item_list = [] exception_map = {} from security_monkey.common.sts_connect import connect for account in self.accounts: account_db = Account.query.filter(Account.name == account).first() account_number = account_db.identifier try: ec2 = connect(account, 'ec2') regions = ec2.get_all_regions() except Exception as e: # EC2ResponseError # Some Accounts don't subscribe to EC2 and will throw an exception here. exc = BotoConnectionIssue(str(e), self.index, account, None) self.slurp_exception((self.index, account), exc, exception_map, source="{}-watcher".format(self.index)) continue for region in regions: app.logger.debug("Checking {}/{}/{}".format( self.index, account, region.name)) try: rec2 = connect(account, 'boto3.ec2.client', region=region) # Retrieve security groups here sgs = self.wrap_aws_rate_limited_call( rec2.describe_security_groups) if self.get_detail_level() != 'NONE': # We fetch tags here to later correlate instances tags = self.wrap_aws_rate_limited_call( rec2.describe_tags) # Retrieve all instances instances = self.wrap_aws_rate_limited_call( rec2.describe_instances) app.logger.info( "Number of instances found in region {}: {}". format(region.name, len(instances))) except Exception as e: if region.name not in TROUBLE_REGIONS: exc = BotoConnectionIssue(str(e), self.index, account, region.name) self.slurp_exception( (self.index, account, region.name), exc, exception_map, source="{}-watcher".format(self.index)) continue app.logger.debug("Found {} {}".format(len(sgs), self.i_am_plural)) if self.get_detail_level() != 'NONE': app.logger.info("Creating mapping of sg_id's to instances") # map sgid => instance sg_instances = {} for reservation in instances['Reservations']: for instance in reservation['Instances']: for group in instance['SecurityGroups']: if group['GroupId'] not in sg_instances: sg_instances[group['GroupId']] = [instance] else: sg_instances[group['GroupId']].append( instance) app.logger.info( "Creating mapping of instance_id's to tags") # map instanceid => tags instance_tags = {} for tag in tags['Tags']: if tag['ResourceId'] not in instance_tags: instance_tags[tag['ResourceId']] = [tag] else: instance_tags[tag['ResourceId']].append(tag) app.logger.info("Done creating mappings") for sg in sgs['SecurityGroups']: if self.check_ignore_list(sg['GroupName']): continue arn = ARN_PREFIX + ':ec2:{region}:{account_number}:security-group/{security_group_id}'.format( region=region.name, account_number=account_number, security_group_id=sg['GroupId']) item_config = { "id": sg['GroupId'], "name": sg['GroupName'], "description": sg.get('Description'), "vpc_id": sg.get('VpcId'), "owner_id": sg.get('OwnerId'), "region": region.name, "rules": [], "assigned_to": None, "arn": arn } for rule in sg['IpPermissions']: item_config['rules'] += self._build_rule( rule, "ingress") for rule in sg['IpPermissionsEgress']: item_config['rules'] += self._build_rule( rule, "egress") if self.get_detail_level() == 'SUMMARY': if 'InstanceId' in sg and sg[ 'InstanceId'] in sg_instances: item_config["assigned_to"] = "{} instances".format( len(sg_instances[sg['GroupId']])) else: item_config["assigned_to"] = "0 instances" elif self.get_detail_level() == 'FULL': assigned_to = [] if sg['GroupId'] in sg_instances: for instance in sg_instances[sg['GroupId']]: if instance['InstanceId'] in instance_tags: tagdict = { tag['Key']: tag['Value'] for tag in instance_tags[ instance['InstanceId']] } tagdict["instance_id"] = instance[ 'InstanceId'] else: tagdict = { "instance_id": instance['InstanceId'] } assigned_to.append(tagdict) item_config["assigned_to"] = assigned_to # Issue 40: Security Groups can have a name collision between EC2 and # VPC or between different VPCs within a given region. if sg.get('VpcId'): sg_name = "{0} ({1} in {2})".format( sg['GroupName'], sg['GroupId'], sg['VpcId']) else: sg_name = "{0} ({1})".format(sg['GroupName'], sg['GroupId']) item = SecurityGroupItem(region=region.name, account=account, name=sg_name, arn=arn, config=item_config, source_watcher=self) item_list.append(item) return item_list, exception_map
def slurp(self): """ :returns: item_list - list of IAM Roles. :returns: exception_map - A dict where the keys are a tuple containing the location of the exception and the value is the actual exception """ self.prep_for_slurp() item_list = [] exception_map = {} from security_monkey.common.sts_connect import connect for account in self.accounts: try: iam_b3 = connect(account, 'iam_boto3') managed_policies = all_managed_policies(iam_b3) iam = connect(account, 'iam') all_roles = [] marker = None while True: roles = self.wrap_aws_rate_limited_call(iam.list_roles, marker=marker) all_roles.extend(roles.roles) if roles.is_truncated == u'true': marker = roles.marker else: break except Exception as e: # Some Accounts don't subscribe to EC2 and will throw an exception here. exc = BotoConnectionIssue(str(e), 'iamrole', account, None) self.slurp_exception((self.index, account, 'universal'), exc, exception_map) continue for role in all_roles: item_config = {} app.logger.debug("Slurping %s (%s) from %s" % (self.i_am_singular, role.role_name, account)) if self.check_ignore_list(role.role_name): continue if managed_policies.has_key(role.arn): item_config['managed_policies'] = managed_policies.get( role.arn) assume_role_policy_document = role.get( 'assume_role_policy_document', '') assume_role_policy_document = urllib.unquote( assume_role_policy_document) assume_role_policy_document = json.loads( assume_role_policy_document) item_config[ 'assume_role_policy_document'] = assume_role_policy_document del role['assume_role_policy_document'] item_config['role'] = dict(role) instance_profiles = self.instance_profiles_for_role(iam, role) if len(instance_profiles) > 0: item_config['instance_profiles'] = [] for instance_profile in instance_profiles: del instance_profile['roles'] item_config['instance_profiles'].append( dict(instance_profile)) item_config['rolepolicies'] = {} for policy_name in self.policy_names_for_role(iam, role): policy_response = self.wrap_aws_rate_limited_call( iam.get_role_policy, role.role_name, policy_name) policy = policy_response.policy_document policy = urllib.unquote(policy) policy = json.loads(policy) item_config['rolepolicies'][policy_name] = policy item = IAMRoleItem(account=account, name=role.role_name, config=item_config) item_list.append(item) return item_list, exception_map
def slurp(self): """ :returns: item_list - list of networkinterface items. :returns: exception_map - A dict where the keys are a tuple containing the location of the exception and the value is the actual exception """ self.prep_for_slurp() item_list = [] exception_map = {} from security_monkey.common.sts_connect import connect for account in self.accounts: try: ec2 = connect(account, 'ec2') app.logger.debug("Checking {}/{}/{}".format( self.index, account, 'universal')) el_nis = self.wrap_aws_rate_limited_call( ec2.get_all_network_interfaces) except Exception as e: # EC2ResponseError # Some Accounts don't subscribe to EC2 and will throw an # exception here. exc = BotoConnectionIssue(str(e), self.index, account, None) self.slurp_exception((self.index, account, 'universal'), exc, exception_map) continue app.logger.debug("Found {} {}.".format(len(el_nis), self.i_am_plural)) for network_interface in el_nis: if self.check_ignore_list(network_interface.id): continue item_config = { 'availability_zone': network_interface.availability_zone, 'description': network_interface.description, 'network_interface_id': network_interface.id, 'mac_address': network_interface.mac_address, 'owner_id': network_interface.owner_id, 'private_ip_address': network_interface.private_ip_address, 'source_dest_check': network_interface.source_dest_check, 'status': network_interface.status, 'vpc_id': network_interface.vpc_id } if hasattr(network_interface, 'allocationId'): item_config[ 'allocation_id'] = network_interface.allocationId if hasattr(network_interface, 'associationId'): item_config[ 'association_id'] = network_interface.associationId if hasattr(network_interface, 'attachment') and (network_interface.attachment is not None): attachment = network_interface.attachment item_config['attachment'] = { 'attach_time': str(attachment.attach_time), 'delete_on_termination': attachment.delete_on_termination, 'device_index': attachment.device_index, 'id': attachment.id, 'instance_id': attachment.instance_id, 'instance_owner_id': attachment.instance_owner_id, 'status': attachment.status } if hasattr(network_interface, 'privateDnsName'): item_config[ 'private_dns_name'] = network_interface.privateDnsName if hasattr(network_interface, 'publicDnsName'): item_config[ 'public_dns_name'] = network_interface.publicDnsName if hasattr(network_interface, 'publicIp'): item_config[ 'public_ip_address'] = network_interface.publicIp if hasattr(network_interface, 'ipOwnerId'): item_config['ip_owner_id'] = network_interface.ipOwnerId item = ENIItem(region='universal', account=account, name=network_interface.id, config=item_config, source_watcher=self) item_list.append(item) return item_list, exception_map
def slurp(self): """ :returns: item_list - list of Elastic IPs. :returns: exception_map - A dict where the keys are a tuple containing the location of the exception and the value is the actual exception """ self.prep_for_slurp() item_list = [] exception_map = {} from security_monkey.common.sts_connect import connect for account in self.accounts: try: ec2 = connect(account, 'ec2') regions = ec2.get_all_regions() except Exception as e: # EC2ResponseError # Some Accounts don't subscribe to EC2 and will throw an exception here. exc = BotoConnectionIssue(str(e), self.index, account, None) self.slurp_exception((self.index, account), exc, exception_map, source="{}-watcher".format(self.index)) continue for region in regions: app.logger.debug("Checking {}/{}/{}".format( self.index, account, region.name)) try: rec2 = connect(account, 'boto3.ec2.client', region=region) el_ips = self.wrap_aws_rate_limited_call( rec2.describe_addresses) # Retrieve account tags to later match assigned EIP to instance tags = self.wrap_aws_rate_limited_call(rec2.describe_tags) except Exception as e: if region.name not in TROUBLE_REGIONS: exc = BotoConnectionIssue(str(e), self.index, account, region.name) self.slurp_exception( (self.index, account, region.name), exc, exception_map, source="{}-watcher".format(self.index)) continue app.logger.debug("Found {} {}".format(len(el_ips), self.i_am_plural)) for ip in el_ips['Addresses']: if self.check_ignore_list(str(ip['PublicIp'])): continue instance_name = None instance_tags = [ x['Value'] for x in tags['Tags'] if x['Key'] == "Name" and x.get('ResourceId') == ip.get('InstanceId') ] if instance_tags: (instance_name, ) = instance_tags if self.check_ignore_list(instance_name): continue item_config = { "assigned_to": instance_name, "public_ip": ip.get('PublicIp'), "instance_id": ip.get('InstanceId'), "domain": ip.get('Domain'), "allocation_id": ip.get('AllocationId'), "association_id": ip.get('AssociationId'), "network_interface_id": ip.get('NetworkInterfaceId'), "network_interface_owner_id": ip.get('NetworkInterfaceOwnerId'), "private_ip_address": ip.get('PrivateIpAddress') } ip_label = "{0}".format(ip.get('PublicIp')) item = ElasticIPItem(region=region.name, account=account, name=ip_label, config=item_config, source_watcher=self) item_list.append(item) return item_list, exception_map
def slurp(self): """ :returns: item_list - list of endpoints. :returns: exception_map - A dict where the keys are a tuple containing the location of the exception and the value is the actual exception """ self.prep_for_slurp() item_list = [] exception_map = {} from security_monkey.common.sts_connect import connect for account in self.accounts: for region in regions(): app.logger.debug("Checking {}/{}/{}".format( self.index, account, region.name)) try: conn = connect(account, 'boto3.ec2.client', region=region) all_vpc_endpoints_resp = self.wrap_aws_rate_limited_call( conn.describe_vpc_endpoints) all_vpc_endpoints = all_vpc_endpoints_resp.get( 'VpcEndpoints', []) except Exception as e: if region.name not in TROUBLE_REGIONS: exc = BotoConnectionIssue(str(e), self.index, account, region.name) self.slurp_exception( (self.index, account, region.name), exc, exception_map) continue app.logger.debug("Found {} {}".format(len(all_vpc_endpoints), self.i_am_plural)) for endpoint in all_vpc_endpoints: endpoint_name = endpoint.get('VpcEndpointId') if self.check_ignore_list(endpoint_name): continue service = endpoint.get('ServiceName', '').split('.')[-1] config = { "id": endpoint.get('VpcEndpointId'), "policy_document": endpoint.get('PolicyDocument', {}), "service_name": endpoint.get('ServiceName'), "service": service, "route_table_ids": endpoint.get('RouteTableIds', []), "creation_time_stamp": str(endpoint.get('CreationTimestamp')), "state": endpoint.get('State'), "vpc_id": endpoint.get('VpcId'), } item = EndpointItem(region=region.name, account=account, name=endpoint_name, config=config, source_watcher=self) item_list.append(item) return item_list, exception_map
def slurp(self): """ :returns: item_list - list of configs. :returns: exception_map - A dict where the keys are a tuple containing the location of the exception and the value is the actual exception """ self.prep_for_slurp() item_list = [] exception_map = {} from security_monkey.common.sts_connect import connect for account in self.accounts: for region in regions(): app.logger.debug( "Checking {}/{}/{}".format(self.index, account, region.name)) if region.name not in AVAILABLE_REGIONS: continue try: configService = connect( account, 'boto3.config.client', region=region) app.logger.debug( "Config policy is: {}".format(configService)) response = self.wrap_aws_rate_limited_call( configService.describe_config_rules ) config_rules = response.get('ConfigRules', []) except Exception as e: app.logger.debug("Exception found: {}".format(e)) if region.name not in TROUBLE_REGIONS: exc = BotoConnectionIssue( str(e), self.index, account, region.name) self.slurp_exception( (self.index, account, region.name), exc, exception_map) continue app.logger.debug("Found {} {}.".format( len(config_rules), self.i_am_plural)) for config_rule in config_rules: name = config_rule.get('ConfigRuleName') if self.check_ignore_list(name): continue item_config = { 'config_rule': name, 'config_rule_arn': config_rule.get('ConfigRuleArn'), 'config_rule_id': config_rule.get('ConfigRuleId'), 'scope': config_rule.get('Scope', {}), 'source': config_rule.get('Source', {}), 'imput_parameters': config_rule.get('InputParameters'), 'maximum_execution_frequency': config_rule.get('MaximumExecutionFrequency'), 'config_rule_state': config_rule.get('ConfigRuleState'), } item = ConfigItem( region=region.name, account=account, name=name, arn=config_rule.get('ConfigRuleArn'), config=item_config) item_list.append(item) return item_list, exception_map
def slurp(self): """ :returns: item_list - list of ACM Certificates with details. :returns: exception_map - A dict where the keys are a tuple containing the location of the exception and the value is the actual exception """ self.prep_for_slurp() item_list = [] exception_map = {} from security_monkey.common.sts_connect import connect for account in self.accounts: try: ec2 = connect(account, 'ec2') regions = ec2.get_all_regions() except Exception as e: # EC2ResponseError # Some Accounts don't subscribe to EC2 and will throw an exception here. exc = BotoConnectionIssue(str(e), 'keypair', account, None) self.slurp_exception((self.index, account), exc, exception_map, source="{}-watcher".format(self.index)) continue for region in regions: app.logger.debug("Checking {}/{}/{}".format( ACM.index, account, region.name)) try: acm = connect(account, 'boto3.acm.client', region=region, debug=1000) response = self.wrap_aws_rate_limited_call( acm.list_certificates) cert_list = response.get('CertificateSummaryList') except Exception as e: if region.name not in TROUBLE_REGIONS: exc = BotoConnectionIssue(str(e), 'acm', account, region.name) self.slurp_exception( (self.index, account, region.name), exc, exception_map, source="{}-watcher".format(self.index)) continue app.logger.debug("Found {} {}".format(len(cert_list), ACM.i_am_plural)) for cert in cert_list: app.logger.debug("Getting {} details for {}".format( ACM.i_am_singular, cert.get('DomainName'))) try: config = self.describe_certificate( acm, cert.get('CertificateArn')).get('Certificate') # Convert the datetime objects into ISO formatted strings in UTC if config.get('NotBefore'): config.update({ 'NotBefore': config.get('NotBefore').astimezone( tzutc()).isoformat() }) if config.get('NotAfter'): config.update({ 'NotAfter': config.get('NotAfter').astimezone( tzutc()).isoformat() }) if config.get('CreatedAt'): config.update({ 'CreatedAt': config.get('CreatedAt').astimezone( tzutc()).isoformat() }) if config.get('IssuedAt'): config.update({ 'IssuedAt': config.get('IssuedAt').astimezone( tzutc()).isoformat() }) if config.get('ImportedAt'): config.update({ 'ImportedAt': config.get('ImportedAt').astimezone( tzutc()).isoformat() }) item = ACMCertificate(region=region.name, account=account, name=cert.get('DomainName'), arn=cert.get('CertificateArn'), config=dict(config)) item_list.append(item) except Exception as e: exc = BotoConnectionIssue(str(e), 'acm', account, region.name) self.slurp_exception( (self.index, account, region.name), exc, exception_map, source="{}-watcher".format(self.index)) return item_list, exception_map
def slurp(self): """ :returns: item_list - list of IAM Groups. :returns: exception_map - A dict where the keys are a tuple containing the location of the exception and the value is the actual exception """ self.prep_for_slurp() item_list = [] exception_map = {} from security_monkey.common.sts_connect import connect for account in self.accounts: all_users = [] try: iam_b3 = connect(account, 'iam_boto3') managed_policies = all_managed_policies(iam_b3) iam = connect(account, 'iam') marker = None while True: users_response = self.wrap_aws_rate_limited_call( iam.get_all_users, marker=marker) # build our iam user list all_users.extend(users_response.users) # ensure that we get every iam user if hasattr(users_response, 'marker'): marker = users_response.marker else: break except Exception as e: exc = BotoConnectionIssue(str(e), 'iamuser', account, None) self.slurp_exception((self.index, account, 'universal'), exc, exception_map) continue for user in all_users: if self.check_ignore_list(user.user_name): continue item_config = { 'user': {}, 'userpolicies': {}, 'accesskeys': {}, 'mfadevices': {}, 'signingcerts': {} } app.logger.debug("Slurping %s (%s) from %s" % (self.i_am_singular, user.user_name, account)) item_config['user'] = dict(user) if managed_policies.has_key(user.arn): item_config['managed_policies'] = managed_policies.get( user.arn) ### USER POLICIES ### policy_names = self.policy_names_for_user(iam, user) for policy_name in policy_names: policy_document = self.wrap_aws_rate_limited_call( iam.get_user_policy, user.user_name, policy_name) policy_document = policy_document.policy_document policy = urllib.unquote(policy_document) try: policydict = json.loads(policy) except: exc = InvalidAWSJSON(policy) self.slurp_exception( (self.index, account, 'universal', user.user_name), exc, exception_map) item_config['userpolicies'][policy_name] = dict(policydict) ### ACCESS KEYS ### access_keys = self.access_keys_for_user(iam, user) for key in access_keys: item_config['accesskeys'][key.access_key_id] = dict(key) ### Multi Factor Authentication Devices ### mfas = self.mfas_for_user(iam, user) for mfa in mfas: item_config['mfadevices'][mfa.serial_number] = dict(mfa) ### LOGIN PROFILE ### login_profile = 'undefined' try: login_profile = self.wrap_aws_rate_limited_call( iam.get_login_profiles, user.user_name) login_profile = login_profile.login_profile item_config['loginprofile'] = dict(login_profile) except: pass ### SIGNING CERTIFICATES ### certificates = self.certificates_for_user(iam, user) for cert in certificates: _cert = dict(cert) del _cert['certificate_body'] item_config['signingcerts'][cert.certificate_id] = dict( _cert) item_list.append( IAMUserItem(account=account, name=user.user_name, config=item_config)) return item_list, exception_map
def slurp(self): """ :returns: item_list - list of RDS DB Clusters. :returns: exception_map - A dict where the keys are a tuple containing the location of the exception and the value is the actual exception """ self.prep_for_slurp() item_list = [] exception_map = {} from security_monkey.common.sts_connect import connect for account in self.accounts: for region in regions(): app.logger.debug("Checking {}/{}/{}".format( self.index, account, region.name)) clusters = [] try: rds = connect(account, 'boto3.rds.client', region=region) marker = None while True: if marker: response = self.wrap_aws_rate_limited_call( rds.describe_db_clusters, Marker=marker) else: response = self.wrap_aws_rate_limited_call( rds.describe_db_clusters) clusters.extend(response.get('DBClusters', [])) if response.get('Marker'): marker = response.get('Marker') else: break except Exception as e: if region.name not in TROUBLE_REGIONS: exc = BotoConnectionIssue(str(e), self.index, account, region.name) self.slurp_exception( (self.index, account, region.name), exc, exception_map) continue app.logger.debug("Found {} {}".format(len(clusters), self.i_am_plural)) for cluster in clusters: name = cluster.get('DBClusterIdentifier') if self.check_ignore_list(name): continue item_config = { 'name': name, 'allocated_storage': cluster.get('AllocatedStorage'), 'availability_zones': cluster.get('AvailabilityZones'), 'backup_retention_period': cluster.get('BackupRetentionPeriod'), 'character_set_name': cluster.get('CharacterSetName'), 'database_name': cluster.get('DatabaseName'), 'db_cluster_parameter_group': cluster.get('DBClusterParameterGroup'), 'db_subnet_group': cluster.get('DBSubnetGroup'), 'status': cluster.get('Status'), 'percent_progress': cluster.get('PercentProgress'), 'earliest_restorable_time': str(cluster.get('EarliestRestorableTime')), 'endpoint': cluster.get('Endpoint'), 'engine': cluster.get('Engine'), 'engine_version': cluster.get('EngineVersion'), 'latest_restorable_time': str(cluster.get('LatestRestorableTime')), 'port': cluster.get('Port'), 'master_username': cluster.get('MasterUsername'), 'db_cluster_option_group_memberships': cluster.get('DBClusterOptionGroupMemberships', []), 'preferred_backup_window': cluster.get('PreferredBackupWindow'), 'preferred_maintenance_window': cluster.get('PreferredMaintenanceWindow'), 'db_cluster_members': cluster.get('DBClusterMembers', []), 'vpc_security_groups': cluster.get('VpcSecurityGroups', []), 'hosted_zoneId': cluster.get('HostedZoneId'), 'storage_encrypted': cluster.get('StorageEncrypted', False), 'kms_key_id': cluster.get('KmsKeyId'), 'db_cluster_resourceId': cluster.get('DbClusterResourceId'), 'arn': cluster.get('DBClusterArn') } item = RDSClusterItem(region=region.name, account=account, name=name, arn=cluster.get('DBClusterArn'), config=item_config) item_list.append(item) return item_list, exception_map
def slurp(self): """ :returns: item_list - list of Flow Logs in use by account :returns: exception_map - A dict where the keys are a tuple containing the location of the exception and the value is the actual exception """ self.prep_for_slurp() from security_monkey.common.sts_connect import connect item_list = [] exception_map = {} for account in self.accounts: for region in regions(): app.logger.debug("Checking {}/{}/{}".format( self.index, account, region.name)) response_items = [] try: conn = connect(account, 'boto3.ec2.client', region=region) next_token = None while True: if next_token: response = self.wrap_aws_rate_limited_call( conn.describe_flow_logs, NextToken=next_token) else: response = self.wrap_aws_rate_limited_call( conn.describe_flow_logs) response_items.extend(response.get('FlowLogs')) if response.get('NextToken'): next_token = response.get('NextToken') else: break except Exception as e: if region.name not in TROUBLE_REGIONS: exc = BotoConnectionIssue(str(e), self.index, account, region.name) self.slurp_exception( (self.index, account, region.name), exc, exception_map) continue app.logger.debug("Found {} {}".format(len(response_items), self.i_am_plural)) for response_item in response_items: name = "{} ({})".format(response_item.get('LogGroupName'), response_item.get('FlowLogId')) if self.check_ignore_list(name): continue config = { 'creation_time': str(response_item.get('CreationTime')), 'deliver_logs_permission_arn': response_item.get('DeliverLogsPermissionArn'), 'flow_log_id': response_item.get('FlowLogId'), 'flow_log_status': response_item.get('FlowLogStatus'), 'log_group_name': response_item.get('LogGroupName'), 'resource_id': response_item.get('ResourceId'), 'traffic_type': response_item.get('TrafficType') } item = FlowLogItem(region=region.name, account=account, name=name, config=dict(config), source_watcher=self) item_list.append(item) return item_list, exception_map
def slurp(self): """ :returns: item_list - list of RDS Security Groups. :returns: exception_map - A dict where the keys are a tuple containing the location of the exception and the value is the actual exception """ self.prep_for_slurp() item_list = [] exception_map = {} from security_monkey.common.sts_connect import connect for account in self.accounts: for region in regions(): app.logger.debug("Checking {}/{}/{}".format( self.index, account, region.name)) sgs = [] try: rds = connect(account, 'rds', region=region) marker = None while True: response = self.wrap_aws_rate_limited_call( rds.get_all_dbsecurity_groups, marker=marker) sgs.extend(response) if response.marker: marker = response.marker else: break except Exception as e: if region.name not in TROUBLE_REGIONS: exc = BotoConnectionIssue(str(e), self.index, account, region.name) self.slurp_exception( (self.index, account, region.name), exc, exception_map) continue app.logger.debug("Found {} {}".format(len(sgs), self.i_am_plural)) for sg in sgs: if self.check_ignore_list(sg.name): continue name = sg.name vpc_id = None if hasattr(sg, 'VpcId'): vpc_id = sg.VpcId name = "{} (in {})".format(sg.name, vpc_id) item_config = { "name": sg.name, "description": sg.description, "owner_id": sg.owner_id, "region": region.name, "ec2_groups": [], "ip_ranges": [], "vpc_id": vpc_id } for ipr in sg.ip_ranges: ipr_config = { "cidr_ip": ipr.cidr_ip, "status": ipr.status, } item_config["ip_ranges"].append(ipr_config) item_config["ip_ranges"] = sorted(item_config["ip_ranges"]) for ec2_sg in sg.ec2_groups: ec2sg_config = { "name": ec2_sg.name, "owner_id": ec2_sg.owner_id, "Status": ec2_sg.Status, } item_config["ec2_groups"].append(ec2sg_config) item_config["ec2_groups"] = sorted( item_config["ec2_groups"]) item = RDSSecurityGroupItem(region=region.name, account=account, name=name, config=item_config) item_list.append(item) return item_list, exception_map