def test_Account(self): json_blob = {u'id': 111111111111, u'name': u'prod'} account = Account(None, json_blob) assert_equal(111111111111, account.local_id) assert_equal("prod", account.name) assert_equal("account", account.node_type) assert_equal("arn:aws:::111111111111:", account.arn) assert_false(account.isLeaf) assert_equal("prod", get_name(json_blob, "name")) assert_false(account.has_leaves) assert_equal([], account.leaves) assert_equal({'data': {'node_data': {u'id': 111111111111, u'name': u'prod'}, 'local_id': 111111111111, 'type': 'account', 'id': 'arn:aws:::111111111111:', 'name': u'prod'}}, account.cytoscape_data())
def test_get_ec2s(self): # This actually uses the demo data files provided json_blob = {u'id': 111111111111, u'name': u'demo'} account = Account(None, json_blob) region = Region(account, {"Endpoint": "ec2.us-east-1.amazonaws.com", "RegionName": "us-east-1"}) vpc = Vpc(region, get_vpcs(region, {})[0]) subnet = Subnet(vpc, {"SubnetId": "subnet-00000001", "CidrBlock": "10.0.0.0/24", "Tags": [{"Value": "Public a1", "Key": "Name"}]})
def get_cidrs_for_account(account, cidrs): account = Account(None, account) # TODO Need to use CloudMapper's prepare to identify trusted IPs that are actually in use. for region_json in get_regions(account): region = Region(account, region_json) sg_json = query_aws(account, "ec2-describe-security-groups", region) sgs = pyjq.all('.SecurityGroups[]', sg_json) for sg in sgs: cidrs_seen = set() cidr_and_name_list = pyjq.all('.IpPermissions[].IpRanges[]|[.CidrIp,.Description]', sg) for cidr, name in cidr_and_name_list: if not is_external_cidr(cidr): continue if is_unneeded_cidr(cidr): print('WARNING: Unneeded cidr used {} in {}'.format(cidr, sg['GroupId'])) continue for cidr_seen in cidrs_seen: if (IPNetwork(cidr_seen) in IPNetwork(cidr) or IPNetwork(cidr) in IPNetwork(cidr_seen)): print('WARNING: Overlapping CIDRs in {}, {} and {}'.format(sg['GroupId'], cidr, cidr_seen)) cidrs_seen.add(cidr) if cidr.startswith('0.0.0.0') and not cidr.endswith('/0'): print('WARNING: Unexpected CIDR for attempted public access {} in {}'.format(cidr, sg['GroupId'])) continue if cidr == '0.0.0.0/0': continue cidrs[cidr] = cidrs.get(cidr, set()) if name is not None: cidrs[cidr].add(name)
def get_account_stats(account): """Returns stats for an account""" account = Account(None, account) log_debug('Collecting stats in account {} ({})'.format(account.name, account.local_id)) # Init stats to {} stats = OrderedDict() for k in resources: stats[k] = {} for region_json in get_regions(account): region = Region(account, region_json) for key, resource in resources.items(): # Skip global services (just CloudFront) if ('region' in resource) and (resource['region'] != region.name): continue # Check exceptions that require special code to perform the count if key == 'route53_record': path = 'account-data/{}/{}/{}'.format( account.name, region.name, 'route53-list-resource-record-sets') if os.path.isdir(path): stats[key][region.name] = 0 for f in listdir(path): json_data = json.load(open(os.path.join(path, urllib.parse.quote_plus(f)))) stats[key][region.name] += sum(pyjq.all('.ResourceRecordSets|length', json_data)) else: # Normal path stats[key][region.name] = sum(pyjq.all(resource['query'], query_aws(region.account, resource['source'], region))) return stats
def api_endpoints(accounts, config): for account in accounts: account = Account(None, account) for region_json in get_regions(account): region = Region(account, region_json) # Look for API Gateway json_blob = query_aws(region.account, 'apigateway-get-rest-apis', region) if json_blob is None: continue for api in json_blob.get('items', []): rest_id = api['id'] deployments = get_parameter_file(region, 'apigateway', 'get-deployments', rest_id) if deployments is None: continue for deployment in deployments['items']: deployment_id = deployment['id'] stages = get_parameter_file(region, 'apigateway', 'get-stages', rest_id) if stages is None: continue for stage in stages['item']: if stage['deploymentId'] == deployment_id: resources = get_parameter_file( region, 'apigateway', 'get-resources', rest_id) if resources is None: continue for resource in resources['items']: print('{}.execute-api.{}.amazonaws.com/{}{}'. format(api['id'], region.name, stage['stageName'], resource['path']))
def audit(accounts, config): """Audit the accounts""" for account in accounts: account = Account(None, account) print('Finding resources in account {} ({})'.format( account.name, account.local_id)) for region_json in get_regions(account): region = Region(account, region_json) if region.name == 'us-east-1': audit_s3_buckets(region) audit_cloudtrail(region) audit_password_policy(region) audit_root_user(region) audit_users(region) audit_route53(region) audit_cloudfront(region) audit_ebs_snapshots(region) audit_rds_snapshots(region) audit_rds(region) audit_amis(region) audit_ecr_repos(region) audit_redshift(region) audit_es(region) audit_ec2(region) audit_elb(region) audit_sg(region) audit_lambda(region) audit_glacier(region) audit_kms(region) audit_sqs(region) audit_sns(region) audit_lightsail(region)
def test_get_vpcs(self): # This actually uses the demo data files provided json_blob = {u'id': 111111111111, u'name': u'demo'} account = Account(None, json_blob) region = Region(account, {"Endpoint": "ec2.us-east-1.amazonaws.com", "RegionName": "us-east-1"}) assert_equal([{"VpcId": "vpc-12345678", "Tags": [{"Value": "Prod", "Key": "Name"}], "InstanceTenancy": "default", "CidrBlockAssociationSet": [{"AssociationId": "vpc-cidr-assoc-12345678", "CidrBlock": "10.0.0.0/16", "CidrBlockState": {"State": "associated"}}], "State": "available", "DhcpOptionsId": "dopt-12345678", "CidrBlock": "10.0.0.0/16", "IsDefault": True}], get_vpcs(region, {}))
def get_account_stats(account, all_resources=False): """Returns stats for an account""" with open("stats_config.yaml", 'r') as f: resources = yaml.safe_load(f) account = Account(None, account) log_debug('Collecting stats in account {} ({})'.format(account.name, account.local_id)) stats = {} stats['keys'] = [] for resource in resources: # If the resource is marked as verbose, and we're not showing all resources, skip it. if resource.get('verbose',False) and not all_resources: continue stats['keys'].append(resource['name']) stats[resource['name']] = {} for region_json in get_regions(account): region = Region(account, region_json) for resource in resources: if resource.get('verbose',False) and not all_resources: continue # Skip global services (just CloudFront) if ('region' in resource) and (resource['region'] != region.name): continue # Normal path stats[resource['name']][region.name] = sum(pyjq.all(resource['query'], query_aws(region.account, resource['source'], region))) return stats
def find_admins(accounts, findings): admins = [] for account in accounts: account = Account(None, account) region = get_us_east_1(account) admins.extend(find_admins_in_account(region, findings)) return admins
def find_unused_resources(accounts): unused_resources = [] for account in accounts: unused_resources_for_account = [] for region_json in get_regions(Account(None, account)): region = Region(Account(None, account), region_json) unused_resources_for_region = {} add_if_exists( unused_resources_for_region, "security_groups", find_unused_security_groups(region), ) add_if_exists(unused_resources_for_region, "volumes", find_unused_volumes(region)) add_if_exists( unused_resources_for_region, "elastic_ips", find_unused_elastic_ips(region), ) add_if_exists( unused_resources_for_region, "network_interfaces", find_unused_network_interfaces(region), ) add_if_exists( unused_resources_for_region, "elastic_load_balancers", find_unused_elastic_load_balancers(region), ) unused_resources_for_account.append({ "region": region_json["RegionName"], "unused_resources": unused_resources_for_region, }) unused_resources.append({ "account": { "id": account["id"], "name": account["name"] }, "regions": unused_resources_for_account, }) return unused_resources
def amis(args, accounts, config): # Loading the list of public images from disk takes a while, so we'll iterate by region regions_file = 'data/aws/us-east-1/ec2-describe-images.json' if not os.path.isfile(regions_file): raise Exception( "You need to download the set of public AMI images. Run:\n" " mkdir -p data/aws\n" " cd data/aws\n" " aws ec2 describe-regions | jq -r '.Regions[].RegionName' | xargs -I{} mkdir {}\n" " aws ec2 describe-regions | jq -r '.Regions[].RegionName' | xargs -I{} sh -c 'aws --region {} ec2 describe-images --executable-users all > {}/ec2-describe-images.json'\n" ) print("{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}".format( 'Account Name', 'Region Name', 'Instance Id', 'Instance Name', 'AMI ID', 'Is Public', 'AMI Description', 'AMI Owner')) for region in listdir('data/aws/'): # Get public images public_images_file = 'data/aws/{}/ec2-describe-images.json'.format( region) public_images = json.load(open(public_images_file)) resource_filter = '.Images[]' public_images = pyjq.all(resource_filter, public_images) for account in accounts: account = Account(None, account) region = Region(account, {'RegionName': region}) instances = query_aws(account, "ec2-describe-instances", region) resource_filter = '.Reservations[].Instances[] | select(.State.Name == "running")' if args.instance_filter != '': resource_filter += '|{}'.format(args.instance_filter) instances = pyjq.all(resource_filter, instances) account_images = query_aws(account, "ec2-describe-images", region) resource_filter = '.Images[]' account_images = pyjq.all(resource_filter, account_images) for instance in instances: image_id = instance['ImageId'] image_description = '' owner = '' image, is_public_image = find_image(image_id, public_images, account_images) if image: # Many images don't have all fields, so try the Name, then Description, then ImageLocation image_description = image.get('Name', '') if image_description == '': image_description = image.get('Description', '') if image_description == '': image_description = image.get('ImageLocation', '') owner = image.get('OwnerId', '') print("{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}".format( account.name, region.name, instance['InstanceId'], get_instance_name(instance), image_id, is_public_image, image_description, owner))
def get_cidrs_for_account(account, cidrs): account = Account(None, account) for region_json in get_regions(account): region = Region(account, region_json) sg_json = query_aws(account, "ec2-describe-security-groups", region) sgs = pyjq.all(".SecurityGroups[]", sg_json) for sg in sgs: cidr_and_name_list = pyjq.all( ".IpPermissions[].IpRanges[]|[.CidrIp,.Description]", sg ) for cidr, name in cidr_and_name_list: if not is_external_cidr(cidr): continue if is_unblockable_cidr(cidr): print( "WARNING: Unneeded cidr used {} in {}".format( cidr, sg["GroupId"] ) ) continue if cidr.startswith("0.0.0.0") and not cidr.endswith("/0"): print( "WARNING: Unexpected CIDR for attempted public access {} in {}".format( cidr, sg["GroupId"] ) ) continue if cidr == "0.0.0.0/0": continue cidrs[cidr] = cidrs.get(cidr, set()) if name is not None: cidrs[cidr].add(name) for ip_permissions in sg["IpPermissions"]: cidrs_seen = set() for ip_ranges in ip_permissions["IpRanges"]: if "CidrIp" not in ip_ranges: continue cidr = ip_ranges["CidrIp"] for cidr_seen in cidrs_seen: if IPNetwork(cidr_seen) in IPNetwork(cidr) or IPNetwork( cidr ) in IPNetwork(cidr_seen): print( "WARNING: Overlapping CIDRs in {}, {} and {}".format( sg["GroupId"], cidr, cidr_seen ) ) cidrs_seen.add(cidr)
def get_collection_date(account): account_struct = Account(None, account) json_blob = query_aws(account_struct, "iam-get-credential-report", get_us_east_1(account_struct)) if not json_blob: raise Exception( "File iam-get-credential-report.json does not exist or is not well-formed. Likely cause is you did not run the collect command for this account." ) # GeneratedTime looks like "2019-01-30T15:43:24+00:00" # so extract the data part "2019-01-30" return json_blob["GeneratedTime"][:10]
def get_account_stats(account, all_resources=False): """Returns stats for an account""" with open("stats_config.yaml", 'r') as f: resources = yaml.safe_load(f) account = Account(None, account) log_debug('Collecting stats in account {} ({})'.format(account.name, account.local_id)) stats = {} stats['keys'] = [] for resource in resources: # If the resource is marked as verbose, and we're not showing all resources, skip it. if resource.get('verbose',False) and not all_resources: continue stats['keys'].append(resource['name']) stats[resource['name']] = {} for region_json in get_regions(account): region = Region(account, region_json) for resource in resources: if resource.get('verbose',False) and not all_resources: continue # Skip global services (just CloudFront) if ('region' in resource) and (resource['region'] != region.name): continue # S3 buckets require special code to identify their location if resource['name'] == 'S3 buckets': if region.name == 'us-east-1': buckets = pyjq.all('.Buckets[].Name', query_aws(region.account, 's3-list-buckets', region)) for bucket in buckets: # Get the bucket's location bucket_region = get_parameter_file(region, 's3', 'get-bucket-location', bucket)['LocationConstraint'] # Convert the value to a name. # See https://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region if bucket_region is None: bucket_region = 'us-east-1' elif bucket_region == 'EU': bucket_region = 'eu-west-1' # Increment the count tmp = stats[resource['name']].get(bucket_region, 0) stats[resource['name']][bucket_region] = tmp + 1 else: # Normal path stats[resource['name']][region.name] = sum(pyjq.all(resource['query'], query_aws(region.account, resource['source'], region))) return stats
def test_Account(self): json_blob = {u"id": 111111111111, u"name": u"prod"} account = Account(None, json_blob) assert_equal(111111111111, account.local_id) assert_equal("prod", account.name) assert_equal("account", account.node_type) assert_equal("arn:aws:::111111111111:", account.arn) assert_false(account.isLeaf) assert_equal("prod", get_name(json_blob, "name")) assert_false(account.has_leaves) assert_equal([], account.leaves) assert_equal( { "data": { "node_data": {"id": 111111111111, "name": "prod"}, "local_id": 111111111111, "type": "account", "id": "arn:aws:::111111111111:", "name": u"prod", } }, account.cytoscape_data(), )
def get_collection_date(account): if type(account) is not Account: account = Account(None, account) account_struct = account json_blob = query_aws( account_struct, "iam-get-credential-report", get_us_east_1(account_struct) ) if not json_blob: raise InvalidAccountData( "File iam-get-credential-report.json does not exist or is not well-formed. Likely cause is you did not run the collect command for account {}".format( account.name ) ) # GeneratedTime looks like "2019-01-30T15:43:24+00:00" return json_blob["GeneratedTime"]
def audit(accounts): findings = Findings() for account in accounts: account = Account(None, account) for region_json in get_regions(account): region = Region(account, region_json) try: if region.name == "us-east-1": audit_s3_buckets(findings, region) audit_cloudtrail(findings, region) audit_iam(findings, region) audit_password_policy(findings, region) audit_root_user(findings, region) audit_users(findings, region) audit_route53(findings, region) audit_cloudfront(findings, region) audit_s3_block_policy(findings, region) audit_guardduty(findings, region) audit_ebs_snapshots(findings, region) audit_rds_snapshots(findings, region) audit_rds(findings, region) audit_amis(findings, region) audit_ecr_repos(findings, region) audit_redshift(findings, region) audit_es(findings, region) audit_ec2(findings, region) audit_sg(findings, region) audit_lambda(findings, region) audit_glacier(findings, region) audit_kms(findings, region) audit_sqs(findings, region) audit_sns(findings, region) audit_lightsail(findings, region) except Exception as e: findings.add( Finding( region, "EXCEPTION", str(e), resource_details={ "exception": str(e), "traceback": str(traceback.format_exc()), }, )) return findings
def find_admins(accounts, args, findings): privs_to_look_for = None if "privs" in args and args.privs: privs_to_look_for = args.privs.split(",") include_restricted = False if "include_restricted" in args: include_restricted = args.include_restricted admins = [] for account in accounts: account = Account(None, account) region = get_us_east_1(account) admins.extend( find_admins_in_account(region, findings, privs_to_look_for, include_restricted)) return admins
def audit(accounts): findings = Findings() for account in accounts: account = Account(None, account) for region_json in get_regions(account): region = Region(account, region_json) try: if region.name == 'us-east-1': audit_s3_buckets(findings, region) audit_cloudtrail(findings, region) audit_iam_policies(findings, region) audit_password_policy(findings, region) audit_root_user(findings, region) audit_users(findings, region) audit_route53(findings, region) audit_cloudfront(findings, region) audit_s3_block_policy(findings, region) audit_guardduty(findings, region) audit_ebs_snapshots(findings, region) audit_rds_snapshots(findings, region) audit_rds(findings, region) audit_amis(findings, region) audit_ecr_repos(findings, region) audit_redshift(findings, region) audit_es(findings, region) audit_ec2(findings, region) audit_sg(findings, region) audit_lambda(findings, region) audit_glacier(findings, region) audit_kms(findings, region) audit_sqs(findings, region) audit_sns(findings, region) audit_lightsail(findings, region) except Exception as e: findings.add( Finding(region, 'EXCEPTION', e, resource_details={ 'exception': e, 'traceback': sys.exc_info() })) return findings
def audit(accounts, config): """Audit the accounts""" for account in accounts: account = Account(None, account) print('Finding resources in account {} ({})'.format( account.name, account.local_id)) for region_json in get_regions(account): region = Region(account, region_json) try: if region.name == 'us-east-1': audit_s3_buckets(region) audit_cloudtrail(region) audit_password_policy(region) audit_root_user(region) audit_users(region) audit_route53(region) audit_cloudfront(region) audit_s3_block_policy(region) audit_guardduty(region) audit_ebs_snapshots(region) audit_rds_snapshots(region) audit_rds(region) audit_amis(region) audit_ecr_repos(region) audit_redshift(region) audit_es(region) audit_ec2(region) audit_elb(region) audit_sg(region) audit_lambda(region) audit_glacier(region) audit_kms(region) audit_sqs(region) audit_sns(region) audit_lightsail(region) except Exception as e: print('Exception in {} in {}'.format(region.account.name, region.name), file=sys.stderr) traceback.print_exc()
def get_cidrs_for_account(account, cidrs): account = Account(None, account) # TODO Need to use CloudMapper's prepare to identify trusted IPs that are actually in use. for region_json in get_regions(account): region = Region(account, region_json) sg_json = query_aws(account, "ec2-describe-security-groups", region) sgs = pyjq.all('.SecurityGroups[]', sg_json) for sg in sgs: cidr_and_name_list = pyjq.all( '.IpPermissions[].IpRanges[]|[.CidrIp,.Description]', sg) for cidr, name in cidr_and_name_list: if not is_external_cidr(cidr): continue if is_unneeded_cidr(cidr): print('WARNING: Unneeded cidr used {}'.format(cidr)) continue if cidr == '0.0.0.0/0': continue cidrs[cidr] = cidrs.get(cidr, set()) if name is not None: cidrs[cidr].add(name)
def api_endpoints(accounts, config): for account in accounts: account = Account(None, account) for region_json in get_regions(account): region = Region(account, region_json) # Look for API Gateway json_blob = query_aws(region.account, "apigateway-get-rest-apis", region) if json_blob is None: continue for api in json_blob.get("items", []): rest_id = api["id"] deployments = get_parameter_file(region, "apigateway", "get-deployments", rest_id) if deployments is None: continue for deployment in deployments["items"]: deployment_id = deployment["id"] stages = get_parameter_file(region, "apigateway", "get-stages", rest_id) if stages is None: continue for stage in stages["item"]: if stage["deploymentId"] == deployment_id: resources = get_parameter_file( region, "apigateway", "get-resources", rest_id) if resources is None: continue for resource in resources["items"]: print("{}.execute-api.{}.amazonaws.com/{}{}". format( api["id"], region.name, stage["stageName"], resource["path"], ))
def test_get_ec2s(self): # This actually uses the demo data files provided json_blob = {u'id': 111111111111, u'name': u'demo'} account = Account(None, json_blob) region = Region(account, { "Endpoint": "ec2.us-east-1.amazonaws.com", "RegionName": "us-east-1" }) vpc = Vpc(region, get_vpcs(region, {})[0]) subnet = Subnet( vpc, { "SubnetId": "subnet-00000001", "CidrBlock": "10.0.0.0/24", "Tags": [{ "Value": "Public a1", "Key": "Name" }] }) instances_passed = get_ec2s(subnet, {"tags": ["Name=Bastion"]}) assert_equal(len(instances_passed), 1) instances_filtered = get_ec2s( subnet, {"tags": ["NonexistentTagName=NonexistentTagValue"]}) assert_equal(len(instances_filtered), 0)
def get_collection_date(account): account_struct = Account(None, account) json_blob = query_aws(account_struct, "iam-get-credential-report", get_us_east_1(account_struct)) # GeneratedTime looks like "2019-01-30T15:43:24+00:00" return json_blob['GeneratedTime'][:10]
def get_public_nodes(account, config, use_cache=False): # TODO Look for IPv6 also # TODO Look at more services from https://github.com/arkadiyt/aws_public_ips # TODO Integrate into something to more easily port scan and screenshot web services # Try reading from cache cache_file_path = "account-data/{}/public_nodes.json".format(account["name"]) if use_cache and os.path.isfile(cache_file_path): with open(cache_file_path) as f: return json.load(f), [] # Get the data from the `prepare` command outputfilter = { "internal_edges": False, "read_replicas": False, "inter_rds_edges": False, "azs": False, "collapse_by_tag": None, "collapse_asgs": True, "mute": True, } network = build_data_structure(account, config, outputfilter) public_nodes = [] warnings = [] # Look at all the edges for ones connected to the public Internet (0.0.0.0/0) for edge in pyjq.all( '.[].data|select(.type=="edge")|select(.source=="0.0.0.0/0")', network ): # Find the node at the other end of this edge target = {"arn": edge["target"], "account": account["name"]} target_node = pyjq.first( '.[].data|select(.id=="{}")'.format(target["arn"]), network, {} ) # Depending on the type of node, identify what the IP or hostname is if target_node["type"] == "elb": target["type"] = "elb" target["hostname"] = target_node["node_data"]["DNSName"] elif target_node["type"] == "elbv2": target["type"] = "elbv2" target["hostname"] = target_node["node_data"]["DNSName"] elif target_node["type"] == "autoscaling": target["type"] = "autoscaling" target["hostname"] = target_node["node_data"].get("PublicIpAddress", "") if target["hostname"] == "": target["hostname"] = target_node["node_data"]["PublicDnsName"] elif target_node["type"] == "rds": target["type"] = "rds" target["hostname"] = target_node["node_data"]["Endpoint"]["Address"] elif target_node["type"] == "ec2": target["type"] = "ec2" dns_name = target_node["node_data"].get("PublicDnsName", "") target["hostname"] = target_node["node_data"].get( "PublicIpAddress", dns_name ) target["tags"] = target_node["node_data"].get("Tags", []) elif target_node["type"] == "ecs": target["type"] = "ecs" target["hostname"] = "" for ip in target_node["node_data"]["ips"]: if is_public_ip(ip): target["hostname"] = ip elif target_node["type"] == "redshift": target["type"] = "redshift" target["hostname"] = ( target_node["node_data"].get("Endpoint", {}).get("Address", "") ) else: # Unknown node raise Exception("Unknown type: {}".format(target_node["type"])) # Check if any protocol is allowed (indicated by IpProtocol == -1) ingress = pyjq.all(".[]", edge.get("node_data", {})) sg_group_allowing_all_protocols = pyjq.first( '.[]|select(.IpPermissions[]?|.IpProtocol=="-1")|.GroupId', ingress, None ) public_sgs = {} if sg_group_allowing_all_protocols is not None: warnings.append( "All protocols allowed access to {} due to {}".format( target, sg_group_allowing_all_protocols ) ) # I would need to redo this code in order to get the name of the security group public_sgs[sg_group_allowing_all_protocols] = {"public_ports": "0-65535"} # from_port and to_port mean the beginning and end of a port range # We only care about TCP (6) and UDP (17) # For more info see https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/security-group-rules-reference.html port_ranges = [] for sg in ingress: sg_port_ranges = [] for ip_permission in sg.get("IpPermissions", []): selection = 'select((.IpProtocol=="tcp") or (.IpProtocol=="udp")) | select(.IpRanges[].CidrIp=="0.0.0.0/0")' sg_port_ranges.extend( pyjq.all("{}| [.FromPort,.ToPort]".format(selection), ip_permission) ) selection = 'select(.IpProtocol=="-1") | select(.IpRanges[].CidrIp=="0.0.0.0/0")' sg_port_ranges.extend( pyjq.all("{}| [0,65535]".format(selection), ip_permission) ) public_sgs[sg["GroupId"]] = { "GroupId": sg["GroupId"], "GroupName": sg["GroupName"], "public_ports": port_ranges_string(regroup_ranges(sg_port_ranges)), } port_ranges.extend(sg_port_ranges) range_string = port_ranges_string(regroup_ranges(port_ranges)) target["ports"] = range_string target["public_sgs"] = public_sgs if target["ports"] == "": issue_msg = "No ports open for tcp or udp (probably can only be pinged). Rules that are not tcp or udp: {} -- {}" warnings.append( issue_msg.format( json.dumps( pyjq.all( '.[]|select((.IpProtocol!="tcp") and (.IpProtocol!="udp"))'.format( selection ), ingress, ) ), account, ) ) public_nodes.append(target) # For the network diagram, if an ELB has availability across 3 subnets, I put one node in each subnet. # We don't care about that when we want to know what is public and it makes it confusing when you # see 3 resources with the same hostname, when you view your environment as only having one ELB. # This same issue exists for RDS. # Reduce these to single nodes. reduced_nodes = {} for node in public_nodes: reduced_nodes[node["hostname"]] = node public_nodes = [] for _, node in reduced_nodes.items(): public_nodes.append(node) account = Account(None, account) for region_json in get_regions(account): region = Region(account, region_json) # Look for CloudFront if region.name == "us-east-1": json_blob = query_aws( region.account, "cloudfront-list-distributions", region ) for distribution in json_blob.get("DistributionList", {}).get("Items", []): if not distribution["Enabled"]: continue target = {"arn": distribution["ARN"], "account": account.name} target["type"] = "cloudfront" target["hostname"] = distribution["DomainName"] target["ports"] = "80,443" public_nodes.append(target) # Look for API Gateway json_blob = query_aws(region.account, "apigateway-get-rest-apis", region) if json_blob is not None: for api in json_blob.get("items", []): target = {"arn": api["id"], "account": account.name} target["type"] = "apigateway" target["hostname"] = "{}.execute-api.{}.amazonaws.com".format( api["id"], region.name ) target["ports"] = "80,443" public_nodes.append(target) # Write cache file with open(cache_file_path, "w") as f: f.write(json.dumps(public_nodes, indent=4, sort_keys=True)) return public_nodes, warnings
def audit(accounts): findings = Findings() custom_auditor = None commands_path = "private_commands" for importer, command_name, _ in pkgutil.iter_modules([commands_path]): if "custom_auditor" != command_name: continue full_package_name = "%s.%s" % (commands_path, command_name) custom_auditor = importlib.import_module(full_package_name) for name, method in inspect.getmembers(custom_auditor, inspect.isfunction): if name.startswith("custom_filter"): global custom_filter custom_filter = method for account in accounts: account = Account(None, account) for region_json in get_regions(account): region = Region(account, region_json) try: if region.name == "us-east-1": audit_s3_buckets(findings, region) audit_cloudtrail(findings, region) audit_iam(findings, region) audit_password_policy(findings, region) audit_root_user(findings, region) audit_users(findings, region) audit_route53(findings, region) audit_cloudfront(findings, region) audit_s3_block_policy(findings, region) audit_guardduty(findings, region) audit_ebs_snapshots(findings, region) audit_rds_snapshots(findings, region) audit_rds(findings, region) audit_amis(findings, region) audit_ecr_repos(findings, region) audit_redshift(findings, region) audit_es(findings, region) audit_ec2(findings, region) audit_sg(findings, region) audit_lambda(findings, region) audit_glacier(findings, region) audit_kms(findings, region) audit_sqs(findings, region) audit_sns(findings, region) audit_lightsail(findings, region) except Exception as e: findings.add( Finding( region, "EXCEPTION", str(e), resource_details={ "exception": str(e), "traceback": str(traceback.format_exc()), }, )) # Run custom auditor if it exists try: if custom_auditor is not None: for name, method in inspect.getmembers( custom_auditor, inspect.isfunction): if name.startswith("custom_audit_"): method(findings, region) except Exception as e: findings.add( Finding( region, "EXCEPTION", str(e), resource_details={ "exception": str(e), "traceback": str(traceback.format_exc()), }, )) return findings
def public(accounts, config): for account in accounts: # Get the data from the `prepare` command outputfilter = { 'internal_edges': False, 'read_replicas': False, 'inter_rds_edges': False, 'azs': False, 'collapse_by_tag': None, 'collapse_asgs': True, 'mute': True } network = build_data_structure(account, config, outputfilter) # Look at all the edges for ones connected to the public Internet (0.0.0.0/0) for edge in pyjq.all( '.[].data|select(.type=="edge")|select(.source=="0.0.0.0/0")', network): # Find the node at the other end of this edge target = {'arn': edge['target'], 'account': account['name']} target_node = pyjq.first( '.[].data|select(.id=="{}")'.format(target['arn']), network, {}) # Depending on the type of node, identify what the IP or hostname is if target_node['type'] == 'elb': target['type'] = 'elb' target['hostname'] = target_node['node_data']['DNSName'] elif target_node['type'] == 'autoscaling': target['type'] = 'autoscaling' target['hostname'] = target_node['node_data'].get( 'PublicIpAddress', '') if target['hostname'] == '': target['hostname'] = target_node['node_data'][ 'PublicDnsName'] elif target_node['type'] == 'rds': target['type'] = 'rds' target['hostname'] = target_node['node_data']['Endpoint'][ 'Address'] elif target_node['type'] == 'ec2': target['type'] = 'ec2' dns_name = target_node['node_data'].get('PublicDnsName', '') target['hostname'] = target_node['node_data'].get( 'PublicIpAddress', dns_name) else: print( pyjq.first( '.[].data|select(.id=="{}")|[.type, (.node_data|keys)]' .format(target['arn']), network, {})) # Check if any protocol is allowed (indicated by IpProtocol == -1) ingress = pyjq.all('.[]', edge.get('node_data', {})) sg_group_allowing_all_protocols = pyjq.first( 'select(.IpPermissions[]|.IpProtocol=="-1")|.GroupId', ingress, None) public_sgs = set() if sg_group_allowing_all_protocols is not None: log_warning( 'All protocols allowed access to {} due to {}'.format( target, sg_group_allowing_all_protocols)) range_string = '0-65535' public_sgs.add(sg_group_allowing_all_protocols) else: # from_port and to_port mean the beginning and end of a port range # We only care about TCP (6) and UDP (17) # For more info see https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/security-group-rules-reference.html port_ranges = [] for sg in ingress: for ip_permission in sg['IpPermissions']: selection = 'select((.IpProtocol=="tcp") or (.IpProtocol=="udp")) | select(.IpRanges[].CidrIp=="0.0.0.0/0")' port_ranges.extend( pyjq.all( '{}| [.FromPort,.ToPort]'.format(selection), ip_permission)) public_sgs.add(sg['GroupId']) range_string = port_ranges_string(regroup_ranges(port_ranges)) target['ports'] = range_string target['public_sgs'] = list(public_sgs) if target['ports'] == "": issue_msg = 'No ports open for tcp or udp (probably can only be pinged). Rules that are not tcp or udp: {} -- {}' log_warning( issue_msg.format( json.dumps( pyjq.all( '.[]|select((.IpProtocol!="tcp") and (.IpProtocol!="udp"))' .format(selection), ingress)), account)) print(json.dumps(target, indent=4, sort_keys=True)) account = Account(None, account) for region_json in get_regions(account): region = Region(account, region_json) # Look for CloudFront if region.name == 'us-east-1': json_blob = query_aws(region.account, 'cloudfront-list-distributions', region) for distribution in json_blob.get('DistributionList', {}).get('Items', []): if not distribution['Enabled']: continue target = { 'arn': distribution['ARN'], 'account': account.name } target['type'] = 'cloudfront' target['hostname'] = distribution['DomainName'] target['ports'] = '80,443' print(json.dumps(target, indent=4, sort_keys=True)) # Look for API Gateway json_blob = query_aws(region.account, 'apigateway-get-rest-apis', region) for api in json_blob.get('items', []): target = {'arn': api['id'], 'account': account.name} target['type'] = 'apigateway' target['hostname'] = '{}.execute-api.{}.amazonaws.com'.format( api['id'], region.name) target['ports'] = '80,443' print(json.dumps(target, indent=4, sort_keys=True))
def report(accounts, config, args): """Create report""" # Create directory for output file if it doesn't already exists try: os.mkdir(os.path.dirname(REPORT_OUTPUT_FILE)) except OSError: # Already exists pass # Read template with open(os.path.join("templates", "report.html"), "r") as report_template: template = Template(report_template.read()) # Data to be passed to the template t = {} t["version"] = __version__ # Get account names and id's t["accounts"] = [] for account in accounts: t["accounts"].append( { "name": account["name"], "id": account["id"], "collection_date": get_collection_date(account)[:10], } ) # Get resource count info # Collect counts account_stats = {} print("* Getting resource counts") for account in accounts: account_stats[account["name"]] = get_account_stats( account, args.stats_all_resources ) print(" - {}".format(account["name"])) # Get names of resources # TODO: Change the structure passed through here to be a dict of dict's like I do for the regions t["resource_names"] = [""] # Just look at the resource names of the first account as they are all the same first_account = list(account_stats.keys())[0] for name in account_stats[first_account]["keys"]: t["resource_names"].append(name) # Create jinja data for the resource stats per account t["resource_stats"] = [] for account in accounts: for resource_name in t["resource_names"]: if resource_name == "": resource_row = [account["name"]] else: count = sum(account_stats[account["name"]][resource_name].values()) resource_row.append(count) t["resource_stats"].append(resource_row) t["resource_names"].pop(0) # Get region names t["region_names"] = [] account = accounts[0] account = Account(None, account) for region in get_regions(account): region = Region(account, region) t["region_names"].append(region.name) # Get stats for the regions region_stats = {} region_stats_tooltip = {} for account in accounts: account = Account(None, account) region_stats[account.name] = {} region_stats_tooltip[account.name] = {} for region in get_regions(account): region = Region(account, region) count = 0 for resource_name in t["resource_names"]: n = account_stats[account.name][resource_name].get(region.name, 0) count += n if n > 0: if region.name not in region_stats_tooltip[account.name]: region_stats_tooltip[account.name][region.name] = "" region_stats_tooltip[account.name][ region.name ] += "{}:{}<br>".format(resource_name, n) if count > 0: has_resources = "Y" else: has_resources = "N" region_stats[account.name][region.name] = has_resources t["region_stats"] = region_stats t["region_stats_tooltip"] = region_stats_tooltip # Pass the account names t["account_names"] = [] for a in accounts: t["account_names"].append(a["name"]) t["resource_data_set"] = [] # Pass data for the resource chart color_index = 0 for resource_name in t["resource_names"]: resource_counts = [] for account_name in t["account_names"]: resource_counts.append( sum(account_stats[account_name][resource_name].values()) ) resource_data = { "label": resource_name, "data": resource_counts, "backgroundColor": COLOR_PALETTE[color_index], "borderWidth": 1, } t["resource_data_set"].append(resource_data) color_index = (color_index + 1) % len(COLOR_PALETTE) # Get IAM access dat print("* Getting IAM data") t["iam_active_data_set"] = [ { "label": "Active users", "stack": "users", "data": [], "backgroundColor": "rgb(162, 203, 249)", "borderWidth": 1, }, { "label": "Inactive users", "stack": "users", "data": [], "backgroundColor": INACTIVE_COLOR, "borderWidth": 1, }, { "label": "Active roles", "stack": "roles", "data": [], "backgroundColor": ACTIVE_COLOR, "borderWidth": 1, }, { "label": "Inactive roles", "stack": "roles", "data": [], "backgroundColor": INACTIVE_COLOR, "borderWidth": 1, }, ] for account in accounts: account = Account(None, account) print(" - {}".format(account.name)) account_stats = get_access_advisor_active_counts(account, args.max_age) # Add to dataset t["iam_active_data_set"][0]["data"].append(account_stats["users"]["active"]) t["iam_active_data_set"][1]["data"].append(account_stats["users"]["inactive"]) t["iam_active_data_set"][2]["data"].append(account_stats["roles"]["active"]) t["iam_active_data_set"][3]["data"].append(account_stats["roles"]["inactive"]) print("* Getting public resource data") # TODO Need to cache this data as this can take a long time t["public_network_resource_type_names"] = [ "ec2", "elb", "elbv2", "rds", "redshift", "ecs", "autoscaling", "cloudfront", "apigateway", ] t["public_network_resource_types"] = {} t["public_ports"] = [] t["account_public_ports"] = {} for account in accounts: print(" - {}".format(account["name"])) t["public_network_resource_types"][account["name"]] = {} t["account_public_ports"][account["name"]] = {} for type_name in t["public_network_resource_type_names"]: t["public_network_resource_types"][account["name"]][type_name] = 0 public_nodes, _ = get_public_nodes(account, config, use_cache=True) for public_node in public_nodes: if public_node["type"] in t["public_network_resource_type_names"]: t["public_network_resource_types"][account["name"]][ public_node["type"] ] += 1 else: raise Exception( "Unknown type {} of public node".format(public_node["type"]) ) if public_node["ports"] not in t["public_ports"]: t["public_ports"].append(public_node["ports"]) t["account_public_ports"][account["name"]][public_node["ports"]] = ( t["account_public_ports"][account["name"]].get(public_node["ports"], 0) + 1 ) # Pass data for the public port chart t["public_ports_data_set"] = [] color_index = 0 for ports in t["public_ports"]: port_counts = [] for account_name in t["account_names"]: port_counts.append(t["account_public_ports"][account_name].get(ports, 0)) # Fix the port range name for '' when ICMP is being allowed if ports == "": ports = "ICMP only" port_data = { "label": ports, "data": port_counts, "backgroundColor": COLOR_PALETTE[color_index], "borderWidth": 1, } t["public_ports_data_set"].append(port_data) color_index = (color_index + 1) % len(COLOR_PALETTE) print("* Auditing accounts") findings = audit(accounts) audit_config = load_audit_config() # Filter findings tmp_findings = [] for finding in findings: conf = audit_config[finding.issue_id] if finding_is_filtered(finding, conf, minimum_severity=args.minimum_severity): continue tmp_findings.append(finding) findings = tmp_findings t["findings_severity_by_account_chart"] = [] # Figure out the counts of findings for each account # Create chart for finding type counts findings_severity_by_account = {} for account in accounts: findings_severity_by_account[account["name"]] = {} for severity in SEVERITIES: findings_severity_by_account[account["name"]][severity["name"]] = {} # Filtering the list of findings down to the ones specific to the current account. for finding in [f for f in findings if f.account_name == account["name"]]: conf = audit_config[finding.issue_id] count = findings_severity_by_account[finding.account_name][ conf["severity"] ].get(finding.issue_id, 0) findings_severity_by_account[finding.account_name][conf["severity"]][ finding.issue_id ] = (count + 1) t["findings_severity_by_account_chart"] = [] for severity in SEVERITIES: severity_counts_by_account = [] for _ in accounts: severity_counts_by_account.append( len( findings_severity_by_account[finding.account_name][severity["name"]] ) ) t["findings_severity_by_account_chart"].append( { "label": severity["name"], "data": severity_counts_by_account, "backgroundColor": severity["color"], "borderWidth": 1, } ) # Create list by severity t["severities"] = {} for severity in SEVERITIES: t["severities"][severity["name"]] = {} for finding in findings: conf = audit_config[finding.issue_id] t["severities"][conf["severity"]][finding.issue_id] = { "title": conf["title"], "id": finding.issue_id, } # Create chart for finding counts finding_type_set = {} for f in findings: finding_type_set[f.issue_id] = 1 t["finding_counts_by_account_chart"] = [] for finding_type in finding_type_set: finding_counts = [] for account in accounts: count = 0 for severity in findings_severity_by_account[account["name"]]: count += findings_severity_by_account[account["name"]][severity].get( finding_type, 0 ) finding_counts.append(count) t["finding_counts_by_account_chart"].append( { "label": finding_type, "data": finding_counts, "backgroundColor": COLOR_PALETTE[color_index], "borderWidth": 1, } ) color_index = (color_index + 1) % len(COLOR_PALETTE) t["findings"] = {} for finding in findings: conf = audit_config[finding.issue_id] group = t["findings"].get(conf["group"], {}) # Get the severity struct for severity in SEVERITIES: if severity["name"] == conf["severity"]: break issue = group.get( finding.issue_id, { "title": conf["title"], "description": conf.get("description", ""), "severity": conf["severity"], "severity_color": severity["color"], "is_global": conf.get("is_global", False), "accounts": {}, }, ) account_hits = issue["accounts"].get( finding.region.account.local_id, {"account_name": finding.region.account.name, "regions": {}}, ) region_hits = account_hits["regions"].get(finding.region.name, {"hits": []}) region_hits["hits"].append( { "resource": finding.resource_id, "details": json.dumps(finding.resource_details, indent=4), } ) account_hits["regions"][finding.region.name] = region_hits issue["accounts"][finding.region.account.local_id] = account_hits group[finding.issue_id] = issue t["findings"][conf["group"]] = group # Generate report from template with open(REPORT_OUTPUT_FILE, "w") as f: f.write(template.render(t=t)) print("Report written to {}".format(REPORT_OUTPUT_FILE))
def build_data_structure(account_data, config, outputfilter): cytoscape_json = [] account = Account(None, account_data) print("Building data for account {} ({})".format(account.name, account.local_id)) cytoscape_json.append(account.cytoscape_data()) for region_json in get_regions(account, outputfilter): node_count_per_region = 0 region = Region(account, region_json) for vpc_json in get_vpcs(region, outputfilter): vpc = Vpc(region, vpc_json) for az_json in get_azs(vpc): # Availibility zones are not a per VPC construct, but VPC's can span AZ's, # so I make VPC a higher level construct az = Az(vpc, az_json) for subnet_json in get_subnets(az): # If we ignore AZz, then tie the subnets up the VPC as the parent if outputfilter["azs"]: parent = az else: parent = vpc subnet = Subnet(parent, subnet_json) # Get EC2's for ec2_json in get_ec2s(subnet): ec2 = Ec2(subnet, ec2_json, outputfilter["collapse_by_tag"]) subnet.addChild(ec2) # Get RDS's for rds_json in get_rds_instances(subnet): rds = Rds(subnet, rds_json) if not outputfilter["read_replicas"] and rds.node_type == "rds_rr": continue subnet.addChild(rds) # Get ELB's for elb_json in get_elbs(subnet): elb = Elb(subnet, elb_json) subnet.addChild(elb) # If there are leaves, then add this subnet to the final graph if len(subnet.leaves) > 0: node_count_per_region += len(subnet.leaves) for leaf in subnet.leaves: cytoscape_json.append(leaf.cytoscape_data()) cytoscape_json.append(subnet.cytoscape_data()) az.addChild(subnet) if az.has_leaves: if outputfilter["azs"]: cytoscape_json.append(az.cytoscape_data()) vpc.addChild(az) if vpc.has_leaves: cytoscape_json.append(vpc.cytoscape_data()) region.addChild(vpc) if region.has_leaves: cytoscape_json.append(region.cytoscape_data()) account.addChild(region) print("- {} nodes built in region {}".format(node_count_per_region, region.local_id)) # Get VPC peerings for region in account.children: for vpc_peering in get_vpc_peerings(region): # For each peering, find the accepter and the requester accepter_id = vpc_peering["AccepterVpcInfo"]["VpcId"] requester_id = vpc_peering["RequesterVpcInfo"]["VpcId"] accepter = None requester = None for vpc in region.children: if accepter_id == vpc.local_id: accepter = vpc if requester_id == vpc.local_id: requester = vpc # If both have been found, add each as peers to one another if accepter and requester: accepter.addPeer(requester) requester.addPeer(accepter) # Get external cidr nodes cidrs = {} for cidr in get_external_cidrs(account, config): cidrs[cidr.arn] = cidr # Find connections between nodes # Only looking at Security Groups currently, which are a VPC level construct connections = {} for region in account.children: for vpc in region.children: for c, reasons in get_connections(cidrs, vpc, outputfilter).items(): r = connections.get(c, []) r.extend(reasons) connections[c] = r # Add external cidr nodes used_cidrs = 0 for _, cidr in cidrs.items(): if cidr.is_used: used_cidrs += 1 cytoscape_json.append(cidr.cytoscape_data()) print("- {} external CIDRs built".format(used_cidrs)) total_number_of_nodes = len(cytoscape_json) # Add the mapping to our graph for c, reasons in connections.items(): if c.source == c.target: # Ensure we don't add connections with the same nodes on either side continue c._json = reasons cytoscape_json.append(c.cytoscape_data()) print("- {} connections built".format(len(connections))) # Check if we have a lot of data, and if so, show a warning # Numbers chosen here are arbitrary MAX_NODES_FOR_WARNING = 200 MAX_EDGES_FOR_WARNING = 500 if total_number_of_nodes > MAX_NODES_FOR_WARNING or len(connections) > MAX_EDGES_FOR_WARNING: print("WARNING: There are {} total nodes and {} total edges.".format(total_number_of_nodes, len(connections))) print(" This will be difficult to display and may be too complex to make sense of.") print(" Consider reducing the number of items in the diagram by viewing a single") print(" region, ignoring internal edges, or other filtering.") return cytoscape_json
def build_data_structure(account_data, config, outputfilter): cytoscape_json = [] if outputfilter.get('mute', False): global MUTE MUTE = True account = Account(None, account_data) log("Building data for account {} ({})".format(account.name, account.local_id)) cytoscape_json.append(account.cytoscape_data()) # Iterate through each region and add all the VPCs, AZs, and Subnets for region_json in get_regions(account, outputfilter): nodes = {} region = Region(account, region_json) for vpc_json in get_vpcs(region, outputfilter): vpc = Vpc(region, vpc_json) for az_json in get_azs(vpc): # Availibility zones are not a per VPC construct, but VPC's can span AZ's, # so I make VPC a higher level construct az = Az(vpc, az_json) for subnet_json in get_subnets(az): # If we ignore AZz, then tie the subnets up the VPC as the parent if outputfilter["azs"]: parent = az else: parent = vpc subnet = Subnet(parent, subnet_json) az.addChild(subnet) vpc.addChild(az) region.addChild(vpc) account.addChild(region) # # In each region, iterate through all the resource types # # EC2 nodes for ec2_json in get_ec2s(region): node = Ec2(region, ec2_json, outputfilter["collapse_by_tag"], outputfilter["collapse_asgs"]) nodes[node.arn] = node # RDS nodes for rds_json in get_rds_instances(region): node = Rds(region, rds_json) if not outputfilter["read_replicas"] and node.node_type == "rds_rr": continue nodes[node.arn] = node # ELB nodes for elb_json in get_elbs(region): node = Elb(region, elb_json) nodes[node.arn] = node for elb_json in get_elbv2s(region): node = Elbv2(region, elb_json) nodes[node.arn] = node # PrivateLink and VPC Endpoints for vpc_endpoint_json in get_vpc_endpoints(region): node = VpcEndpoint(region, vpc_endpoint_json) nodes[node.arn] = node # ECS tasks for ecs_json in get_ecs_tasks(region): node = Ecs(region, ecs_json) nodes[node.arn] = node # Lambda functions for lambda_json in get_lambda_functions(region): node = Lambda(region, lambda_json) nodes[node.arn] = node # Redshift clusters for node_json in get_redshift(region): node = Redshift(region, node_json) nodes[node.arn] = node # ElasticSearch clusters for node_json in get_elasticsearch(region): node = ElasticSearch(region, node_json) nodes[node.arn] = node # Filter out nodes based on tags if len(outputfilter.get("tags", [])) > 0: for node_id in list(nodes): has_match = False node = nodes[node_id] # For each node, look to see if its tags match one of the tag sets # Ex. --tags Env=Prod --tags Team=Dev,Name=Bastion for tag_set in outputfilter.get("tags", []): conditions = [c.split("=") for c in tag_set.split(",")] condition_matches = 0 # For a tag set, see if all conditions match, ex. [["Team","Dev"],["Name","Bastion"]] for pair in conditions: # Given ["Team","Dev"], see if it matches one of the tags in the node for tag in node.tags: if tag.get('Key','') == pair[0] and tag.get('Value','') == pair[1]: condition_matches += 1 # We have a match if all of the conditions matched if condition_matches == len(conditions): has_match = True # If there were no matches, remove the node if not has_match: del nodes[node_id] # Add the nodes to their respective subnets for node_arn in list(nodes): node = nodes[node_arn] add_node_to_subnets(region, node, nodes) # From the root of the tree (the account), add in the children if there are leaves # If not, mark the item for removal if region.has_leaves: cytoscape_json.append(region.cytoscape_data()) region_children_to_remove = set() for vpc in region.children: if vpc.has_leaves: cytoscape_json.append(vpc.cytoscape_data()) vpc_children_to_remove = set() for vpc_child in vpc.children: if vpc_child.has_leaves: if outputfilter["azs"]: cytoscape_json.append(vpc_child.cytoscape_data()) elif vpc_child.node_type != 'az': # Add VPC children that are not AZs, such as Gateway endpoints cytoscape_json.append(vpc_child.cytoscape_data()) az_children_to_remove = set() for subnet in vpc_child.children: if subnet.has_leaves: cytoscape_json.append(subnet.cytoscape_data()) for leaf in subnet.leaves: cytoscape_json.append(leaf.cytoscape_data(subnet.arn)) else: az_children_to_remove.add(subnet) for subnet in az_children_to_remove: vpc_child.removeChild(subnet) else: vpc_children_to_remove.add(vpc_child) for az in vpc_children_to_remove: vpc.removeChild(az) else: region_children_to_remove.add(vpc) for vpc in region_children_to_remove: region.removeChild(vpc) log("- {} nodes built in region {}".format(len(nodes), region.local_id)) # Get VPC peerings for region in account.children: for vpc_peering in get_vpc_peerings(region): # For each peering, find the accepter and the requester accepter_id = vpc_peering["AccepterVpcInfo"]["VpcId"] requester_id = vpc_peering["RequesterVpcInfo"]["VpcId"] accepter = None requester = None for vpc in region.children: if accepter_id == vpc.local_id: accepter = vpc if requester_id == vpc.local_id: requester = vpc # If both have been found, add each as peers to one another if accepter and requester: accepter.addPeer(requester) requester.addPeer(accepter) # Get external cidr nodes cidrs = {} for cidr in get_external_cidrs(account, config): cidrs[cidr.arn] = cidr # Find connections between nodes # Only looking at Security Groups currently, which are a VPC level construct connections = {} for region in account.children: for vpc in region.children: for c, reasons in get_connections(cidrs, vpc, outputfilter).items(): r = connections.get(c, []) r.extend(reasons) connections[c] = r # # Collapse CIDRs # # Get a list of the current CIDRs current_cidrs = [] for cidr_string in cidrs: current_cidrs.append(cidr_string) # Iterate through them for cidr_string in current_cidrs: # Find CIDRs in the config that our CIDR falls inside # It may fall inside multiple ranges matching_known_cidrs = {} for named_cidr in config["cidrs"]: if IPNetwork(cidr_string) in IPNetwork(named_cidr): # Match found matching_known_cidrs[named_cidr] = IPNetwork(named_cidr).size if len(matching_known_cidrs) > 0: # A match was found. Find the smallest matching range. sorted_matches = sorted(matching_known_cidrs.items(), key=operator.itemgetter(1)) # Get first item to get (CIDR,size); and first item of that to get just the CIDR smallest_matched_cidr_string = sorted_matches[0][0] smallest_matched_cidr_name = config["cidrs"][smallest_matched_cidr_string]['name'] # Check if we have a CIDR node that doesn't match the smallest one possible. if cidrs[cidr_string].name != config["cidrs"][smallest_matched_cidr_string]['name']: # See if we need to create the larger known range if cidrs.get(smallest_matched_cidr_string, "") == "": cidrs[smallest_matched_cidr_string] = Cidr(smallest_matched_cidr_string, smallest_matched_cidr_name) # The existing CIDR node needs to be removed and rebuilt as the larger known range del cidrs[cidr_string] # Get the larger known range new_source = cidrs[smallest_matched_cidr_string] new_source.is_used = True # Find all the connections to the old node connections_to_remove = [] for c in connections: if c.source.node_type == 'ip' and c.source.arn == cidr_string: connections_to_remove.append(c) # Create new connections to the new node for c in connections_to_remove: r = connections[c] del connections[c] connections[Connection(new_source, c._target)] = r # Add external cidr nodes used_cidrs = 0 for _, cidr in cidrs.items(): if cidr.is_used: used_cidrs += 1 cytoscape_json.append(cidr.cytoscape_data()) log("- {} external CIDRs built".format(used_cidrs)) total_number_of_nodes = len(cytoscape_json) # Add the mapping to our graph for c, reasons in connections.items(): if c.source == c.target: # Ensure we don't add connections with the same nodes on either side continue c._json = reasons cytoscape_json.append(c.cytoscape_data()) log("- {} connections built".format(len(connections))) # Check if we have a lot of data, and if so, show a warning # Numbers chosen here are arbitrary MAX_NODES_FOR_WARNING = 200 MAX_EDGES_FOR_WARNING = 500 if total_number_of_nodes > MAX_NODES_FOR_WARNING or len(connections) > MAX_EDGES_FOR_WARNING: log("WARNING: There are {} total nodes and {} total edges.".format(total_number_of_nodes, len(connections))) log(" This will be difficult to display and may be too complex to make sense of.") log(" Consider reducing the number of items in the diagram by viewing a single") log(" region, ignoring internal edges, or other filtering.") return cytoscape_json