def describe_vpc(record): """Attempts to describe vpc ids.""" account_id = record['account'] vpc_name = cloudwatch.filter_request_parameters('vpcName', record) vpc_id = cloudwatch.filter_request_parameters('vpcId', record) try: if vpc_id and vpc_name: return describe_vpcs(account_number=account_id, assume_role=HISTORICAL_ROLE, region=CURRENT_REGION, Filters=[{ 'Name': 'vpc-id', 'Values': [vpc_id] }]) elif vpc_id: return describe_vpcs(account_number=account_id, assume_role=HISTORICAL_ROLE, region=CURRENT_REGION, VpcIds=[vpc_id]) else: raise Exception('[X] Describe requires VpcId.') except ClientError as e: if e.response['Error']['Code'] == 'InvalidVpc.NotFound': return [] raise e
def handler(event, context): """ Historical VPC Poller. This Poller is run at a set interval in order to ensure that changes do not go undetected by historical. Historical Pollers generate `polling events` which simulate changes. These polling events contain configuration data such as the account/region defining where the collector should attempt to gather data from. """ log.debug('Running poller. Configuration: {}'.format(event)) queue_url = get_queue_url( os.environ.get('POLLER_QUEUE_NAME', 'HistoricalVPCPoller')) for account in get_historical_accounts(): for region in POLL_REGIONS: try: vpcs = describe_vpcs(account_number=account['id'], assume_role=HISTORICAL_ROLE, region=region) events = [ vpc_polling_schema.serialize(account['id'], v) for v in vpcs ] produce_events(events, queue_url) log.debug('Finished generating polling events. Account: {}/{} ' 'Events Created: {}'.format(account['id'], region, len(events))) except ClientError as e: log.warning( 'Unable to generate events for account/region. Account Id/Region: {account_id}/{region}' ' Reason: {reason}'.format(account_id=account['id'], region=region, reason=e))
def handler(event, context): """ Historical security group event poller. This poller is run at a set interval in order to ensure that changes do not go undetected by historical. Historical pollers generate `polling events` which simulate changes. These polling events contain configuration data such as the account/region defining where the collector should attempt to gather data from. """ log.debug('Running poller. Configuration: {}'.format(event)) for account in get_historical_accounts(): try: vpcs = describe_vpcs(account_number=account['id'], assume_role=HISTORICAL_ROLE, region=CURRENT_REGION) events = [ vpc_polling_schema.serialize(account['id'], v) for v in vpcs['Vpcs'] ] produce_events( events, os.environ.get('HISTORICAL_STREAM', 'HistoricalVPCPollerStream')) log.debug( 'Finished generating polling events. Account: {} Events Created: {}' .format(account['id'], len(events))) except ClientError as e: log.warning( 'Unable to generate events for account. AccountId: {account_id} Reason: {reason}' .format(account_id=account['id'], reason=e))
def poller_processor_handler(event, context): # pylint: disable=W0613 """ Historical Security Group Poller Processor. This will receive events from the Poller Tasker, and will list all objects of a given technology for an account/region pair. This will generate `polling events` which simulate changes. These polling events contain configuration data such as the account/region defining where the collector should attempt to gather data from. """ LOG.debug('[@] Running Poller...') queue_url = get_queue_url(os.environ.get('POLLER_QUEUE_NAME', 'HistoricalVPCPoller')) records = deserialize_records(event['Records']) for record in records: # Skip accounts that have role assumption errors: try: vpcs = describe_vpcs( account_number=record['account_id'], assume_role=HISTORICAL_ROLE, region=record['region'] ) events = [VPC_POLLING_SCHEMA.serialize(record['account_id'], v) for v in vpcs] produce_events(events, queue_url, randomize_delay=RANDOMIZE_POLLER) LOG.debug(f"[@] Finished generating polling events. Account: {record['account_id']}/{record['region']} " f"Events Created: {len(events)}") except ClientError as exc: LOG.error(f"[X] Unable to generate events for account/region. Account Id/Region: {record['account_id']}" f"/{record['region']} Reason: {exc}")
def get_base(vpc, **conn): """ The base will return: - ARN - Region - Name - Id - Tags - IsDefault - InstanceTenancy - CidrBlock - CidrBlockAssociationSet - Ipv6CidrBlockAssociationSet - DhcpOptionsId - Attributes - _version :param bucket_name: :param conn: :return: """ # Get the base: base_result = describe_vpcs(VpcIds=[vpc["id"]], **conn)[0] # The name of the VPC is in the tags: vpc_name = None for t in base_result.get("Tags", []): if t["Key"] == "Name": vpc_name = t["Value"] dhcp_opts = None # Get the DHCP Options: if base_result.get("DhcpOptionsId"): # There should only be exactly 1 attached to a VPC: dhcp_opts = describe_dhcp_options(DhcpOptionsIds=[base_result["DhcpOptionsId"]], **conn)[0]["DhcpOptionsId"] # Get the Attributes: attributes = {} attr_vals = [ ("EnableDnsHostnames", "enableDnsHostnames"), ("EnableDnsSupport", "enableDnsSupport") ] for attr, query in attr_vals: attributes[attr] = describe_vpc_attribute(VpcId=vpc["id"], Attribute=query, **conn)[attr] vpc.update({ 'name': vpc_name, 'region': conn["region"], 'tags': base_result.get("Tags", []), 'is_default': base_result["IsDefault"], 'instance_tenancy': base_result["InstanceTenancy"], 'dhcp_options_id': dhcp_opts, 'cidr_block': base_result["CidrBlock"], 'cidr_block_association_set': base_result.get("CidrBlockAssociationSet", []), 'ipv6_cidr_block_association_set': base_result.get("Ipv6CidrBlockAssociationSet", []), 'attributes': attributes, '_version': 1 }) return vpc
def test_describe_dhcp_options(test_vpc, dhcp_options): result = describe_dhcp_options(DhcpOptionsIds=[dhcp_options["DhcpOptionsId"]]) assert len(result) == 1 assert result[0]["DhcpConfigurations"][0]["Key"] == 'domain-name-servers' assert len(result[0]["DhcpConfigurations"][0]["Values"]) == 2 test_vpc = describe_vpcs(VpcIds=[test_vpc["VpcId"]]) assert test_vpc[0]["DhcpOptionsId"] == result[0]["DhcpOptionsId"]
def poller_processor_handler(event, context): """ Historical Security Group Poller Processor. This will receive events from the Poller Tasker, and will list all objects of a given technology for an account/region pair. This will generate `polling events` which simulate changes. These polling events contain configuration data such as the account/region defining where the collector should attempt to gather data from. """ log.debug('[@] Running Poller...') queue_url = get_queue_url( os.environ.get('POLLER_QUEUE_NAME', 'HistoricalVPCPoller')) records = deserialize_records(event['Records']) for record in records: # Skip accounts that have role assumption errors: try: vpcs = describe_vpcs(account_number=record['account_id'], assume_role=HISTORICAL_ROLE, region=record['region']) events = [ vpc_polling_schema.serialize(record['account_id'], v) for v in vpcs ] produce_events(events, queue_url) log.debug('[@] Finished generating polling events. Account: {}/{} ' 'Events Created: {}'.format(record['account_id'], record['region'], len(events))) except ClientError as e: log.error( '[X] Unable to generate events for account/region. Account Id/Region: {account_id}/{region}' ' Reason: {reason}'.format(account_id=record['account_id'], region=record['region'], reason=e)) log.debug('[@] Finished generating polling events. Events Created: {}'. format(len(record['account_id'])))
def test_describe_vpcs(test_vpc): result = describe_vpcs() assert len(result) == 2 # Find the non-default VPC and do stuff: for v in result: if v["CidrBlock"] == '172.31.0.0/16': assert v["IsDefault"] else: vpc_cidrs = ["10.0.0.0/16", "10.1.0.0/16"] assert v["CidrBlock"] == "10.0.0.0/16" assert v["InstanceTenancy"] == "default" assert len(v["Tags"]) == 2 assert not v["IsDefault"] assert len(v["CidrBlockAssociationSet"]) == 2 vpc_cidrs.remove(v["CidrBlockAssociationSet"][0]["CidrBlock"]) vpc_cidrs.remove(v["CidrBlockAssociationSet"][1]["CidrBlock"]) assert not vpc_cidrs assert len(v["Ipv6CidrBlockAssociationSet"]) == 1 assert v["Ipv6CidrBlockAssociationSet"][0]["Ipv6CidrBlock"] == "fde8:2d1a:7900:ea96::/64"
def list_method(self, **kwargs): return describe_vpcs(**kwargs)