def describe_vpc(record):
"""Attempts to describe vpc ids."""
account_id = record['account']
vpc_name = cloudwatch.filter_request_parameters('vpcName', record)
vpc_id = cloudwatch.filter_request_parameters('vpcId', record)
try:
if vpc_id and vpc_name:
return describe_vpcs(account_number=account_id,
assume_role=HISTORICAL_ROLE,
region=CURRENT_REGION,
Filters=[{
'Name': 'vpc-id',
'Values': [vpc_id]
}])
elif vpc_id:
return describe_vpcs(account_number=account_id,
assume_role=HISTORICAL_ROLE,
region=CURRENT_REGION,
VpcIds=[vpc_id])
else:
raise Exception('[X] Describe requires VpcId.')
except ClientError as e:
if e.response['Error']['Code'] == 'InvalidVpc.NotFound':
return []
raise e
def handler(event, context):
"""
Historical VPC Poller.
This Poller is run at a set interval in order to ensure that changes do not go undetected by historical.
Historical Pollers generate `polling events` which simulate changes. These polling events contain configuration
data such as the account/region defining where the collector should attempt to gather data from.
"""
log.debug('Running poller. Configuration: {}'.format(event))
queue_url = get_queue_url(
os.environ.get('POLLER_QUEUE_NAME', 'HistoricalVPCPoller'))
for account in get_historical_accounts():
for region in POLL_REGIONS:
try:
vpcs = describe_vpcs(account_number=account['id'],
assume_role=HISTORICAL_ROLE,
region=region)
events = [
vpc_polling_schema.serialize(account['id'], v)
for v in vpcs
]
produce_events(events, queue_url)
log.debug('Finished generating polling events. Account: {}/{} '
'Events Created: {}'.format(account['id'], region,
len(events)))
except ClientError as e:
log.warning(
'Unable to generate events for account/region. Account Id/Region: {account_id}/{region}'
' Reason: {reason}'.format(account_id=account['id'],
region=region,
reason=e))
def handler(event, context):
"""
Historical security group event poller.
This poller is run at a set interval in order to ensure that changes do not go undetected by historical.
Historical pollers generate `polling events` which simulate changes. These polling events contain configuration
data such as the account/region defining where the collector should attempt to gather data from.
"""
log.debug('Running poller. Configuration: {}'.format(event))
for account in get_historical_accounts():
try:
vpcs = describe_vpcs(account_number=account['id'],
assume_role=HISTORICAL_ROLE,
region=CURRENT_REGION)
events = [
vpc_polling_schema.serialize(account['id'], v)
for v in vpcs['Vpcs']
]
produce_events(
events,
os.environ.get('HISTORICAL_STREAM',
'HistoricalVPCPollerStream'))
log.debug(
'Finished generating polling events. Account: {} Events Created: {}'
.format(account['id'], len(events)))
except ClientError as e:
log.warning(
'Unable to generate events for account. AccountId: {account_id} Reason: {reason}'
.format(account_id=account['id'], reason=e))
def poller_processor_handler(event, context): # pylint: disable=W0613
"""
Historical Security Group Poller Processor.
This will receive events from the Poller Tasker, and will list all objects of a given technology for an
account/region pair. This will generate `polling events` which simulate changes. These polling events contain
configuration data such as the account/region defining where the collector should attempt to gather data from.
"""
LOG.debug('[@] Running Poller...')
queue_url = get_queue_url(os.environ.get('POLLER_QUEUE_NAME', 'HistoricalVPCPoller'))
records = deserialize_records(event['Records'])
for record in records:
# Skip accounts that have role assumption errors:
try:
vpcs = describe_vpcs(
account_number=record['account_id'],
assume_role=HISTORICAL_ROLE,
region=record['region']
)
events = [VPC_POLLING_SCHEMA.serialize(record['account_id'], v) for v in vpcs]
produce_events(events, queue_url, randomize_delay=RANDOMIZE_POLLER)
LOG.debug(f"[@] Finished generating polling events. Account: {record['account_id']}/{record['region']} "
f"Events Created: {len(events)}")
except ClientError as exc:
LOG.error(f"[X] Unable to generate events for account/region. Account Id/Region: {record['account_id']}"
f"/{record['region']} Reason: {exc}")
def get_base(vpc, **conn):
"""
The base will return:
- ARN
- Region
- Name
- Id
- Tags
- IsDefault
- InstanceTenancy
- CidrBlock
- CidrBlockAssociationSet
- Ipv6CidrBlockAssociationSet
- DhcpOptionsId
- Attributes
- _version
:param bucket_name:
:param conn:
:return:
"""
# Get the base:
base_result = describe_vpcs(VpcIds=[vpc["id"]], **conn)[0]
# The name of the VPC is in the tags:
vpc_name = None
for t in base_result.get("Tags", []):
if t["Key"] == "Name":
vpc_name = t["Value"]
dhcp_opts = None
# Get the DHCP Options:
if base_result.get("DhcpOptionsId"):
# There should only be exactly 1 attached to a VPC:
dhcp_opts = describe_dhcp_options(DhcpOptionsIds=[base_result["DhcpOptionsId"]], **conn)[0]["DhcpOptionsId"]
# Get the Attributes:
attributes = {}
attr_vals = [
("EnableDnsHostnames", "enableDnsHostnames"),
("EnableDnsSupport", "enableDnsSupport")
]
for attr, query in attr_vals:
attributes[attr] = describe_vpc_attribute(VpcId=vpc["id"], Attribute=query, **conn)[attr]
vpc.update({
'name': vpc_name,
'region': conn["region"],
'tags': base_result.get("Tags", []),
'is_default': base_result["IsDefault"],
'instance_tenancy': base_result["InstanceTenancy"],
'dhcp_options_id': dhcp_opts,
'cidr_block': base_result["CidrBlock"],
'cidr_block_association_set': base_result.get("CidrBlockAssociationSet", []),
'ipv6_cidr_block_association_set': base_result.get("Ipv6CidrBlockAssociationSet", []),
'attributes': attributes,
'_version': 1
})
return vpc
def test_describe_dhcp_options(test_vpc, dhcp_options):
result = describe_dhcp_options(DhcpOptionsIds=[dhcp_options["DhcpOptionsId"]])
assert len(result) == 1
assert result[0]["DhcpConfigurations"][0]["Key"] == 'domain-name-servers'
assert len(result[0]["DhcpConfigurations"][0]["Values"]) == 2
test_vpc = describe_vpcs(VpcIds=[test_vpc["VpcId"]])
assert test_vpc[0]["DhcpOptionsId"] == result[0]["DhcpOptionsId"]
def poller_processor_handler(event, context):
"""
Historical Security Group Poller Processor.
This will receive events from the Poller Tasker, and will list all objects of a given technology for an
account/region pair. This will generate `polling events` which simulate changes. These polling events contain
configuration data such as the account/region defining where the collector should attempt to gather data from.
"""
log.debug('[@] Running Poller...')
queue_url = get_queue_url(
os.environ.get('POLLER_QUEUE_NAME', 'HistoricalVPCPoller'))
records = deserialize_records(event['Records'])
for record in records:
# Skip accounts that have role assumption errors:
try:
vpcs = describe_vpcs(account_number=record['account_id'],
assume_role=HISTORICAL_ROLE,
region=record['region'])
events = [
vpc_polling_schema.serialize(record['account_id'], v)
for v in vpcs
]
produce_events(events, queue_url)
log.debug('[@] Finished generating polling events. Account: {}/{} '
'Events Created: {}'.format(record['account_id'],
record['region'],
len(events)))
except ClientError as e:
log.error(
'[X] Unable to generate events for account/region. Account Id/Region: {account_id}/{region}'
' Reason: {reason}'.format(account_id=record['account_id'],
region=record['region'],
reason=e))
log.debug('[@] Finished generating polling events. Events Created: {}'.
format(len(record['account_id'])))
def test_describe_vpcs(test_vpc):
result = describe_vpcs()
assert len(result) == 2
# Find the non-default VPC and do stuff:
for v in result:
if v["CidrBlock"] == '172.31.0.0/16':
assert v["IsDefault"]
else:
vpc_cidrs = ["10.0.0.0/16", "10.1.0.0/16"]
assert v["CidrBlock"] == "10.0.0.0/16"
assert v["InstanceTenancy"] == "default"
assert len(v["Tags"]) == 2
assert not v["IsDefault"]
assert len(v["CidrBlockAssociationSet"]) == 2
vpc_cidrs.remove(v["CidrBlockAssociationSet"][0]["CidrBlock"])
vpc_cidrs.remove(v["CidrBlockAssociationSet"][1]["CidrBlock"])
assert not vpc_cidrs
assert len(v["Ipv6CidrBlockAssociationSet"]) == 1
assert v["Ipv6CidrBlockAssociationSet"][0]["Ipv6CidrBlock"] == "fde8:2d1a:7900:ea96::/64"
def list_method(self, **kwargs):
return describe_vpcs(**kwargs)
def list_method(self, **kwargs):
return describe_vpcs(**kwargs)