def poller_processor_handler(event, context):
"""
Historical S3 Poller Processor.
This will receive events from the Poller Tasker, and will list all objects of a given technology for an
account/region pair. This will generate `polling events` which simulate changes. These polling events contain
configuration data such as the account/region defining where the collector should attempt to gather data from.
"""
log.debug('[@] Running Poller...')
queue_url = get_queue_url(os.environ.get('POLLER_QUEUE_NAME', 'HistoricalS3Poller'))
records = deserialize_records(event['Records'])
for record in records:
# Skip accounts that have role assumption errors:
try:
# List all buckets in the account:
all_buckets = list_buckets(account_number=record['account_id'],
assume_role=HISTORICAL_ROLE,
session_name="historical-cloudwatch-s3list",
region=record['region'])["Buckets"]
events = [s3_polling_schema.serialize_me(record['account_id'], bucket) for bucket in all_buckets]
produce_events(events, queue_url)
except ClientError as e:
log.error('[X] Unable to generate events for account. Account Id: {account_id} Reason: {reason}'.format(
account_id=record['account_id'], reason=e))
log.debug('[@] Finished generating polling events. Events Created: {}'.format(len(record['account_id'])))
def poller_tasker_handler(event, context): # pylint: disable=W0613
"""
Historical Security Group Poller Tasker.
The Poller is run at a set interval in order to ensure that changes do not go undetected by Historical.
Historical pollers generate `polling events` which simulate changes. These polling events contain configuration
data such as the account/region defining where the collector should attempt to gather data from.
This is the entry point. This will task subsequent Poller lambdas to list all of a given resource in a select few
AWS accounts.
"""
LOG.debug('[@] Running Poller Tasker...')
queue_url = get_queue_url(
os.environ.get('POLLER_TASKER_QUEUE_NAME',
'HistoricalSecurityGroupPollerTasker'))
poller_task_schema = HistoricalPollerTaskEventModel()
events = []
for account in get_historical_accounts():
for region in POLL_REGIONS:
events.append(
poller_task_schema.serialize_me(account['id'], region))
try:
produce_events(events, queue_url, randomize_delay=RANDOMIZE_POLLER)
except ClientError as exc:
LOG.error(
f'[X] Unable to generate poller tasker events! Reason: {exc}')
LOG.debug('[@] Finished tasking the pollers.')
def poller_tasker_handler(event, context):
"""
Historical S3 Poller Tasker.
The Poller is run at a set interval in order to ensure that changes do not go undetected by Historical.
Historical pollers generate `polling events` which simulate changes. These polling events contain configuration
data such as the account/region defining where the collector should attempt to gather data from.
This is the entry point. This will task subsequent Poller lambdas to list all of a given resource in a select few
AWS accounts.
"""
log.debug('[@] Running Poller Tasker...')
queue_url = get_queue_url(os.environ.get('POLLER_TASKER_QUEUE_NAME', 'HistoricalS3PollerTasker'))
poller_task_schema = HistoricalPollerTaskEventModel()
events = [poller_task_schema.serialize_me(account['id'], CURRENT_REGION) for account in get_historical_accounts()]
try:
produce_events(events, queue_url)
except ClientError as e:
log.error('[X] Unable to generate poller tasker events! Reason: {reason}'.format(reason=e))
log.debug('[@] Finished tasking the pollers.')
def handler(event, context):
"""Historical S3 DynamoDB Stream Forwarder (the 'Proxy').
Passes events from the Historical DynamoDB stream and passes it to SNS or SQS for additional events to trigger.
You can optionally use SNS or SQS. It is preferable to use SNS -> SQS, but in some cases, such as the Current stream
to the Differ, this will make use of SQS to directly feed into the differ for performance purposes.
"""
queue_url = os.environ.get('PROXY_QUEUE_URL')
topic_arn = os.environ.get('PROXY_TOPIC_ARN')
if not queue_url and not topic_arn:
raise MissingProxyConfigurationException('[X] Must set the `PROXY_QUEUE_URL` or the `PROXY_TOPIC_ARN` vars.')
items_to_ship = []
for record in event['Records']:
item = make_proper_record(record)
# If there are no items, don't append anything:
if item:
items_to_ship.append(item)
if items_to_ship:
# SQS:
if queue_url:
# Need to reduce batch size to avoid SQS size limitations
produce_events(items_to_ship, queue_url, batch_size=5)
# SNS:
else:
client = boto3.client("sns", region_name=CURRENT_REGION)
for i in items_to_ship:
_publish_sns_message(client, i, topic_arn)
def handler(event, context):
"""
Historical Security Group Poller.
This Poller is run at a set interval in order to ensure that changes do not go undetected by historical.
Historical pollers generate `polling events` which simulate changes. These polling events contain configuration
data such as the account/region defining where the collector should attempt to gather data from.
"""
log.debug('Running poller. Configuration: {}'.format(event))
queue_url = get_queue_url(
os.environ.get('POLLER_QUEUE_NAME', 'HistoricalSecurityGroupPoller'))
for account in get_historical_accounts():
for region in POLL_REGIONS:
try:
groups = describe_security_groups(account_number=account['id'],
assume_role=HISTORICAL_ROLE,
region=region)
events = [
security_group_polling_schema.serialize(account['id'], g)
for g in groups['SecurityGroups']
]
produce_events(events, queue_url)
log.debug('Finished generating polling events. Account: {}/{} '
'Events Created: {}'.format(account['id'], region,
len(events)))
except ClientError as e:
log.warning(
'Unable to generate events for account/region. Account Id/Region: {account_id}/{region}'
' Reason: {reason}'.format(account_id=account['id'],
region=region,
reason=e))
def poller_processor_handler(event, context): # pylint: disable=W0613
"""
Historical Security Group Poller Processor.
This will receive events from the Poller Tasker, and will list all objects of a given technology for an
account/region pair. This will generate `polling events` which simulate changes. These polling events contain
configuration data such as the account/region defining where the collector should attempt to gather data from.
"""
LOG.debug('[@] Running Poller...')
queue_url = get_queue_url(os.environ.get('POLLER_QUEUE_NAME', 'HistoricalVPCPoller'))
records = deserialize_records(event['Records'])
for record in records:
# Skip accounts that have role assumption errors:
try:
vpcs = describe_vpcs(
account_number=record['account_id'],
assume_role=HISTORICAL_ROLE,
region=record['region']
)
events = [VPC_POLLING_SCHEMA.serialize(record['account_id'], v) for v in vpcs]
produce_events(events, queue_url, randomize_delay=RANDOMIZE_POLLER)
LOG.debug(f"[@] Finished generating polling events. Account: {record['account_id']}/{record['region']} "
f"Events Created: {len(events)}")
except ClientError as exc:
LOG.error(f"[X] Unable to generate events for account/region. Account Id/Region: {record['account_id']}"
f"/{record['region']} Reason: {exc}")
def handler(event, context): # pylint: disable=W0613
"""Historical S3 DynamoDB Stream Forwarder (the 'Proxy').
Passes events from the Historical DynamoDB stream and passes it to SNS or SQS for additional events to trigger.
You can optionally use SNS or SQS. It is preferable to use SNS -> SQS, but in some cases, such as the Current stream
to the Differ, this will make use of SQS to directly feed into the differ for performance purposes.
"""
queue_url = os.environ.get('PROXY_QUEUE_URL')
topic_arn = os.environ.get('PROXY_TOPIC_ARN')
if not queue_url and not topic_arn:
raise MissingProxyConfigurationException('[X] Must set the `PROXY_QUEUE_URL` or the `PROXY_TOPIC_ARN` vars.')
items_to_ship = []
# Must ALWAYS shrink for SQS because of 256KB limit of sending batched messages
force_shrink = True if queue_url else False
# Is this a "Simple Durable Proxy" -- that is -- are we stripping out all of the DynamoDB data from
# the Differ?
record_maker = make_proper_simple_record if SIMPLE_DURABLE_PROXY else make_proper_dynamodb_record
for record in event['Records']:
# We should NOT be processing this if the item in question does not
# reside in the PROXY_REGIONS
correct_region = True
for img in ['NewImage', 'OldImage']:
if record['dynamodb'].get(img):
if record['dynamodb'][img][REGION_ATTR]['S'] not in PROXY_REGIONS:
LOG.debug(f"[/] Not processing record -- record event took place in:"
f" {record['dynamodb'][img][REGION_ATTR]['S']}")
correct_region = False
break
if not correct_region:
continue
# Global DynamoDB tables will update a record with the global table specific fields. This creates 2 events
# whenever there is an update. The second update, which is a MODIFY event is not relevant and noise. This
# needs to be skipped over to prevent duplicated events. This is a "gotcha" in Global DynamoDB tables.
if detect_global_table_updates(record):
continue
items_to_ship.append(record_maker(record, force_shrink=force_shrink))
if items_to_ship:
# SQS:
if queue_url:
produce_events(items_to_ship, queue_url, batch_size=int(os.environ.get('PROXY_BATCH_SIZE', 10)))
# SNS:
else:
client = boto3.client("sns", region_name=CURRENT_REGION)
for i in items_to_ship:
_publish_sns_message(client, i, topic_arn)
def poller_processor_handler(event, context):
"""
Historical Security Group Poller Processor.
This will receive events from the Poller Tasker, and will list all objects of a given technology for an
account/region pair. This will generate `polling events` which simulate changes. These polling events contain
configuration data such as the account/region defining where the collector should attempt to gather data from.
"""
log.debug('[@] Running Poller...')
queue_url = get_queue_url(
os.environ.get('POLLER_QUEUE_NAME', 'HistoricalSecurityGroupPoller'))
records = deserialize_records(event['Records'])
for record in records:
# Skip accounts that have role assumption errors:
try:
groups = describe_security_groups(
account_number=record['account_id'],
assume_role=HISTORICAL_ROLE,
region=record['region'])
events = [
security_group_polling_schema.serialize(
record['account_id'], g) for g in groups['SecurityGroups']
]
produce_events(events, queue_url)
log.debug('[@] Finished generating polling events. Account: {}/{} '
'Events Created: {}'.format(record['account_id'],
record['region'],
len(events)))
except ClientError as e:
log.error(
'[X] Unable to generate events for account/region. Account Id/Region: {account_id}/{region}'
' Reason: {reason}'.format(account_id=record['account_id'],
region=record['region'],
reason=e))
log.debug('[@] Finished generating polling events. Events Created: {}'.
format(len(record['account_id'])))
def handler(event, context):
"""
Historical S3 Poller.
This poller is run at a set interval in order to ensure that changes do not go undetected by historical.
Historical pollers generate `polling events` which simulate changes. These polling events contain configuration
data such as the account/region defining where the collector should attempt to gather data from.
"""
log.debug('Running poller. Configuration: {}'.format(event))
queue_url = get_queue_url(
os.environ.get('POLLER_QUEUE_NAME', 'HistoricalS3Poller'))
for account in get_historical_accounts():
# Skip accounts that have role assumption errors:
try:
# List all buckets in the account:
all_buckets = list_buckets(
account_number=account['id'],
assume_role=HISTORICAL_ROLE,
session_name="historical-cloudwatch-s3list",
region=CURRENT_REGION)["Buckets"]
events = [
s3_polling_schema.serialize_me(account['id'], bucket)
for bucket in all_buckets
]
produce_events(events, queue_url)
except ClientError as e:
log.warning(
'Unable to generate events for account. AccountId: {account_id} Reason: {reason}'
.format(account_id=account['id'], reason=e))
log.debug(
'Finished generating polling events. Events Created: {}'.format(
len(account['id'])))
def poller_processor_handler(event, context): # pylint: disable=W0613
"""
Historical Security Group Poller Processor.
This will receive events from the Poller Tasker, and will list all objects of a given technology for an
account/region pair. This will generate `polling events` which simulate changes. These polling events contain
configuration data such as the account/region defining where the collector should attempt to gather data from.
"""
LOG.debug('[@] Running Poller...')
collector_poller_queue_url = get_queue_url(
os.environ.get('POLLER_QUEUE_NAME', 'HistoricalSecurityGroupPoller'))
takser_queue_url = get_queue_url(
os.environ.get('POLLER_TASKER_QUEUE_NAME',
'HistoricalSecurityGroupPollerTasker'))
poller_task_schema = HistoricalPollerTaskEventModel()
records = deserialize_records(event['Records'])
for record in records:
# Skip accounts that have role assumption errors:
try:
# Did we get a NextToken?
if record.get('NextToken'):
LOG.debug(
f"[@] Received pagination token: {record['NextToken']}")
groups = describe_security_groups(
account_number=record['account_id'],
assume_role=HISTORICAL_ROLE,
region=record['region'],
MaxResults=200,
NextToken=record['NextToken'])
else:
groups = describe_security_groups(
account_number=record['account_id'],
assume_role=HISTORICAL_ROLE,
region=record['region'],
MaxResults=200)
# FIRST THINGS FIRST: Did we get a `NextToken`? If so, we need to enqueue that ASAP because
# 'NextToken`s expire in 60 seconds!
if groups.get('NextToken'):
logging.debug(
f"[-->] Pagination required {groups['NextToken']}. Tasking continuation."
)
produce_events([
poller_task_schema.serialize_me(
record['account_id'],
record['region'],
next_token=groups['NextToken'])
], takser_queue_url)
# Task the collector to perform all the DDB logic -- this will pass in the collected data to the
# collector in very small batches.
events = [
SECURITY_GROUP_POLLING_SCHEMA.serialize(
record['account_id'], g, record['region'])
for g in groups['SecurityGroups']
]
produce_events(events, collector_poller_queue_url, batch_size=3)
LOG.debug(
f"[@] Finished generating polling events. Account: {record['account_id']}/{record['region']} "
f"Events Created: {len(events)}")
except ClientError as exc:
LOG.error(
f"[X] Unable to generate events for account/region. Account Id/Region: {record['account_id']}"
f"/{record['region']} Reason: {exc}")