def poller_tasker_handler(event, context):
"""
Historical S3 Poller Tasker.
The Poller is run at a set interval in order to ensure that changes do not go undetected by Historical.
Historical pollers generate `polling events` which simulate changes. These polling events contain configuration
data such as the account/region defining where the collector should attempt to gather data from.
This is the entry point. This will task subsequent Poller lambdas to list all of a given resource in a select few
AWS accounts.
"""
log.debug('[@] Running Poller Tasker...')
queue_url = get_queue_url(os.environ.get('POLLER_TASKER_QUEUE_NAME', 'HistoricalS3PollerTasker'))
poller_task_schema = HistoricalPollerTaskEventModel()
events = [poller_task_schema.serialize_me(account['id'], CURRENT_REGION) for account in get_historical_accounts()]
try:
produce_events(events, queue_url)
except ClientError as e:
log.error('[X] Unable to generate poller tasker events! Reason: {reason}'.format(reason=e))
log.debug('[@] Finished tasking the pollers.')
def poller_tasker_handler(event, context): # pylint: disable=W0613
"""
Historical Security Group Poller Tasker.
The Poller is run at a set interval in order to ensure that changes do not go undetected by Historical.
Historical pollers generate `polling events` which simulate changes. These polling events contain configuration
data such as the account/region defining where the collector should attempt to gather data from.
This is the entry point. This will task subsequent Poller lambdas to list all of a given resource in a select few
AWS accounts.
"""
LOG.debug('[@] Running Poller Tasker...')
queue_url = get_queue_url(
os.environ.get('POLLER_TASKER_QUEUE_NAME',
'HistoricalSecurityGroupPollerTasker'))
poller_task_schema = HistoricalPollerTaskEventModel()
events = []
for account in get_historical_accounts():
for region in POLL_REGIONS:
events.append(
poller_task_schema.serialize_me(account['id'], region))
try:
produce_events(events, queue_url, randomize_delay=RANDOMIZE_POLLER)
except ClientError as exc:
LOG.error(
f'[X] Unable to generate poller tasker events! Reason: {exc}')
LOG.debug('[@] Finished tasking the pollers.')
def test_poller_tasker_handler(mock_lambda_environment, historical_sqs, swag_accounts):
from historical.common.accounts import get_historical_accounts
from historical.constants import CURRENT_REGION
messages = make_poller_events()
all_historical_accounts = get_historical_accounts()
assert len(messages) == len(all_historical_accounts) == 1
poller_events = HistoricalPollerTaskEventModel().loads(messages[0]['body']).data
assert poller_events['account_id'] == all_historical_accounts[0]['id']
assert poller_events['region'] == CURRENT_REGION
def poller_processor_handler(event, context): # pylint: disable=W0613
"""
Historical Security Group Poller Processor.
This will receive events from the Poller Tasker, and will list all objects of a given technology for an
account/region pair. This will generate `polling events` which simulate changes. These polling events contain
configuration data such as the account/region defining where the collector should attempt to gather data from.
"""
LOG.debug('[@] Running Poller...')
collector_poller_queue_url = get_queue_url(
os.environ.get('POLLER_QUEUE_NAME', 'HistoricalSecurityGroupPoller'))
takser_queue_url = get_queue_url(
os.environ.get('POLLER_TASKER_QUEUE_NAME',
'HistoricalSecurityGroupPollerTasker'))
poller_task_schema = HistoricalPollerTaskEventModel()
records = deserialize_records(event['Records'])
for record in records:
# Skip accounts that have role assumption errors:
try:
# Did we get a NextToken?
if record.get('NextToken'):
LOG.debug(
f"[@] Received pagination token: {record['NextToken']}")
groups = describe_security_groups(
account_number=record['account_id'],
assume_role=HISTORICAL_ROLE,
region=record['region'],
MaxResults=200,
NextToken=record['NextToken'])
else:
groups = describe_security_groups(
account_number=record['account_id'],
assume_role=HISTORICAL_ROLE,
region=record['region'],
MaxResults=200)
# FIRST THINGS FIRST: Did we get a `NextToken`? If so, we need to enqueue that ASAP because
# 'NextToken`s expire in 60 seconds!
if groups.get('NextToken'):
logging.debug(
f"[-->] Pagination required {groups['NextToken']}. Tasking continuation."
)
produce_events([
poller_task_schema.serialize_me(
record['account_id'],
record['region'],
next_token=groups['NextToken'])
], takser_queue_url)
# Task the collector to perform all the DDB logic -- this will pass in the collected data to the
# collector in very small batches.
events = [
SECURITY_GROUP_POLLING_SCHEMA.serialize(
record['account_id'], g, record['region'])
for g in groups['SecurityGroups']
]
produce_events(events, collector_poller_queue_url, batch_size=3)
LOG.debug(
f"[@] Finished generating polling events. Account: {record['account_id']}/{record['region']} "
f"Events Created: {len(events)}")
except ClientError as exc:
LOG.error(
f"[X] Unable to generate events for account/region. Account Id/Region: {record['account_id']}"
f"/{record['region']} Reason: {exc}")