def poller_tasker_handler(event, context):
"""
Historical Security Group Poller Tasker.
The Poller is run at a set interval in order to ensure that changes do not go undetected by Historical.
Historical pollers generate `polling events` which simulate changes. These polling events contain configuration
data such as the account/region defining where the collector should attempt to gather data from.
This is the entry point. This will task subsequent Poller lambdas to list all of a given resource in a select few
AWS accounts.
"""
log.debug('[@] Running Poller Tasker...')
queue_url = get_queue_url(
os.environ.get('POLLER_TASKER_QUEUE_NAME',
'HistoricalSecurityGroupPollerTasker'))
poller_task_schema = HistoricalPollerTaskEventModel()
events = []
for account in get_historical_accounts():
for region in POLL_REGIONS:
events.append(
poller_task_schema.serialize_me(account['id'], region))
try:
produce_events(events, queue_url)
except ClientError as e:
log.error(
'[X] Unable to generate poller tasker events! Reason: {reason}'.
format(reason=e))
log.debug('[@] Finished tasking the pollers.')
def handler(event, context):
"""
Historical S3 event poller.
This poller is run at a set interval in order to ensure that changes do not go undetected by historical.
Historical pollers generate `polling events` which simulate changes. These polling events contain configuration
data such as the account/region defining where the collector should attempt to gather data from.
"""
log.debug('Running poller. Configuration: {}'.format(event))
for account in get_historical_accounts():
# Skip accounts that have role assumption errors:
try:
create_polling_event(
account['id'],
os.environ.get("HISTORICAL_STREAM",
"HistoricalS3PollerStream"))
except ClientError as e:
log.warning(
'Unable to generate events for account. AccountId: {account_id} Reason: {reason}'
.format(account_id=account['id'], reason=e))
log.debug(
'Finished generating polling events. Events Created: {}'.format(
len(account['id'])))
def handler(event, context):
"""
Historical security group event poller.
This poller is run at a set interval in order to ensure that changes do not go undetected by historical.
Historical pollers generate `polling events` which simulate changes. These polling events contain configuration
data such as the account/region defining where the collector should attempt to gather data from.
"""
log.debug('Running poller. Configuration: {}'.format(event))
for account in get_historical_accounts():
try:
groups = describe_security_groups(account_number=account['id'],
assume_role=HISTORICAL_ROLE,
region=CURRENT_REGION)
events = [
security_group_polling_schema.serialize(account['id'], g)
for g in groups['SecurityGroups']
]
produce_events(
events,
os.environ.get('HISTORICAL_STREAM',
'HistoricalSecurityGroupPollerStream'))
log.debug(
'Finished generating polling events. Account: {} Events Created: {}'
.format(account['id'], len(events)))
except ClientError as e:
log.warning(
'Unable to generate events for account. AccountId: {account_id} Reason: {reason}'
.format(account_id=account['id'], reason=e))
def poller_tasker_handler(event, context): # pylint: disable=W0613
"""
Historical S3 Poller Tasker.
The Poller is run at a set interval in order to ensure that changes do not go undetected by Historical.
Historical pollers generate `polling events` which simulate changes. These polling events contain configuration
data such as the account/region defining where the collector should attempt to gather data from.
This is the entry point. This will task subsequent Poller lambdas to list all of a given resource in a select few
AWS accounts.
"""
LOG.debug('[@] Running Poller Tasker...')
queue_url = get_queue_url(
os.environ.get('POLLER_TASKER_QUEUE_NAME', 'HistoricalS3PollerTasker'))
poller_task_schema = HistoricalPollerTaskEventModel()
events = [
poller_task_schema.serialize_me(account['id'], CURRENT_REGION)
for account in get_historical_accounts()
]
try:
produce_events(events, queue_url, randomize_delay=RANDOMIZE_POLLER)
except ClientError as exc:
LOG.error(
f'[X] Unable to generate poller tasker events! Reason: {exc}')
LOG.debug('[@] Finished tasking the pollers.')
def handler(event, context):
"""
Historical VPC Poller.
This Poller is run at a set interval in order to ensure that changes do not go undetected by historical.
Historical Pollers generate `polling events` which simulate changes. These polling events contain configuration
data such as the account/region defining where the collector should attempt to gather data from.
"""
log.debug('Running poller. Configuration: {}'.format(event))
queue_url = get_queue_url(
os.environ.get('POLLER_QUEUE_NAME', 'HistoricalVPCPoller'))
for account in get_historical_accounts():
for region in POLL_REGIONS:
try:
vpcs = describe_vpcs(account_number=account['id'],
assume_role=HISTORICAL_ROLE,
region=region)
events = [
vpc_polling_schema.serialize(account['id'], v)
for v in vpcs
]
produce_events(events, queue_url)
log.debug('Finished generating polling events. Account: {}/{} '
'Events Created: {}'.format(account['id'], region,
len(events)))
except ClientError as e:
log.warning(
'Unable to generate events for account/region. Account Id/Region: {account_id}/{region}'
' Reason: {reason}'.format(account_id=account['id'],
region=region,
reason=e))
def test_poller_tasker_handler(mock_lambda_environment, historical_sqs, swag_accounts):
from historical.common.accounts import get_historical_accounts
from historical.constants import CURRENT_REGION
messages = make_poller_events()
all_historical_accounts = get_historical_accounts()
assert len(messages) == len(all_historical_accounts) == 1
poller_events = HistoricalPollerTaskEventModel().loads(messages[0]['body']).data
assert poller_events['account_id'] == all_historical_accounts[0]['id']
assert poller_events['region'] == CURRENT_REGION
def test_get_accounts_with_env_var():
"""Tests that passing in a CSV of account IDs in for the ENABLED_ACCOUNTS variable works."""
from historical.common.accounts import get_historical_accounts
account_ids = ['012345678910', '111111111111', '222222222222']
os.environ['ENABLED_ACCOUNTS'] = ','.join(account_ids)
result = get_historical_accounts()
assert len(result) == len(account_ids)
for account in result:
account_ids.remove(account['id'])
assert not account_ids
def handler(event, context):
"""
Historical S3 Poller.
This poller is run at a set interval in order to ensure that changes do not go undetected by historical.
Historical pollers generate `polling events` which simulate changes. These polling events contain configuration
data such as the account/region defining where the collector should attempt to gather data from.
"""
log.debug('Running poller. Configuration: {}'.format(event))
queue_url = get_queue_url(
os.environ.get('POLLER_QUEUE_NAME', 'HistoricalS3Poller'))
for account in get_historical_accounts():
# Skip accounts that have role assumption errors:
try:
# List all buckets in the account:
all_buckets = list_buckets(
account_number=account['id'],
assume_role=HISTORICAL_ROLE,
session_name="historical-cloudwatch-s3list",
region=CURRENT_REGION)["Buckets"]
events = [
s3_polling_schema.serialize_me(account['id'], bucket)
for bucket in all_buckets
]
produce_events(events, queue_url)
except ClientError as e:
log.warning(
'Unable to generate events for account. AccountId: {account_id} Reason: {reason}'
.format(account_id=account['id'], reason=e))
log.debug(
'Finished generating polling events. Events Created: {}'.format(
len(account['id'])))
def test_get_only_test_accounts(swag_accounts):
"""Tests that the SWAG logic will only return 'test' accounts if specified."""
from historical.common.accounts import get_historical_accounts
# Setup:
bucket_name = 'SWAG'
data_file = 'accounts.json'
region = 'us-east-1'
owner = 'third-party'
os.environ['SWAG_BUCKET'] = bucket_name
os.environ['SWAG_DATA_FILE'] = data_file
os.environ['SWAG_REGION'] = region
os.environ['SWAG_OWNER'] = owner
swag_opts = {
'swag.type': 's3',
'swag.bucket_name': bucket_name,
'swag.data_file': data_file,
'swag.region': region,
'swag.cache_expires': 0
}
swag = SWAGManager(**parse_swag_config_options(swag_opts))
# Production account:
account = {
'aliases': ['prod'],
'contacts': ['*****@*****.**'],
'description':
'LOL, PROD account',
'email':
'*****@*****.**',
'environment':
'prod',
'id':
'999999999999',
'name':
'prodaccount',
'owner':
'third-party',
'provider':
'aws',
'sensitive':
False,
'account_status':
'ready',
'services': [{
'name': 'historical',
'status': [{
'region': 'all',
'enabled': True
}]
}]
}
swag.create(account)
# Get all the swag accounts:
result = get_historical_accounts()
assert len(result) == 2
assert result[1]['environment'] == 'prod'
assert result[1]['id'] == '999999999999'
# Only test accounts:
os.environ['TEST_ACCOUNTS_ONLY'] = 'True'
result = get_historical_accounts()
assert len(result) == 1
assert result[0]['environment'] == 'test'
assert result[0]['id'] != '999999999999'
# Test the boolean logic:
os.environ['TEST_ACCOUNTS_ONLY'] = ''
result = get_historical_accounts()
assert len(result) == 2
os.environ['TEST_ACCOUNTS_ONLY'] = 'false'
result = get_historical_accounts()
assert len(result) == 2
# Make sure that disabled/deleted accounts are not in the results:
account['account_status'] = 'deleted'
swag.update(account)
result = get_historical_accounts()
assert len(result) == 1
