def poller_tasker_handler(event, context): """ Historical Security Group Poller Tasker. The Poller is run at a set interval in order to ensure that changes do not go undetected by Historical. Historical pollers generate `polling events` which simulate changes. These polling events contain configuration data such as the account/region defining where the collector should attempt to gather data from. This is the entry point. This will task subsequent Poller lambdas to list all of a given resource in a select few AWS accounts. """ log.debug('[@] Running Poller Tasker...') queue_url = get_queue_url( os.environ.get('POLLER_TASKER_QUEUE_NAME', 'HistoricalSecurityGroupPollerTasker')) poller_task_schema = HistoricalPollerTaskEventModel() events = [] for account in get_historical_accounts(): for region in POLL_REGIONS: events.append( poller_task_schema.serialize_me(account['id'], region)) try: produce_events(events, queue_url) except ClientError as e: log.error( '[X] Unable to generate poller tasker events! Reason: {reason}'. format(reason=e)) log.debug('[@] Finished tasking the pollers.')
def handler(event, context): """ Historical S3 event poller. This poller is run at a set interval in order to ensure that changes do not go undetected by historical. Historical pollers generate `polling events` which simulate changes. These polling events contain configuration data such as the account/region defining where the collector should attempt to gather data from. """ log.debug('Running poller. Configuration: {}'.format(event)) for account in get_historical_accounts(): # Skip accounts that have role assumption errors: try: create_polling_event( account['id'], os.environ.get("HISTORICAL_STREAM", "HistoricalS3PollerStream")) except ClientError as e: log.warning( 'Unable to generate events for account. AccountId: {account_id} Reason: {reason}' .format(account_id=account['id'], reason=e)) log.debug( 'Finished generating polling events. Events Created: {}'.format( len(account['id'])))
def handler(event, context): """ Historical security group event poller. This poller is run at a set interval in order to ensure that changes do not go undetected by historical. Historical pollers generate `polling events` which simulate changes. These polling events contain configuration data such as the account/region defining where the collector should attempt to gather data from. """ log.debug('Running poller. Configuration: {}'.format(event)) for account in get_historical_accounts(): try: groups = describe_security_groups(account_number=account['id'], assume_role=HISTORICAL_ROLE, region=CURRENT_REGION) events = [ security_group_polling_schema.serialize(account['id'], g) for g in groups['SecurityGroups'] ] produce_events( events, os.environ.get('HISTORICAL_STREAM', 'HistoricalSecurityGroupPollerStream')) log.debug( 'Finished generating polling events. Account: {} Events Created: {}' .format(account['id'], len(events))) except ClientError as e: log.warning( 'Unable to generate events for account. AccountId: {account_id} Reason: {reason}' .format(account_id=account['id'], reason=e))
def poller_tasker_handler(event, context): # pylint: disable=W0613 """ Historical S3 Poller Tasker. The Poller is run at a set interval in order to ensure that changes do not go undetected by Historical. Historical pollers generate `polling events` which simulate changes. These polling events contain configuration data such as the account/region defining where the collector should attempt to gather data from. This is the entry point. This will task subsequent Poller lambdas to list all of a given resource in a select few AWS accounts. """ LOG.debug('[@] Running Poller Tasker...') queue_url = get_queue_url( os.environ.get('POLLER_TASKER_QUEUE_NAME', 'HistoricalS3PollerTasker')) poller_task_schema = HistoricalPollerTaskEventModel() events = [ poller_task_schema.serialize_me(account['id'], CURRENT_REGION) for account in get_historical_accounts() ] try: produce_events(events, queue_url, randomize_delay=RANDOMIZE_POLLER) except ClientError as exc: LOG.error( f'[X] Unable to generate poller tasker events! Reason: {exc}') LOG.debug('[@] Finished tasking the pollers.')
def handler(event, context): """ Historical VPC Poller. This Poller is run at a set interval in order to ensure that changes do not go undetected by historical. Historical Pollers generate `polling events` which simulate changes. These polling events contain configuration data such as the account/region defining where the collector should attempt to gather data from. """ log.debug('Running poller. Configuration: {}'.format(event)) queue_url = get_queue_url( os.environ.get('POLLER_QUEUE_NAME', 'HistoricalVPCPoller')) for account in get_historical_accounts(): for region in POLL_REGIONS: try: vpcs = describe_vpcs(account_number=account['id'], assume_role=HISTORICAL_ROLE, region=region) events = [ vpc_polling_schema.serialize(account['id'], v) for v in vpcs ] produce_events(events, queue_url) log.debug('Finished generating polling events. Account: {}/{} ' 'Events Created: {}'.format(account['id'], region, len(events))) except ClientError as e: log.warning( 'Unable to generate events for account/region. Account Id/Region: {account_id}/{region}' ' Reason: {reason}'.format(account_id=account['id'], region=region, reason=e))
def test_poller_tasker_handler(mock_lambda_environment, historical_sqs, swag_accounts): from historical.common.accounts import get_historical_accounts from historical.constants import CURRENT_REGION messages = make_poller_events() all_historical_accounts = get_historical_accounts() assert len(messages) == len(all_historical_accounts) == 1 poller_events = HistoricalPollerTaskEventModel().loads(messages[0]['body']).data assert poller_events['account_id'] == all_historical_accounts[0]['id'] assert poller_events['region'] == CURRENT_REGION
def test_get_accounts_with_env_var(): """Tests that passing in a CSV of account IDs in for the ENABLED_ACCOUNTS variable works.""" from historical.common.accounts import get_historical_accounts account_ids = ['012345678910', '111111111111', '222222222222'] os.environ['ENABLED_ACCOUNTS'] = ','.join(account_ids) result = get_historical_accounts() assert len(result) == len(account_ids) for account in result: account_ids.remove(account['id']) assert not account_ids
def handler(event, context): """ Historical S3 Poller. This poller is run at a set interval in order to ensure that changes do not go undetected by historical. Historical pollers generate `polling events` which simulate changes. These polling events contain configuration data such as the account/region defining where the collector should attempt to gather data from. """ log.debug('Running poller. Configuration: {}'.format(event)) queue_url = get_queue_url( os.environ.get('POLLER_QUEUE_NAME', 'HistoricalS3Poller')) for account in get_historical_accounts(): # Skip accounts that have role assumption errors: try: # List all buckets in the account: all_buckets = list_buckets( account_number=account['id'], assume_role=HISTORICAL_ROLE, session_name="historical-cloudwatch-s3list", region=CURRENT_REGION)["Buckets"] events = [ s3_polling_schema.serialize_me(account['id'], bucket) for bucket in all_buckets ] produce_events(events, queue_url) except ClientError as e: log.warning( 'Unable to generate events for account. AccountId: {account_id} Reason: {reason}' .format(account_id=account['id'], reason=e)) log.debug( 'Finished generating polling events. Events Created: {}'.format( len(account['id'])))
def test_get_only_test_accounts(swag_accounts): """Tests that the SWAG logic will only return 'test' accounts if specified.""" from historical.common.accounts import get_historical_accounts # Setup: bucket_name = 'SWAG' data_file = 'accounts.json' region = 'us-east-1' owner = 'third-party' os.environ['SWAG_BUCKET'] = bucket_name os.environ['SWAG_DATA_FILE'] = data_file os.environ['SWAG_REGION'] = region os.environ['SWAG_OWNER'] = owner swag_opts = { 'swag.type': 's3', 'swag.bucket_name': bucket_name, 'swag.data_file': data_file, 'swag.region': region, 'swag.cache_expires': 0 } swag = SWAGManager(**parse_swag_config_options(swag_opts)) # Production account: account = { 'aliases': ['prod'], 'contacts': ['*****@*****.**'], 'description': 'LOL, PROD account', 'email': '*****@*****.**', 'environment': 'prod', 'id': '999999999999', 'name': 'prodaccount', 'owner': 'third-party', 'provider': 'aws', 'sensitive': False, 'account_status': 'ready', 'services': [{ 'name': 'historical', 'status': [{ 'region': 'all', 'enabled': True }] }] } swag.create(account) # Get all the swag accounts: result = get_historical_accounts() assert len(result) == 2 assert result[1]['environment'] == 'prod' assert result[1]['id'] == '999999999999' # Only test accounts: os.environ['TEST_ACCOUNTS_ONLY'] = 'True' result = get_historical_accounts() assert len(result) == 1 assert result[0]['environment'] == 'test' assert result[0]['id'] != '999999999999' # Test the boolean logic: os.environ['TEST_ACCOUNTS_ONLY'] = '' result = get_historical_accounts() assert len(result) == 2 os.environ['TEST_ACCOUNTS_ONLY'] = 'false' result = get_historical_accounts() assert len(result) == 2 # Make sure that disabled/deleted accounts are not in the results: account['account_status'] = 'deleted' swag.update(account) result = get_historical_accounts() assert len(result) == 1