def poller_processor_handler(event, context):
"""
Historical S3 Poller Processor.
This will receive events from the Poller Tasker, and will list all objects of a given technology for an
account/region pair. This will generate `polling events` which simulate changes. These polling events contain
configuration data such as the account/region defining where the collector should attempt to gather data from.
"""
log.debug('[@] Running Poller...')
queue_url = get_queue_url(os.environ.get('POLLER_QUEUE_NAME', 'HistoricalS3Poller'))
records = deserialize_records(event['Records'])
for record in records:
# Skip accounts that have role assumption errors:
try:
# List all buckets in the account:
all_buckets = list_buckets(account_number=record['account_id'],
assume_role=HISTORICAL_ROLE,
session_name="historical-cloudwatch-s3list",
region=record['region'])["Buckets"]
events = [s3_polling_schema.serialize_me(record['account_id'], bucket) for bucket in all_buckets]
produce_events(events, queue_url)
except ClientError as e:
log.error('[X] Unable to generate events for account. Account Id: {account_id} Reason: {reason}'.format(
account_id=record['account_id'], reason=e))
log.debug('[@] Finished generating polling events. Events Created: {}'.format(len(record['account_id'])))
def create_polling_event(account, stream):
# Place onto the S3 Kinesis stream each S3 bucket for each account...
# This should probably fan out on an account-by-account basis (we'll need to examine if this is an issue)
all_buckets = list_buckets(account_number=account,
assume_role=HISTORICAL_ROLE,
session_name="historical-cloudwatch-s3list",
region=CURRENT_REGION)["Buckets"]
client = boto3.client("kinesis", region_name=CURRENT_REGION)
# Need to add all buckets into the stream:
limiter = int(os.environ.get("MAX_BUCKET_BATCH", 50))
current_batch = 1
total_batch = int(len(all_buckets) / limiter)
remainder = len(all_buckets) % limiter
offset = 0
while current_batch <= total_batch:
records = []
while offset < (limiter * current_batch):
records.append(get_record(all_buckets, offset, account))
offset += 1
client.put_records(Records=records, StreamName=stream)
current_batch += 1
# Process remainder:
if remainder:
records = []
while offset < len(all_buckets):
records.append(get_record(all_buckets, offset, account))
offset += 1
client.put_records(Records=records, StreamName=stream)
def handler(event, context):
"""
Historical S3 Poller.
This poller is run at a set interval in order to ensure that changes do not go undetected by historical.
Historical pollers generate `polling events` which simulate changes. These polling events contain configuration
data such as the account/region defining where the collector should attempt to gather data from.
"""
log.debug('Running poller. Configuration: {}'.format(event))
queue_url = get_queue_url(
os.environ.get('POLLER_QUEUE_NAME', 'HistoricalS3Poller'))
for account in get_historical_accounts():
# Skip accounts that have role assumption errors:
try:
# List all buckets in the account:
all_buckets = list_buckets(
account_number=account['id'],
assume_role=HISTORICAL_ROLE,
session_name="historical-cloudwatch-s3list",
region=CURRENT_REGION)["Buckets"]
events = [
s3_polling_schema.serialize_me(account['id'], bucket)
for bucket in all_buckets
]
produce_events(events, queue_url)
except ClientError as e:
log.warning(
'Unable to generate events for account. AccountId: {account_id} Reason: {reason}'
.format(account_id=account['id'], reason=e))
log.debug(
'Finished generating polling events. Events Created: {}'.format(
len(account['id'])))
def fetch_id(**kwargs):
return list_buckets(**kwargs["conn_dict"])
def list_method(self, **kwargs):
buckets = list_buckets(**kwargs)['Buckets']
return [bucket['Name'] for bucket in buckets]
def fetch_id(**kwargs):
return list_buckets(**kwargs["conn_dict"])
def list_method(self, **kwargs):
buckets = list_buckets(**kwargs)['Buckets']
return [bucket['Name'] for bucket in buckets]
def list_buckets(self, **kwargs):
buckets = list_buckets(**kwargs)
return [
bucket['Name'] for bucket in buckets['Buckets']
if not self.check_ignore_list(bucket['Name'])
]
def list_buckets(self, **kwargs):
buckets = list_buckets(**kwargs)
return [bucket['Name'] for bucket in buckets['Buckets'] if not self.check_ignore_list(bucket['Name'])]
