def poller_processor_handler(event, context): """ Historical S3 Poller Processor. This will receive events from the Poller Tasker, and will list all objects of a given technology for an account/region pair. This will generate `polling events` which simulate changes. These polling events contain configuration data such as the account/region defining where the collector should attempt to gather data from. """ log.debug('[@] Running Poller...') queue_url = get_queue_url(os.environ.get('POLLER_QUEUE_NAME', 'HistoricalS3Poller')) records = deserialize_records(event['Records']) for record in records: # Skip accounts that have role assumption errors: try: # List all buckets in the account: all_buckets = list_buckets(account_number=record['account_id'], assume_role=HISTORICAL_ROLE, session_name="historical-cloudwatch-s3list", region=record['region'])["Buckets"] events = [s3_polling_schema.serialize_me(record['account_id'], bucket) for bucket in all_buckets] produce_events(events, queue_url) except ClientError as e: log.error('[X] Unable to generate events for account. Account Id: {account_id} Reason: {reason}'.format( account_id=record['account_id'], reason=e)) log.debug('[@] Finished generating polling events. Events Created: {}'.format(len(record['account_id'])))
def create_polling_event(account, stream): # Place onto the S3 Kinesis stream each S3 bucket for each account... # This should probably fan out on an account-by-account basis (we'll need to examine if this is an issue) all_buckets = list_buckets(account_number=account, assume_role=HISTORICAL_ROLE, session_name="historical-cloudwatch-s3list", region=CURRENT_REGION)["Buckets"] client = boto3.client("kinesis", region_name=CURRENT_REGION) # Need to add all buckets into the stream: limiter = int(os.environ.get("MAX_BUCKET_BATCH", 50)) current_batch = 1 total_batch = int(len(all_buckets) / limiter) remainder = len(all_buckets) % limiter offset = 0 while current_batch <= total_batch: records = [] while offset < (limiter * current_batch): records.append(get_record(all_buckets, offset, account)) offset += 1 client.put_records(Records=records, StreamName=stream) current_batch += 1 # Process remainder: if remainder: records = [] while offset < len(all_buckets): records.append(get_record(all_buckets, offset, account)) offset += 1 client.put_records(Records=records, StreamName=stream)
def handler(event, context): """ Historical S3 Poller. This poller is run at a set interval in order to ensure that changes do not go undetected by historical. Historical pollers generate `polling events` which simulate changes. These polling events contain configuration data such as the account/region defining where the collector should attempt to gather data from. """ log.debug('Running poller. Configuration: {}'.format(event)) queue_url = get_queue_url( os.environ.get('POLLER_QUEUE_NAME', 'HistoricalS3Poller')) for account in get_historical_accounts(): # Skip accounts that have role assumption errors: try: # List all buckets in the account: all_buckets = list_buckets( account_number=account['id'], assume_role=HISTORICAL_ROLE, session_name="historical-cloudwatch-s3list", region=CURRENT_REGION)["Buckets"] events = [ s3_polling_schema.serialize_me(account['id'], bucket) for bucket in all_buckets ] produce_events(events, queue_url) except ClientError as e: log.warning( 'Unable to generate events for account. AccountId: {account_id} Reason: {reason}' .format(account_id=account['id'], reason=e)) log.debug( 'Finished generating polling events. Events Created: {}'.format( len(account['id'])))
def fetch_id(**kwargs): return list_buckets(**kwargs["conn_dict"])
def list_method(self, **kwargs): buckets = list_buckets(**kwargs)['Buckets'] return [bucket['Name'] for bucket in buckets]
def fetch_id(**kwargs): return list_buckets(**kwargs["conn_dict"])
def list_method(self, **kwargs): buckets = list_buckets(**kwargs)['Buckets'] return [bucket['Name'] for bucket in buckets]
def list_buckets(self, **kwargs): buckets = list_buckets(**kwargs) return [ bucket['Name'] for bucket in buckets['Buckets'] if not self.check_ignore_list(bucket['Name']) ]
def list_buckets(self, **kwargs): buckets = list_buckets(**kwargs) return [bucket['Name'] for bucket in buckets['Buckets'] if not self.check_ignore_list(bucket['Name'])]