class StreamAlert(object):
"""Wrapper class for handling StreamAlert classificaiton and processing"""
config = {}
# Used to detect special characters in payload keys.
# This is necessary for sanitization of data prior to searching in Athena.
special_char_regex = re.compile(r'\W')
special_char_sub = '_'
def __init__(self, context, enable_alert_processor=True):
"""Initializer
Args:
context (dict): An AWS context object which provides metadata on the currently
executing lambda function.
enable_alert_processor (bool): If the user wants to send the alerts using their
own methods, 'enable_alert_processor' can be set to False to suppress
sending with the StreamAlert alert processor.
"""
# Load the config. Validation occurs during load, which will
# raise exceptions on any ConfigErrors
StreamAlert.config = StreamAlert.config or load_config()
# Load the environment from the context arn
self.env = load_env(context)
# Instantiate the sink here to handle sending the triggered alerts to the
# alert processor
self.sinker = StreamSink(self.env)
# Instantiate a classifier that is used for this run
self.classifier = StreamClassifier(config=self.config)
self.enable_alert_processor = enable_alert_processor
self._failed_record_count = 0
self._processed_size = 0
self._alerts = []
# Create a dictionary to hold parsed payloads by log type.
# Firehose needs this information to send to its corresponding
# delivery stream.
self.categorized_payloads = defaultdict(list)
# Firehose client initialization
self.firehose_client = None
StreamThreatIntel.load_intelligence(self.config)
def run(self, event):
"""StreamAlert Lambda function handler.
Loads the configuration for the StreamAlert function which contains
available data sources, log schemas, normalized types, and outputs.
Classifies logs sent into a parsed type.
Matches records against rules.
Args:
event (dict): An AWS event mapped to a specific source/entity
containing data read by Lambda.
Returns:
bool: True if all logs being parsed match a schema
"""
records = event.get('Records', [])
LOGGER.debug('Number of Records: %d', len(records))
if not records:
return False
MetricLogger.log_metric(FUNCTION_NAME, MetricLogger.TOTAL_RECORDS, len(records))
firehose_config = self.config['global'].get(
'infrastructure', {}).get('firehose', {})
if firehose_config.get('enabled'):
self.firehose_client = boto3.client('firehose',
region_name=self.env['lambda_region'])
for raw_record in records:
# Get the service and entity from the payload. If the service/entity
# is not in our config, log and error and go onto the next record
service, entity = self.classifier.extract_service_and_entity(raw_record)
if not service:
LOGGER.error('No valid service found in payload\'s raw record. Skipping '
'record: %s', raw_record)
continue
if not entity:
LOGGER.error(
'Unable to extract entity from payload\'s raw record for service %s. '
'Skipping record: %s', service, raw_record)
continue
# Cache the log sources for this service and entity on the classifier
if not self.classifier.load_sources(service, entity):
continue
# Create the StreamPayload to use for encapsulating parsed info
payload = load_stream_payload(service, entity, raw_record)
if not payload:
continue
self._process_alerts(payload)
MetricLogger.log_metric(FUNCTION_NAME,
MetricLogger.TOTAL_PROCESSED_SIZE,
self._processed_size)
LOGGER.debug('Invalid record count: %d', self._failed_record_count)
MetricLogger.log_metric(FUNCTION_NAME,
MetricLogger.FAILED_PARSES,
self._failed_record_count)
LOGGER.debug('%s alerts triggered', len(self._alerts))
MetricLogger.log_metric(
FUNCTION_NAME, MetricLogger.TRIGGERED_ALERTS, len(
self._alerts))
# Check if debugging logging is on before json dumping alerts since
# this can be time consuming if there are a lot of alerts
if self._alerts and LOGGER.isEnabledFor(LOG_LEVEL_DEBUG):
LOGGER.debug('Alerts:\n%s', json.dumps(self._alerts, indent=2))
if self.firehose_client:
self._send_to_firehose()
return self._failed_record_count == 0
def get_alerts(self):
"""Public method to return alerts from class. Useful for testing.
Returns:
list: list of alerts as dictionaries
"""
return self._alerts
@staticmethod
def _segment_records_by_count(record_list, max_count):
"""Segment records by length
Args:
record_list (list): The original records list to be segmented
max_count (int): The max amount of records to yield per group
"""
for index in range(0, len(record_list), max_count):
yield record_list[index:index + max_count]
def _segment_records_by_size(self, record_batch):
"""Segment record groups by size
Args:
record_batch (list): The original record batch to measure and segment
Returns:
generator: Used to iterate on each newly segmented group
"""
split_factor = 1
len_batch = len(record_batch)
# Sample the first batch of records to determine the split factor.
# Generally, it's very rare for a group of records to have
# drastically different sizes in a single Lambda invocation.
while len(json.dumps(record_batch[:len_batch / split_factor],
separators=(",", ":"))) > MAX_BATCH_SIZE:
split_factor += 1
return self._segment_records_by_count(record_batch, len_batch / split_factor)
@staticmethod
def _limit_record_size(batch):
"""Limit the record size to be sent to Firehose
Args:
batch (list): Record batch to iterate on
"""
for index, record in enumerate(batch):
if len(json.dumps(record, separators=(",", ":"))) > MAX_RECORD_SIZE:
# Show the first 1k bytes in order to not overload
# CloudWatch logs
LOGGER.error('The following record is too large'
'be sent to Firehose: %s', str(record)[:1000])
MetricLogger.log_metric(FUNCTION_NAME,
MetricLogger.FIREHOSE_FAILED_RECORDS,
1)
batch.pop(index)
@classmethod
def sanitize_keys(cls, record):
"""Remove special characters from parsed record keys
This is required when searching in Athena. Keys can only have
a period or underscore
Args:
record (dict): Original parsed record
Returns:
dict: A sanitized record
"""
new_record = {}
for key, value in record.iteritems():
sanitized_key = re.sub(cls.special_char_regex,
cls.special_char_sub,
key)
# Handle nested objects
if isinstance(value, dict):
new_record[sanitized_key] = cls.sanitize_keys(record[key])
else:
new_record[sanitized_key] = record[key]
return new_record
def _firehose_request_helper(self, stream_name, record_batch):
"""Send record batches to Firehose
Args:
stream_name (str): The name of the Delivery Stream to send to
record_batch (list): The records to send
"""
record_batch_size = len(record_batch)
resp = {}
try:
LOGGER.debug('Sending %d records to Firehose:%s',
record_batch_size,
stream_name)
resp = self.firehose_client.put_record_batch(
DeliveryStreamName=stream_name,
# The newline at the end is required by Firehose,
# otherwise all records will be on a single line and
# unsearchable in Athena.
Records=[{'Data': json.dumps(self.sanitize_keys(record),
separators=(",", ":")) + '\n'}
for record
in record_batch])
except ClientError as firehose_err:
LOGGER.error(firehose_err)
MetricLogger.log_metric(FUNCTION_NAME,
MetricLogger.FIREHOSE_FAILED_RECORDS,
record_batch_size)
return
# Error handle if failures occured in PutRecordBatch
# TODO(jack) implement backoff here for additional message reliability
if resp.get('FailedPutCount') > 0:
failed_records = [failed
for failed
in resp['RequestResponses']
if failed.get('ErrorCode')]
MetricLogger.log_metric(FUNCTION_NAME,
MetricLogger.FIREHOSE_FAILED_RECORDS,
resp['FailedPutCount'])
# Only print the first 100 failed records to Cloudwatch logs
LOGGER.error('The following records failed to Put to the'
'Delivery stream %s: %s',
stream_name,
json.dumps(failed_records[:100], indent=2))
else:
MetricLogger.log_metric(FUNCTION_NAME,
MetricLogger.FIREHOSE_RECORDS_SENT,
record_batch_size)
LOGGER.info('Successfully sent %d messages to Firehose:%s',
record_batch_size,
stream_name)
def _send_to_firehose(self):
"""Send all classified records to a respective Firehose Delivery Stream"""
delivery_stream_name_pattern = 'streamalert_data_{}'
# Iterate through each payload type
for log_type, records in self.categorized_payloads.items():
# This same method is used when naming the Delivery Streams
formatted_log_type = log_type.replace(':', '_')
for record_batch in self._segment_records_by_count(records, MAX_BATCH_COUNT):
stream_name = delivery_stream_name_pattern.format(formatted_log_type)
self._limit_record_size(record_batch)
for sized_batch in self._segment_records_by_size(record_batch):
self._firehose_request_helper(stream_name, sized_batch)
def _process_alerts(self, payload):
"""Process records for alerts and send them to the correct places
Args:
payload (StreamPayload): StreamAlert payload object being processed
"""
for record in payload.pre_parse():
# Increment the processed size using the length of this record
self._processed_size += len(record.pre_parsed_record)
self.classifier.classify_record(record)
if not record.valid:
if self.env['lambda_alias'] != 'development':
LOGGER.error('Record does not match any defined schemas: %s\n%s',
record, record.pre_parsed_record)
self._failed_record_count += 1
continue
LOGGER.debug(
'Classified and Parsed Payload: <Valid: %s, Log Source: %s, Entity: %s>',
record.valid,
record.log_source,
record.entity)
record_alerts = StreamRules.process(record)
LOGGER.debug('Processed %d valid record(s) that resulted in %d alert(s).',
len(payload.records),
len(record_alerts))
# Add all parsed records to the categorized payload dict
# only if Firehose is enabled
if self.firehose_client:
# Only send payloads with enabled types
if payload.log_source.split(':')[0] not in self.config['global'] \
['infrastructure'].get('firehose', {}).get('disabled_logs', []):
self.categorized_payloads[payload.log_source].extend(payload.records)
if not record_alerts:
continue
# Extend the list of alerts with any new ones so they can be returned
self._alerts.extend(record_alerts)
if self.enable_alert_processor:
self.sinker.sink(record_alerts)
class StreamAlert(object):
"""Wrapper class for handling all StreamAlert classificaiton and processing"""
def __init__(self, context, enable_alert_processor=True):
"""
Args:
context: An AWS context object which provides metadata on the currently
executing lambda function.
enable_alert_processor: If the user wants to send the alerts using their
own methods, 'enable_alert_processor' can be set to False to suppress
sending with the StreamAlert alert processor.
"""
self.env = load_env(context)
self.enable_alert_processor = enable_alert_processor
# Instantiate the sink here to handle sending the triggered alerts to the alert processor
self.sinker = StreamSink(self.env)
self._failed_record_count = 0
self._alerts = []
def run(self, event):
"""StreamAlert Lambda function handler.
Loads the configuration for the StreamAlert function which contains:
available data sources, log formats, parser modes, and sinks. Classifies
logs sent into the stream into a parsed type. Matches records against
rules.
Args:
event: An AWS event mapped to a specific source/entity (kinesis stream or
an s3 bucket event) containing data emitted to the stream.
Returns:
[boolean] True if all logs being parsed match a schema
"""
LOGGER.debug('Number of Records: %d', len(event.get('Records', [])))
config = load_config()
for record in event.get('Records', []):
payload = StreamPayload(raw_record=record)
classifier = StreamClassifier(config=config)
# If the kinesis stream, s3 bucket, or sns topic is not in our config,
# go onto the next record
if not classifier.map_source(payload):
continue
if payload.service == 's3':
self._s3_process(payload, classifier)
elif payload.service == 'kinesis':
self._kinesis_process(payload, classifier)
elif payload.service == 'sns':
self._sns_process(payload, classifier)
else:
LOGGER.error('Unsupported service: %s', payload.service)
LOGGER.debug('%s alerts triggered', len(self._alerts))
LOGGER.debug('\n%s\n', json.dumps(self._alerts, indent=4))
return self._failed_record_count == 0
def get_alerts(self):
"""Public method to return alerts from class. Useful for testing.
Returns:
[list] list of alerts as dictionaries
"""
return self._alerts
def _kinesis_process(self, payload, classifier):
"""Process Kinesis data for alerts"""
data = StreamPreParsers.pre_parse_kinesis(payload.raw_record)
self._process_alerts(classifier, payload, data)
def _s3_process(self, payload, classifier):
"""Process S3 data for alerts"""
s3_file, s3_object_size = StreamPreParsers.pre_parse_s3(payload.raw_record)
count, processed_size = 0, 0
for data in StreamPreParsers.read_s3_file(s3_file):
payload.refresh_record(data)
self._process_alerts(classifier, payload, data)
# Add the current data to the total processed size, +1 to account for line feed
processed_size += (len(data) + 1)
count += 1
# Log an info message on every 100 lines processed
if count % 100 == 0:
avg_record_size = ((processed_size - 1) / count)
approx_record_count = s3_object_size / avg_record_size
LOGGER.info('Processed %s records out of an approximate total of %s '
'(average record size: %s bytes, total size: %s bytes)',
count, approx_record_count, avg_record_size, s3_object_size)
def _sns_process(self, payload, classifier):
"""Process SNS data for alerts"""
data = StreamPreParsers.pre_parse_sns(payload.raw_record)
self._process_alerts(classifier, payload, data)
def _process_alerts(self, classifier, payload, data):
"""Process records for alerts and send them to the correct places
Args:
classifier [StreamClassifier]: Handler for classifying a record's data
payload [StreamPayload]: StreamAlert payload object being processed
data [string]: Pre parsed data string from a raw_event to be parsed
"""
classifier.classify_record(payload, data)
if not payload.valid:
if self.env['lambda_alias'] != 'development':
LOGGER.error('Record does not match any defined schemas: %s\n%s', payload, data)
self._failed_record_count += 1
return
alerts = StreamRules.process(payload)
LOGGER.debug('Processed %d valid record(s) that resulted in %d alert(s).',
len(payload.records),
len(alerts))
if not alerts:
return
# Extend the list of alerts with any new ones so they can be returned
self._alerts.extend(alerts)
if self.enable_alert_processor:
self.sinker.sink(alerts)
class StreamAlert(object):
"""Wrapper class for handling StreamAlert classificaiton and processing"""
__config = {}
def __init__(self, context, enable_alert_processor=True):
"""Initializer
Args:
context (dict): An AWS context object which provides metadata on the currently
executing lambda function.
enable_alert_processor (bool): If the user wants to send the alerts using their
own methods, 'enable_alert_processor' can be set to False to suppress
sending with the StreamAlert alert processor.
"""
# Load the config. Validation occurs during load, which will
# raise exceptions on any ConfigErrors
StreamAlert.__config = StreamAlert.__config or load_config()
# Load the environment from the context arn
self.env = load_env(context)
# Instantiate the sink here to handle sending the triggered alerts to the
# alert processor
self.sinker = StreamSink(self.env)
# Instantiate a classifier that is used for this run
self.classifier = StreamClassifier(config=self.__config)
self.enable_alert_processor = enable_alert_processor
self._failed_record_count = 0
self._processed_size = 0
self._alerts = []
# Create a dictionary to hold parsed payloads by log type.
# Firehose needs this information to send to its corresponding
# delivery stream.
self.categorized_payloads = defaultdict(list)
# Firehose client initialization
self.firehose_client = None
def run(self, event):
"""StreamAlert Lambda function handler.
Loads the configuration for the StreamAlert function which contains
available data sources, log schemas, normalized types, and outputs.
Classifies logs sent into a parsed type.
Matches records against rules.
Args:
event (dict): An AWS event mapped to a specific source/entity
containing data read by Lambda.
Returns:
bool: True if all logs being parsed match a schema
"""
records = event.get('Records', [])
LOGGER.debug('Number of Records: %d', len(records))
if not records:
return False
MetricLogger.log_metric(FUNCTION_NAME, MetricLogger.TOTAL_RECORDS, len(records))
firehose_config = self.__config['global'].get(
'infrastructure', {}).get('firehose', {})
if firehose_config.get('enabled'):
self.firehose_client = boto3.client('firehose',
region_name=self.env['lambda_region'])
for raw_record in records:
# Get the service and entity from the payload. If the service/entity
# is not in our config, log and error and go onto the next record
service, entity = self.classifier.extract_service_and_entity(raw_record)
if not service:
LOGGER.error('No valid service found in payload\'s raw record. Skipping '
'record: %s', raw_record)
continue
if not entity:
LOGGER.error(
'Unable to extract entity from payload\'s raw record for service %s. '
'Skipping record: %s', service, raw_record)
continue
# Cache the log sources for this service and entity on the classifier
if not self.classifier.load_sources(service, entity):
continue
# Create the StreamPayload to use for encapsulating parsed info
payload = load_stream_payload(service, entity, raw_record)
if not payload:
continue
self._process_alerts(payload)
MetricLogger.log_metric(FUNCTION_NAME,
MetricLogger.TOTAL_PROCESSED_SIZE,
self._processed_size)
LOGGER.debug('Invalid record count: %d', self._failed_record_count)
MetricLogger.log_metric(FUNCTION_NAME,
MetricLogger.FAILED_PARSES,
self._failed_record_count)
LOGGER.debug('%s alerts triggered', len(self._alerts))
MetricLogger.log_metric(
FUNCTION_NAME, MetricLogger.TRIGGERED_ALERTS, len(
self._alerts))
# Check if debugging logging is on before json dumping alerts since
# this can be time consuming if there are a lot of alerts
if self._alerts and LOGGER.isEnabledFor(LOG_LEVEL_DEBUG):
LOGGER.debug('Alerts:\n%s', json.dumps(self._alerts, indent=2))
if self.firehose_client:
self._send_to_firehose()
return self._failed_record_count == 0
def get_alerts(self):
"""Public method to return alerts from class. Useful for testing.
Returns:
list: list of alerts as dictionaries
"""
return self._alerts
def _send_to_firehose(self):
"""Send all classified records to a respective Firehose Delivery Stream"""
def _chunk(record_list, chunk_size):
"""Helper function to chunk payloads"""
for item in range(0, len(record_list), chunk_size):
yield record_list[item:item + chunk_size]
def _check_record_batch(batch):
"""Helper function to verify record size"""
for index, record in enumerate(batch):
if len(str(record)) > MAX_RECORD_SIZE:
# Show the first 1k bytes in order to not overload
# CloudWatch logs
LOGGER.error('The following record is too large'
'be sent to Firehose: %s', str(record)[:1000])
MetricLogger.log_metric(FUNCTION_NAME,
MetricLogger.FIREHOSE_FAILED_RECORDS,
1)
batch.pop(index)
delivery_stream_name_pattern = 'streamalert_data_{}'
# Iterate through each payload type
for log_type, records in self.categorized_payloads.items():
# This same method is used when naming the Delivery Streams
formatted_log_type = log_type.replace(':', '_')
for record_batch in _chunk(records, MAX_BATCH_SIZE):
stream_name = delivery_stream_name_pattern.format(formatted_log_type)
_check_record_batch(record_batch)
resp = self.firehose_client.put_record_batch(
DeliveryStreamName=stream_name,
# The newline at the end is required by Firehose,
# otherwise all records will be on a single line and
# unsearchable in Athena.
Records=[{'Data': json.dumps(record, separators=(",", ":")) + '\n'}
for record
in record_batch])
# Error handle if failures occured
# TODO(jack) implement backoff here once the rule processor is split
if resp.get('FailedPutCount') > 0:
failed_records = [failed
for failed
in resp['RequestResponses']
if failed.get('ErrorCode')]
MetricLogger.log_metric(FUNCTION_NAME,
MetricLogger.FIREHOSE_FAILED_RECORDS,
resp['FailedPutCount'])
# Only print the first 100 failed records
LOGGER.error('The following records failed to Put to the'
'Delivery stream %s: %s',
stream_name,
json.dumps(failed_records[:100], indent=2))
else:
MetricLogger.log_metric(FUNCTION_NAME,
MetricLogger.FIREHOSE_RECORDS_SENT,
len(record_batch))
LOGGER.info('Successfully sent %d messages to Firehose:%s',
len(record_batch),
stream_name)
def _process_alerts(self, payload):
"""Process records for alerts and send them to the correct places
Args:
payload (StreamPayload): StreamAlert payload object being processed
"""
for record in payload.pre_parse():
# Increment the processed size using the length of this record
self._processed_size += len(record.pre_parsed_record)
self.classifier.classify_record(record)
if not record.valid:
if self.env['lambda_alias'] != 'development':
LOGGER.error('Record does not match any defined schemas: %s\n%s',
record, record.pre_parsed_record)
self._failed_record_count += 1
continue
LOGGER.debug(
'Classified and Parsed Payload: <Valid: %s, Log Source: %s, Entity: %s>',
record.valid,
record.log_source,
record.entity)
record_alerts = StreamRules.process(record)
LOGGER.debug('Processed %d valid record(s) that resulted in %d alert(s).',
len(payload.records),
len(record_alerts))
# Add all parsed records to the categorized payload dict
# only if Firehose is enabled
if self.firehose_client:
self.categorized_payloads[payload.log_source].extend(payload.records)
if not record_alerts:
continue
# Extend the list of alerts with any new ones so they can be returned
self._alerts.extend(record_alerts)
if self.enable_alert_processor:
self.sinker.sink(record_alerts)
class StreamAlert(object):
"""Wrapper class for handling StreamAlert classificaiton and processing"""
config = {}
def __init__(self, context, enable_alert_processor=True):
"""Initializer
Args:
context (dict): An AWS context object which provides metadata on the currently
executing lambda function.
enable_alert_processor (bool): If the user wants to send the alerts using their
own methods, 'enable_alert_processor' can be set to False to suppress
sending with the StreamAlert alert processor.
"""
# Load the config. Validation occurs during load, which will
# raise exceptions on any ConfigErrors
StreamAlert.config = StreamAlert.config or load_config()
# Load the environment from the context arn
self.env = load_env(context)
# Instantiate the sink here to handle sending the triggered alerts to the
# alert processor
self.sinker = StreamSink(self.env)
# Instantiate a classifier that is used for this run
self.classifier = StreamClassifier(config=self.config)
self.enable_alert_processor = enable_alert_processor
self._failed_record_count = 0
self._processed_record_count = 0
self._processed_size = 0
self._alerts = []
# Create an instance of the StreamRules class that gets cached in the
# StreamAlert class as an instance property
self._rule_engine = StreamRules(self.config)
# Firehose client attribute
self._firehose_client = None
def run(self, event):
"""StreamAlert Lambda function handler.
Loads the configuration for the StreamAlert function which contains
available data sources, log schemas, normalized types, and outputs.
Classifies logs sent into a parsed type.
Matches records against rules.
Args:
event (dict): An AWS event mapped to a specific source/entity
containing data read by Lambda.
Returns:
bool: True if all logs being parsed match a schema
"""
records = event.get('Records', [])
LOGGER.debug('Number of incoming records: %d', len(records))
if not records:
return False
firehose_config = self.config['global'].get('infrastructure',
{}).get('firehose', {})
if firehose_config.get('enabled'):
self._firehose_client = StreamAlertFirehose(
self.env['lambda_region'], firehose_config,
self.config['logs'])
payload_with_normalized_records = []
for raw_record in records:
# Get the service and entity from the payload. If the service/entity
# is not in our config, log and error and go onto the next record
service, entity = self.classifier.extract_service_and_entity(
raw_record)
if not service:
LOGGER.error(
'No valid service found in payload\'s raw record. Skipping '
'record: %s', raw_record)
continue
if not entity:
LOGGER.error(
'Unable to extract entity from payload\'s raw record for service %s. '
'Skipping record: %s', service, raw_record)
continue
# Cache the log sources for this service and entity on the classifier
if not self.classifier.load_sources(service, entity):
continue
# Create the StreamPayload to use for encapsulating parsed info
payload = load_stream_payload(service, entity, raw_record)
if not payload:
continue
payload_with_normalized_records.extend(
self._process_alerts(payload))
# Log normalized records metric
MetricLogger.log_metric(FUNCTION_NAME, MetricLogger.NORMALIZED_RECORDS,
len(payload_with_normalized_records))
# Apply Threat Intel to normalized records in the end of Rule Processor invocation
record_alerts = self._rule_engine.threat_intel_match(
payload_with_normalized_records)
self._alerts.extend(record_alerts)
if record_alerts and self.enable_alert_processor:
self.sinker.sink(record_alerts)
MetricLogger.log_metric(FUNCTION_NAME, MetricLogger.TOTAL_RECORDS,
self._processed_record_count)
MetricLogger.log_metric(FUNCTION_NAME,
MetricLogger.TOTAL_PROCESSED_SIZE,
self._processed_size)
LOGGER.debug('Invalid record count: %d', self._failed_record_count)
MetricLogger.log_metric(FUNCTION_NAME, MetricLogger.FAILED_PARSES,
self._failed_record_count)
LOGGER.debug('%s alerts triggered', len(self._alerts))
MetricLogger.log_metric(FUNCTION_NAME, MetricLogger.TRIGGERED_ALERTS,
len(self._alerts))
# Check if debugging logging is on before json dumping alerts since
# this can be time consuming if there are a lot of alerts
if self._alerts and LOGGER.isEnabledFor(LOG_LEVEL_DEBUG):
LOGGER.debug('Alerts:\n%s', json.dumps(self._alerts, indent=2))
if self._firehose_client:
self._firehose_client.send()
return self._failed_record_count == 0
def get_alerts(self):
"""Public method to return alerts from class. Useful for testing.
Returns:
list: list of alerts as dictionaries
"""
return self._alerts
def _process_alerts(self, payload):
"""Process records for alerts and send them to the correct places
Args:
payload (StreamPayload): StreamAlert payload object being processed
"""
payload_with_normalized_records = []
for record in payload.pre_parse():
# Increment the processed size using the length of this record
self._processed_size += len(record.pre_parsed_record)
self.classifier.classify_record(record)
if not record.valid:
if self.env['lambda_alias'] != 'development':
LOGGER.error(
'Record does not match any defined schemas: %s\n%s',
record, record.pre_parsed_record)
self._failed_record_count += 1
continue
# Increment the total processed records to get an accurate assessment of throughput
self._processed_record_count += len(record.records)
LOGGER.debug(
'Classified and Parsed Payload: <Valid: %s, Log Source: %s, Entity: %s>',
record.valid, record.log_source, record.entity)
record_alerts, normalized_records = self._rule_engine.process(
record)
payload_with_normalized_records.extend(normalized_records)
LOGGER.debug(
'Processed %d valid record(s) that resulted in %d alert(s).',
len(payload.records), len(record_alerts))
# Add all parsed records to the categorized payload dict only if Firehose is enabled
if self._firehose_client:
# Only send payloads with enabled log sources
if self._firehose_client.enabled_log_source(
payload.log_source):
self._firehose_client.categorized_payloads[
payload.log_source].extend(payload.records)
if not record_alerts:
continue
# Extend the list of alerts with any new ones so they can be returned
self._alerts.extend(record_alerts)
if self.enable_alert_processor:
self.sinker.sink(record_alerts)
return payload_with_normalized_records
class StreamAlert(object):
"""Wrapper class for handling all StreamAlert classificaiton and processing"""
def __init__(self, context, return_alerts=False):
"""
Args:
context: An AWS context object which provides metadata on the currently
executing lambda function.
return_alerts: If the user wants to handle the sinking
of alerts to external endpoints, return a list of
generated alerts.
"""
self.return_alerts = return_alerts
self.env = load_env(context)
# Instantiate the sink here to handle sending the triggered alerts to the alert processor
self.sinker = StreamSink(self.env)
self.alerts = []
def run(self, event):
"""StreamAlert Lambda function handler.
Loads the configuration for the StreamAlert function which contains:
available data sources, log formats, parser modes, and sinks. Classifies
logs sent into the stream into a parsed type. Matches records against
rules.
Args:
event: An AWS event mapped to a specific source/entity (kinesis stream or
an s3 bucket event) containing data emitted to the stream.
Returns:
None
"""
LOGGER.debug('Number of Records: %d', len(event.get('Records', [])))
config = load_config()
for record in event.get('Records', []):
payload = StreamPayload(raw_record=record)
classifier = StreamClassifier(config=config)
# If the kinesis stream, s3 bucket, or sns topic is not in our config,
# go onto the next record
if not classifier.map_source(payload):
continue
if payload.service == 's3':
self._s3_process(payload, classifier)
elif payload.service == 'kinesis':
self._kinesis_process(payload, classifier)
elif payload.service == 'sns':
self._sns_process(payload, classifier)
else:
LOGGER.info('Unsupported service: %s', payload.service)
LOGGER.debug('%s alerts triggered', len(self.alerts))
LOGGER.debug('\n%s\n', json.dumps(self.alerts, indent=4))
if self.return_alerts:
return self.alerts
def _kinesis_process(self, payload, classifier):
"""Process Kinesis data for alerts"""
data = StreamPreParsers.pre_parse_kinesis(payload.raw_record)
self._process_alerts(classifier, payload, data)
def _s3_process(self, payload, classifier):
"""Process S3 data for alerts"""
s3_file, s3_object_size = StreamPreParsers.pre_parse_s3(
payload.raw_record)
count, processed_size = 0, 0
for data in StreamPreParsers.read_s3_file(s3_file):
payload.refresh_record(data)
self._process_alerts(classifier, payload, data)
# Add the current data to the total processed size, +1 to account for line feed
processed_size += (len(data) + 1)
count += 1
# Log an info message on every 100 lines processed
if count % 100 == 0:
avg_record_size = ((processed_size - 1) / count)
approx_record_count = s3_object_size / avg_record_size
LOGGER.info(
'Processed %s records out of an approximate total of %s '
'(average record size: %s bytes, total size: %s bytes)',
count, approx_record_count, avg_record_size,
s3_object_size)
def _sns_process(self, payload, classifier):
"""Process SNS data for alerts"""
data = StreamPreParsers.pre_parse_sns(payload.raw_record)
self._process_alerts(classifier, payload, data)
def _process_alerts(self, classifier, payload, data):
"""Process records for alerts and send them to the correct places
Args:
classifier [StreamClassifier]: Handler for classifying a record's data
payload [StreamPayload]: StreamAlert payload object being processed
data [string]: Pre parsed data string from a raw_event to be parsed
"""
classifier.classify_record(payload, data)
if not payload.valid:
LOGGER.error('Invalid data: %s\n%s', payload,
json.dumps(payload.raw_record, indent=4))
return
alerts = StreamRules.process(payload)
if not alerts:
LOGGER.debug('Valid data, no alerts')
return
# If we want alerts returned to the caller, extend the list. Otherwise
# attempt to send them to the alert processor
if self.return_alerts:
self.alerts.extend(alerts)
else:
self.sinker.sink(alerts)
class StreamAlert(object):
"""Wrapper class for handling all StreamAlert classificaiton and processing"""
def __init__(self, context, enable_alert_processor=True):
"""
Args:
context: An AWS context object which provides metadata on the currently
executing lambda function.
enable_alert_processor (bool): If the user wants to send the alerts using their
own methods, 'enable_alert_processor' can be set to False to suppress
sending with the StreamAlert alert processor.
"""
# Load the config. Validation occurs during load, which will
# raise exceptions on any ConfigErrors
config = load_config()
# Load the environment from the context arn
self.env = load_env(context)
# Instantiate the sink here to handle sending the triggered alerts to the
# alert processor
self.sinker = StreamSink(self.env)
# Instantiate a classifier that is used for this run
self.classifier = StreamClassifier(config=config)
self.enable_alert_processor = enable_alert_processor
self._failed_record_count = 0
self._alerts = []
def run(self, event):
"""StreamAlert Lambda function handler.
Loads the configuration for the StreamAlert function which contains:
available data sources, log formats, parser modes, and sinks. Classifies
logs sent into the stream into a parsed type. Matches records against
rules.
Args:
event: An AWS event mapped to a specific source/entity (kinesis stream or
an s3 bucket event) containing data emitted to the stream.
Returns:
bool: True if all logs being parsed match a schema
"""
records = event.get('Records', [])
LOGGER.debug('Number of Records: %d', len(records))
if not records:
return False
MetricLogger.log_metric(FUNCTION_NAME, MetricLogger.TOTAL_RECORDS,
len(records))
for raw_record in records:
# Get the service and entity from the payload. If the service/entity
# is not in our config, log and error and go onto the next record
service, entity = self.classifier.extract_service_and_entity(
raw_record)
if not service:
LOGGER.error(
'No valid service found in payload\'s raw record. Skipping '
'record: %s', raw_record)
continue
if not entity:
LOGGER.error(
'Unable to extract entity from payload\'s raw record for service %s. '
'Skipping record: %s', service, raw_record)
continue
# Cache the log sources for this service and entity on the classifier
if not self.classifier.load_sources(service, entity):
continue
# Create the StreamPayload to use for encapsulating parsed info
payload = load_stream_payload(service, entity, raw_record)
if not payload:
continue
self._process_alerts(payload)
LOGGER.debug('Invalid record count: %d', self._failed_record_count)
MetricLogger.log_metric(FUNCTION_NAME, MetricLogger.FAILED_PARSES,
self._failed_record_count)
LOGGER.debug('%s alerts triggered', len(self._alerts))
MetricLogger.log_metric(FUNCTION_NAME, MetricLogger.TRIGGERED_ALERTS,
len(self._alerts))
# Check if debugging logging is on before json dumping alerts since
# this can be time consuming if there are a lot of alerts
if self._alerts and LOGGER.isEnabledFor(LOG_LEVEL_DEBUG):
LOGGER.debug('Alerts:\n%s', json.dumps(self._alerts, indent=2))
return self._failed_record_count == 0
def get_alerts(self):
"""Public method to return alerts from class. Useful for testing.
Returns:
list: list of alerts as dictionaries
"""
return self._alerts
def _process_alerts(self, payload):
"""Process records for alerts and send them to the correct places
Args:
payload (StreamPayload): StreamAlert payload object being processed
"""
for record in payload.pre_parse():
self.classifier.classify_record(record)
if not record.valid:
if self.env['lambda_alias'] != 'development':
LOGGER.error(
'Record does not match any defined schemas: %s\n%s',
record, record.pre_parsed_record)
self._failed_record_count += 1
continue
LOGGER.debug(
'Classified and Parsed Payload: <Valid: %s, Log Source: %s, Entity: %s>',
record.valid, record.log_source, record.entity)
record_alerts = StreamRules.process(record)
LOGGER.debug(
'Processed %d valid record(s) that resulted in %d alert(s).',
len(payload.records), len(record_alerts))
if not record_alerts:
continue
# Extend the list of alerts with any new ones so they can be returned
self._alerts.extend(record_alerts)
if self.enable_alert_processor:
self.sinker.sink(record_alerts)