def test_refresh_record(self):
kinesis_data = {
'key3': 'key3data',
'key2': 'key2data',
'key1': 'key1data'
}
kinesis_data_json = json.dumps(kinesis_data)
first_raw_record = self.make_kinesis_record(
kinesis_stream='test_kinesis_stream',
kinesis_data=kinesis_data_json)
payload = StreamPayload(raw_record=first_raw_record,
env=self.env,
config=self.config)
# generate a second record
new_kinesis_data = {
'key4': 'key4data',
'key5': 'key5data',
'key6': 'key6data'
}
new_kinesis_data_json = json.dumps(new_kinesis_data)
second_record = self.make_kinesis_record(
kinesis_stream='test_kinesis_stream',
kinesis_data=new_kinesis_data_json)
payload.refresh_record(second_record)
# check loaded record
assert_equal(payload.raw_record, second_record)
assert_not_equal(payload.raw_record, first_raw_record)
def handler(event, context):
"""StreamAlert Lambda function handler.
Loads the configuration for the StreamAlert function which contains:
available data sources, log formats, parser modes, and sinks. Classifies
logs sent into the stream into a parsed type. Matches records against
rules.
Args:
event: An AWS event mapped to a specific source/entity (kinesis stream or
an s3 bucket event) containing data emitted to the stream.
context: An AWS context object which provides metadata on the currently
executing lambda function.
Returns:
None
"""
logger.debug('Number of Records: %d', len(event.get('Records')))
config = load_config()
env = load_env(context)
# process_alerts(event['Records'])
alerts_to_send = []
# TODO(jack): Move this into classification
for record in event.get('Records'):
payload = StreamPayload(raw_record=record)
classifier = StreamClassifier(config=config)
classifier.map_source(payload)
# If the kinesis stream or s3 bucket is not in our config,
# go onto the next record.
if not payload.valid_source:
continue
if payload.service == 's3':
s3_file_lines = StreamPreParsers.pre_parse_s3(payload.raw_record)
for line in s3_file_lines:
data = line.rstrip()
payload.refresh_record(data)
classifier.classify_record(payload, data)
process_alerts(payload, alerts_to_send)
elif payload.service == 'kinesis':
data = StreamPreParsers.pre_parse_kinesis(payload.raw_record)
classifier.classify_record(payload, data)
process_alerts(payload, alerts_to_send)
if alerts_to_send:
if env['lambda_alias'] == 'development':
logger.info('%s alerts triggered', len(alerts_to_send))
for alert in alerts_to_send:
logger.info(alert)
StreamSink(alerts_to_send, config, env).sink()
else:
logger.debug('Valid data, no alerts: %s', payload)