def test_print_rule_stats(self, log_mock):
"""Stats - Print Rule Stats"""
self._timed_func_helper()
stats.print_rule_stats()
log_mock.assert_called_with(
'Rule statistics:\n%s',
'test_rule 10.00000000 ms 1 calls 10.00000000 avg')
def run(self, records):
"""Run rules against the records sent from the Classifier function
Args:
records (list): Dictionaries of records sent from the classifier function
Record Format:
{
'cluster': 'prod',
'log_schema_type': 'cloudwatch:cloudtrail',
'record': {
'key': 'value'
},
'service': 'kinesis',
'resource': 'kinesis_stream_name'
'data_type': 'json'
}
Returns:
list: Alerts that have been triggered by this data
"""
LOGGER.info('Processing %d records', len(records))
# Extract any threat intelligence matches from the records
self._extract_threat_intel(records)
alerts = []
for payload in records:
rules = Rule.rules_for_log_type(payload['log_schema_type'])
if not rules:
LOGGER.debug('No rules to process for %s', payload)
continue
for rule in rules:
# subkey check
if not self._process_subkeys(payload['record'], rule):
continue
# matcher check
if not rule.check_matchers(payload['record']):
continue
alert = self._rule_analysis(payload, rule)
if alert:
alerts.append(alert)
self._alert_forwarder.send_alerts(alerts)
# Only log rule info here if this is deployed in Lambda
# During testing, this gets logged at the end and printing here could be confusing
# since stress testing calls this method multiple times
if self._in_lambda:
print_rule_stats(True)
MetricLogger.log_metric(FUNCTION_NAME, MetricLogger.TRIGGERED_ALERTS, len(alerts))
return alerts
def test_print_rule_stats_reset(self, ):
"""Stats - Print Rule Stats, Reset"""
self._timed_func_helper()
stats.print_rule_stats(True)
assert_equal(len(stats.RULE_STATS), 0)
def test_print_rule_stats_empty(self, log_mock):
"""Stats - Print Rule Stats, None"""
stats.print_rule_stats()
log_mock.assert_called_with('No rule statistics to print')
def stream_alert_test(options, config):
"""High level function to wrap the integration testing entry point.
This encapsulates the testing function and is used to specify if calls
should be mocked.
Args:
options (namedtuple): CLI options (debug, processor, etc)
config (CLIConfig): Configuration for this StreamAlert setup that
includes cluster info, etc that can be used for constructing
an aws context object
"""
# get the options in a dictionary so we can do easy lookups
run_options = vars(options)
context = helpers.get_context_from_config(run_options.get('cluster'),
config)
@helpers.mock_me(context)
def run_tests(options, context):
"""Actual protected function for running tests
Args:
options (namedtuple): CLI options (debug, processor, etc)
context (namedtuple): A constructed aws context object
"""
# The Rule Processor and Alert Processor need environment variables for many things
prefix = config['global']['account']['prefix']
alerts_table = '{}_streamalert_alerts'.format(prefix)
os.environ[
'ALERT_PROCESSOR'] = '{}_streamalert_alert_processor'.format(
prefix)
os.environ['ALERTS_TABLE'] = alerts_table
os.environ['AWS_DEFAULT_REGION'] = config['global']['account'][
'region']
os.environ['CLUSTER'] = run_options.get('cluster') or ''
if not options.debug:
# Add a filter to suppress a few noisy log messages
LOGGER_SA.addFilter(SuppressNoise())
# Create an in memory logging buffer to be used to caching all error messages
log_mem_handler = get_log_memory_handler()
# Check if the rule processor should be run for these tests
# Using NOT set.isdisjoint will check to see if there are commonalities between
# the options in 'processor' and {'rule', 'all'}
test_rules = (not run_options.get('processor').isdisjoint(
{'rule', 'all'}) if run_options.get('processor') else
run_options.get('command') == 'live-test'
or run_options.get('command') == 'validate-schemas')
# Check if the alert processor should be run for these tests
# Using NOT set.isdisjoint will check to see if there are commonalities between
# the options in 'processor' and {'alert', 'all'}
test_alerts = (not run_options.get('processor').isdisjoint(
{'alert', 'all'}) if run_options.get('processor') else
run_options.get('command') == 'live-test')
validate_schemas = options.command == 'validate-schemas'
rules_filter = run_options.get('rules', {})
files_filter = run_options.get('files', {})
# Run the rule processor for all rules or designated rule set
if context.mocked:
helpers.setup_mock_alerts_table(alerts_table)
# Mock S3 bucket for lookup tables testing
helpers.mock_s3_bucket(config)
rule_proc_tester = RuleProcessorTester(context, config, test_rules)
alert_proc_tester = AlertProcessorTester(config, context)
for _ in range(run_options.get('repeat', 1)):
for alerts in rule_proc_tester.test_processor(
rules_filter, files_filter, validate_schemas):
# If the alert processor should be tested, process any alerts
if test_alerts:
alert_proc_tester.test_processor(alerts)
# Report summary information for the alert processor if it was ran
if test_alerts:
AlertProcessorTester.report_output_summary()
all_test_rules = None
if rules_filter:
all_test_rules = helpers.get_rules_from_test_events(
TEST_EVENTS_DIR)
check_invalid_rules_filters(rules_filter, all_test_rules)
# If this is not just a validation run, and rule/file filters are not in place
# then warn the user if there are test files without corresponding rules
# Also check all of the rule files to make sure they have tests configured
if not (validate_schemas or rules_filter or files_filter):
all_test_rules = all_test_rules or helpers.get_rules_from_test_events(
TEST_EVENTS_DIR)
check_untested_files(all_test_rules)
check_untested_rules(all_test_rules)
if not (rule_proc_tester.all_tests_passed
and alert_proc_tester.all_tests_passed):
return 1 # will exit with error
# If there are any log records in the memory buffer, then errors occurred somewhere
if log_mem_handler.buffer:
# Release the MemoryHandler so we can do some other logging now
logging.getLogger().removeHandler(log_mem_handler)
LOGGER_CLI.error(
'%sSee %d miscellaneous error(s) below '
'that were encountered and may need to be addressed%s',
COLOR_RED, len(log_mem_handler.buffer), COLOR_RESET)
log_mem_handler.setTarget(LOGGER_CLI)
log_mem_handler.flush()
return 1 # will exit with error
return 0 # will exit without error
result = run_tests(options, context)
if run_options.get('stats'):
stats.print_rule_stats()
sys.exit(result)
def run(self, event):
"""StreamAlert Lambda function handler.
Loads the configuration for the StreamAlert function which contains
available data sources, log schemas, normalized types, and outputs.
Classifies logs sent into a parsed type.
Matches records against rules.
Args:
event (dict): An AWS event mapped to a specific source/entity
containing data read by Lambda.
Returns:
bool: True if all logs being parsed match a schema
"""
records = event.get('Records', [])
LOGGER.debug('Number of incoming records: %d', len(records))
if not records:
return False
firehose_config = self.config['global'].get('infrastructure',
{}).get('firehose', {})
if firehose_config.get('enabled'):
self._firehose_client = StreamAlertFirehose(
self.env['lambda_region'], firehose_config,
self.config['logs'])
payload_with_normalized_records = []
for raw_record in records:
# Get the service and entity from the payload. If the service/entity
# is not in our config, log and error and go onto the next record
service, entity = self.classifier.extract_service_and_entity(
raw_record)
if not service:
LOGGER.error(
'No valid service found in payload\'s raw record. Skipping '
'record: %s', raw_record)
continue
if not entity:
LOGGER.error(
'Unable to extract entity from payload\'s raw record for service %s. '
'Skipping record: %s', service, raw_record)
continue
# Cache the log sources for this service and entity on the classifier
if not self.classifier.load_sources(service, entity):
continue
# Create the StreamPayload to use for encapsulating parsed info
payload = load_stream_payload(service, entity, raw_record)
if not payload:
continue
payload_with_normalized_records.extend(
self._process_alerts(payload))
# Log normalized records metric
MetricLogger.log_metric(FUNCTION_NAME, MetricLogger.NORMALIZED_RECORDS,
len(payload_with_normalized_records))
# Apply Threat Intel to normalized records in the end of Rule Processor invocation
record_alerts = self._rules_engine.threat_intel_match(
payload_with_normalized_records)
self._alerts.extend(record_alerts)
if record_alerts:
self.alert_forwarder.send_alerts(record_alerts)
MetricLogger.log_metric(FUNCTION_NAME, MetricLogger.TOTAL_RECORDS,
self._processed_record_count)
MetricLogger.log_metric(FUNCTION_NAME,
MetricLogger.TOTAL_PROCESSED_SIZE,
self._processed_size)
LOGGER.debug('Invalid record count: %d', self._failed_record_count)
MetricLogger.log_metric(FUNCTION_NAME, MetricLogger.FAILED_PARSES,
self._failed_record_count)
LOGGER.debug('%s alerts triggered', len(self._alerts))
MetricLogger.log_metric(FUNCTION_NAME, MetricLogger.TRIGGERED_ALERTS,
len(self._alerts))
# Check if debugging logging is on before json dumping alerts since
# this can be time consuming if there are a lot of alerts
if self._alerts and LOGGER.isEnabledFor(LOG_LEVEL_DEBUG):
LOGGER.debug(
'Alerts:\n%s',
json.dumps([alert.output_dict() for alert in self._alerts],
indent=2,
sort_keys=True))
if self._firehose_client:
self._firehose_client.send()
# Only log rule info here if this is not running tests
# During testing, this gets logged at the end and printing here could be confusing
# since stress testing calls this method multiple times
if self.env['lambda_alias'] != 'development':
stats.print_rule_stats(True)
return self._failed_record_count == 0
