def generate_athena(self):
"""Generate a base Athena config"""
if 'athena_partition_refresh_config' in self.config['lambda']:
LOGGER_CLI.warn('The Athena configuration already exists, skipping.')
return
athena_config_template = {
'enabled': True,
'enable_metrics': False,
'current_version': '$LATEST',
'refresh_type': {
'add_hive_partition': {},
'repair_hive_table': {}
},
'handler': 'stream_alert.athena_partition_refresh.main.handler',
'timeout': '60',
'memory': '128',
'log_level': 'info',
'source_bucket': 'PREFIX_GOES_HERE.streamalert.source',
'source_current_hash': '<auto_generated>',
'source_object_key': '<auto_generated>',
'third_party_libraries': []
}
# Check if the prefix has ever been set
if self.config['global']['account']['prefix'] != 'PREFIX_GOES_HERE':
athena_config_template['source_bucket'] = self.config['lambda'] \
['rule_processor_config']['source_bucket']
self.config['lambda']['athena_partition_refresh_config'] = athena_config_template
self.write()
LOGGER_CLI.info('Athena configuration successfully created')
def generate_athena(self):
"""Generate a base Athena config"""
if 'athena_partition_refresh_config' in self.config['lambda']:
LOGGER_CLI.warn(
'The Athena configuration already exists, skipping.')
return
athena_config_template = {
'enabled': True,
'current_version': '$LATEST',
'refresh_type': {
'add_hive_partition': {},
'repair_hive_table': {}
},
'handler': 'stream_alert.athena_partition_refresh.main.handler',
'timeout': '60',
'memory': '128',
'source_bucket': 'PREFIX_GOES_HERE.streamalert.source',
'source_current_hash': '<auto_generated>',
'source_object_key': '<auto_generated',
'third_party_libraries': ['backoff']
}
if self.config['global']['account']['prefix'] != 'PREFIX_GOES_HERE':
athena_config_template['source_bucket'] = athena_config_template[
'source_bucket'].replace(
'PREFIX_GOES_HERE',
self.config['global']['account']['prefix'])
self.config['lambda'][
'athena_partition_refresh_config'] = athena_config_template
self.write()
def generate_athena(self):
"""Generate a base Athena config"""
if 'athena_partition_refresh_config' in self.config['lambda']:
LOGGER_CLI.warn(
'The Athena configuration already exists, skipping.')
return
prefix = self.config['global']['account']['prefix']
athena_config_template = {
'enable_metrics': False,
'buckets': {
'{}.streamalerts'.format(prefix): 'alert'
},
'timeout': '60',
'memory': '128',
'log_level': 'info',
'third_party_libraries': []
}
self.config['lambda'][
'athena_partition_refresh_config'] = athena_config_template
self.write()
LOGGER_CLI.info('Athena configuration successfully created')
def generate_athena(self):
"""Generate a base Athena config"""
if 'athena_partition_refresh_config' in self.config['lambda']:
LOGGER_CLI.warn(
'The Athena configuration already exists, skipping.')
return
prefix = self.config['global']['account']['prefix']
athena_config_template = {
'enable_metrics': False,
'current_version': '$LATEST',
'buckets': {
'{}.streamalerts'.format(prefix): 'alert'
},
'handler': 'stream_alert.athena_partition_refresh.main.handler',
'timeout': '60',
'memory': '128',
'log_level': 'info',
'source_bucket': '{}.streamalert.source'.format(prefix),
'source_current_hash': '<auto_generated>',
'source_object_key': '<auto_generated>',
'third_party_libraries': []
}
self.config['lambda'][
'athena_partition_refresh_config'] = athena_config_template
self.write()
LOGGER_CLI.info('Athena configuration successfully created')
def _try_decrement_version(lambda_config, function_name):
"""Log a warning if the lambda version cannot be rolled back."""
changed = _decrement_version(lambda_config)
if not changed:
LOGGER_CLI.warn('%s cannot be rolled back from version %s',
function_name, str(lambda_config['current_version']))
return changed
def _rollback_production(lambda_client, function_name):
"""Rollback the production alias for the given function name."""
version = lambda_client.get_alias(FunctionName=function_name,
Name='production')['FunctionVersion']
if version == '$LATEST':
# This won't happen with Terraform, but the alias could have been manually changed.
LOGGER_CLI.error(
'%s:production is pointing to $LATEST instead of a published version',
function_name)
return
current_version = int(version)
if current_version == 1:
LOGGER_CLI.warn('%s:production is already at version 1', function_name)
return
LOGGER_CLI.info('Rolling back %s:production from version %d => %d',
function_name, current_version, current_version - 1)
try:
lambda_client.update_alias(FunctionName=function_name,
Name='production',
FunctionVersion=str(current_version - 1))
except ClientError:
LOGGER_CLI.exception('version not updated')
def check_invalid_rules_filters(rules_filter, all_test_rules):
"""Log warning message for filtered rules that do not exist.
Args:
rules_filter (set): A collection of rules to filter on, passed in by the user
all_test_rules (set): A collection of all of the rules being tested
"""
invalid_rules = rules_filter.difference(all_test_rules)
if invalid_rules:
LOGGER_CLI.warn('%sNo test events configured for the following rules being filtered. '
'This error could also be caused by a typo in the list of filtered '
'rules\n\t%s%s', COLOR_YELLOW, '\n\t'.join(invalid_rules), COLOR_RESET)
def check_untested_rules(all_test_rules):
"""Log warning message for rules that exist but do not have proper test events.
Args:
all_test_rules (set): A collection of all of the rules being tested
"""
untested_rules = set(rule.Rule.rule_names()).difference(all_test_rules)
if untested_rules:
LOGGER_CLI.warn('%sNo test events configured for the following rules. Please add '
'corresponding tests for these rules in \'%s\' to avoid seeing '
'this warning\n\t%s%s', COLOR_YELLOW, TEST_EVENTS_DIR,
'\n\t'.join(untested_rules), COLOR_RESET)
def check_untested_files(all_test_rules):
"""Log warning message for test events that exist with invalid rule names.
Args:
all_test_rules (set): A collection of all of the rules being tested
"""
invalid_rules = all_test_rules.difference(set(rule.Rule.rule_names()))
if invalid_rules:
LOGGER_CLI.warn('%sNo rules found in \'rules/\' that match the rules declared within '
'\'trigger_rules\' in a test event. Please update the list of '
'\'trigger_rules\' with valid rule names to avoid seeing this '
'warning and any associated errors '
'above\n\t%s%s', COLOR_YELLOW, '\n\t'.join(invalid_rules), COLOR_RESET)
def check_untested_rules():
"""Function that prints warning log messages for rules that exist but do
not have proper integration tests configured.
"""
all_test_files = {os.path.splitext(test_file)[0] for _, _, test_rule_files
in os.walk(DIR_RULES) for test_file in test_rule_files}
untested_rules = set(StreamRules.get_rules()).difference(all_test_files)
for rule in untested_rules:
LOGGER_CLI.warn('%sNo tests configured for rule: \'%s\'. Please add a '
'corresponding test file for this rule in \'%s\' with the '
'name \'%s.json\' to avoid seeing this warning%s', COLOR_YELLOW,
rule, DIR_RULES, rule, COLOR_RESET)
def check_untested_files():
"""Function that prints warning log messages for integration test files
that exist but do not have a corresponding rule configured.
"""
all_test_files = {os.path.splitext(test_file)[0] for _, _, test_rule_files
in os.walk(DIR_RULES) for test_file in test_rule_files}
untested_rules = all_test_files.difference(set(StreamRules.get_rules()))
for rule in untested_rules:
LOGGER_CLI.warn('%sNo rules configured for test file: \'%s.json\'. Please '
'add a corresponding rule for this test file in \'rules/\' with '
'the name \'%s.py\' to avoid seeing this warning and any associated '
'errors above%s', COLOR_YELLOW,
rule, rule, COLOR_RESET)
def report_output_summary(self):
"""Helper function to print the summary results of all tests"""
failure_messages = [
item for item in self.status_messages
if item.type == StatusMessage.FAILURE
]
warning_messages = [
item for item in self.status_messages
if item.type == StatusMessage.WARNING
]
passed_tests = sum(1 for item in self.status_messages
if item.type == StatusMessage.SUCCESS)
passed_tests = self.total_tests - len(failure_messages)
# Print some lines at the bottom of output to make it more readable
# This occurs here so there is always space and not only when the
# successful test info prints
print '\n\n'
# Only print success info if we explicitly want to print output
# but always print any errors or warnings below
if self.print_output:
# Print a message indicating how many of the total tests passed
LOGGER_CLI.info('%s(%d/%d) Successful Tests%s', COLOR_GREEN,
passed_tests, self.total_tests, COLOR_RESET)
# Check if there were failed tests and report on them appropriately
if failure_messages:
# Print a message indicating how many of the total tests failed
LOGGER_CLI.error('%s(%d/%d) Failures%s', COLOR_RED,
len(failure_messages), self.total_tests,
COLOR_RESET)
# Iterate over the rule_name values in the failed list and report on them
for index, failure in enumerate(failure_messages, start=1):
LOGGER_CLI.error('%s(%d/%d) [%s] %s%s', COLOR_RED, index,
len(failure_messages), failure.rule,
failure.message, COLOR_RESET)
# Check if there were any warnings and report on them
if warning_messages:
warning_count = len(warning_messages)
LOGGER_CLI.warn('%s%d Warning%s%s', COLOR_YELLOW, warning_count,
('s' if warning_count > 1 else ''), COLOR_RESET)
for index, warning in enumerate(warning_messages, start=1):
LOGGER_CLI.warn('%s(%d/%d) [%s] %s%s', COLOR_YELLOW, index,
warning_count, warning.rule, warning.message,
COLOR_RESET)
