def generate_athena(self): """Generate a base Athena config""" if 'athena_partition_refresh_config' in self.config['lambda']: LOGGER_CLI.warn('The Athena configuration already exists, skipping.') return athena_config_template = { 'enabled': True, 'enable_metrics': False, 'current_version': '$LATEST', 'refresh_type': { 'add_hive_partition': {}, 'repair_hive_table': {} }, 'handler': 'stream_alert.athena_partition_refresh.main.handler', 'timeout': '60', 'memory': '128', 'log_level': 'info', 'source_bucket': 'PREFIX_GOES_HERE.streamalert.source', 'source_current_hash': '<auto_generated>', 'source_object_key': '<auto_generated>', 'third_party_libraries': [] } # Check if the prefix has ever been set if self.config['global']['account']['prefix'] != 'PREFIX_GOES_HERE': athena_config_template['source_bucket'] = self.config['lambda'] \ ['rule_processor_config']['source_bucket'] self.config['lambda']['athena_partition_refresh_config'] = athena_config_template self.write() LOGGER_CLI.info('Athena configuration successfully created')
def generate_athena(self): """Generate a base Athena config""" if 'athena_partition_refresh_config' in self.config['lambda']: LOGGER_CLI.warn( 'The Athena configuration already exists, skipping.') return athena_config_template = { 'enabled': True, 'current_version': '$LATEST', 'refresh_type': { 'add_hive_partition': {}, 'repair_hive_table': {} }, 'handler': 'stream_alert.athena_partition_refresh.main.handler', 'timeout': '60', 'memory': '128', 'source_bucket': 'PREFIX_GOES_HERE.streamalert.source', 'source_current_hash': '<auto_generated>', 'source_object_key': '<auto_generated', 'third_party_libraries': ['backoff'] } if self.config['global']['account']['prefix'] != 'PREFIX_GOES_HERE': athena_config_template['source_bucket'] = athena_config_template[ 'source_bucket'].replace( 'PREFIX_GOES_HERE', self.config['global']['account']['prefix']) self.config['lambda'][ 'athena_partition_refresh_config'] = athena_config_template self.write()
def generate_athena(self): """Generate a base Athena config""" if 'athena_partition_refresh_config' in self.config['lambda']: LOGGER_CLI.warn( 'The Athena configuration already exists, skipping.') return prefix = self.config['global']['account']['prefix'] athena_config_template = { 'enable_metrics': False, 'buckets': { '{}.streamalerts'.format(prefix): 'alert' }, 'timeout': '60', 'memory': '128', 'log_level': 'info', 'third_party_libraries': [] } self.config['lambda'][ 'athena_partition_refresh_config'] = athena_config_template self.write() LOGGER_CLI.info('Athena configuration successfully created')
def generate_athena(self): """Generate a base Athena config""" if 'athena_partition_refresh_config' in self.config['lambda']: LOGGER_CLI.warn( 'The Athena configuration already exists, skipping.') return prefix = self.config['global']['account']['prefix'] athena_config_template = { 'enable_metrics': False, 'current_version': '$LATEST', 'buckets': { '{}.streamalerts'.format(prefix): 'alert' }, 'handler': 'stream_alert.athena_partition_refresh.main.handler', 'timeout': '60', 'memory': '128', 'log_level': 'info', 'source_bucket': '{}.streamalert.source'.format(prefix), 'source_current_hash': '<auto_generated>', 'source_object_key': '<auto_generated>', 'third_party_libraries': [] } self.config['lambda'][ 'athena_partition_refresh_config'] = athena_config_template self.write() LOGGER_CLI.info('Athena configuration successfully created')
def _try_decrement_version(lambda_config, function_name): """Log a warning if the lambda version cannot be rolled back.""" changed = _decrement_version(lambda_config) if not changed: LOGGER_CLI.warn('%s cannot be rolled back from version %s', function_name, str(lambda_config['current_version'])) return changed
def _rollback_production(lambda_client, function_name): """Rollback the production alias for the given function name.""" version = lambda_client.get_alias(FunctionName=function_name, Name='production')['FunctionVersion'] if version == '$LATEST': # This won't happen with Terraform, but the alias could have been manually changed. LOGGER_CLI.error( '%s:production is pointing to $LATEST instead of a published version', function_name) return current_version = int(version) if current_version == 1: LOGGER_CLI.warn('%s:production is already at version 1', function_name) return LOGGER_CLI.info('Rolling back %s:production from version %d => %d', function_name, current_version, current_version - 1) try: lambda_client.update_alias(FunctionName=function_name, Name='production', FunctionVersion=str(current_version - 1)) except ClientError: LOGGER_CLI.exception('version not updated')
def check_invalid_rules_filters(rules_filter, all_test_rules): """Log warning message for filtered rules that do not exist. Args: rules_filter (set): A collection of rules to filter on, passed in by the user all_test_rules (set): A collection of all of the rules being tested """ invalid_rules = rules_filter.difference(all_test_rules) if invalid_rules: LOGGER_CLI.warn('%sNo test events configured for the following rules being filtered. ' 'This error could also be caused by a typo in the list of filtered ' 'rules\n\t%s%s', COLOR_YELLOW, '\n\t'.join(invalid_rules), COLOR_RESET)
def check_untested_rules(all_test_rules): """Log warning message for rules that exist but do not have proper test events. Args: all_test_rules (set): A collection of all of the rules being tested """ untested_rules = set(rule.Rule.rule_names()).difference(all_test_rules) if untested_rules: LOGGER_CLI.warn('%sNo test events configured for the following rules. Please add ' 'corresponding tests for these rules in \'%s\' to avoid seeing ' 'this warning\n\t%s%s', COLOR_YELLOW, TEST_EVENTS_DIR, '\n\t'.join(untested_rules), COLOR_RESET)
def check_untested_files(all_test_rules): """Log warning message for test events that exist with invalid rule names. Args: all_test_rules (set): A collection of all of the rules being tested """ invalid_rules = all_test_rules.difference(set(rule.Rule.rule_names())) if invalid_rules: LOGGER_CLI.warn('%sNo rules found in \'rules/\' that match the rules declared within ' '\'trigger_rules\' in a test event. Please update the list of ' '\'trigger_rules\' with valid rule names to avoid seeing this ' 'warning and any associated errors ' 'above\n\t%s%s', COLOR_YELLOW, '\n\t'.join(invalid_rules), COLOR_RESET)
def check_untested_rules(): """Function that prints warning log messages for rules that exist but do not have proper integration tests configured. """ all_test_files = {os.path.splitext(test_file)[0] for _, _, test_rule_files in os.walk(DIR_RULES) for test_file in test_rule_files} untested_rules = set(StreamRules.get_rules()).difference(all_test_files) for rule in untested_rules: LOGGER_CLI.warn('%sNo tests configured for rule: \'%s\'. Please add a ' 'corresponding test file for this rule in \'%s\' with the ' 'name \'%s.json\' to avoid seeing this warning%s', COLOR_YELLOW, rule, DIR_RULES, rule, COLOR_RESET)
def check_untested_files(): """Function that prints warning log messages for integration test files that exist but do not have a corresponding rule configured. """ all_test_files = {os.path.splitext(test_file)[0] for _, _, test_rule_files in os.walk(DIR_RULES) for test_file in test_rule_files} untested_rules = all_test_files.difference(set(StreamRules.get_rules())) for rule in untested_rules: LOGGER_CLI.warn('%sNo rules configured for test file: \'%s.json\'. Please ' 'add a corresponding rule for this test file in \'rules/\' with ' 'the name \'%s.py\' to avoid seeing this warning and any associated ' 'errors above%s', COLOR_YELLOW, rule, rule, COLOR_RESET)
def report_output_summary(self): """Helper function to print the summary results of all tests""" failure_messages = [ item for item in self.status_messages if item.type == StatusMessage.FAILURE ] warning_messages = [ item for item in self.status_messages if item.type == StatusMessage.WARNING ] passed_tests = sum(1 for item in self.status_messages if item.type == StatusMessage.SUCCESS) passed_tests = self.total_tests - len(failure_messages) # Print some lines at the bottom of output to make it more readable # This occurs here so there is always space and not only when the # successful test info prints print '\n\n' # Only print success info if we explicitly want to print output # but always print any errors or warnings below if self.print_output: # Print a message indicating how many of the total tests passed LOGGER_CLI.info('%s(%d/%d) Successful Tests%s', COLOR_GREEN, passed_tests, self.total_tests, COLOR_RESET) # Check if there were failed tests and report on them appropriately if failure_messages: # Print a message indicating how many of the total tests failed LOGGER_CLI.error('%s(%d/%d) Failures%s', COLOR_RED, len(failure_messages), self.total_tests, COLOR_RESET) # Iterate over the rule_name values in the failed list and report on them for index, failure in enumerate(failure_messages, start=1): LOGGER_CLI.error('%s(%d/%d) [%s] %s%s', COLOR_RED, index, len(failure_messages), failure.rule, failure.message, COLOR_RESET) # Check if there were any warnings and report on them if warning_messages: warning_count = len(warning_messages) LOGGER_CLI.warn('%s%d Warning%s%s', COLOR_YELLOW, warning_count, ('s' if warning_count > 1 else ''), COLOR_RESET) for index, warning in enumerate(warning_messages, start=1): LOGGER_CLI.warn('%s(%d/%d) [%s] %s%s', COLOR_YELLOW, index, warning_count, warning.rule, warning.message, COLOR_RESET)