def run_tests(options, context): """Actual protected function for running tests Args: options [namedtuple]: CLI options (debug, processor, etc) context [namedtuple]: A constructed aws context object """ # Instantiate two status items - one for the rule processor # and one for the alert processor rp_status, ap_status = True, True if options.debug: LOGGER_SA.setLevel(logging.DEBUG) LOGGER_SO.setLevel(logging.DEBUG) LOGGER_CLI.setLevel(logging.DEBUG) else: # Add a filter to suppress a few noisy log messages LOGGER_SA.addFilter(TestingSuppressFilter()) # Check if the rule processor should be run for these tests test_rules = (run_options.get('processor') in {'rule', 'all'} or run_options.get('command') == 'live-test') # Check if the alert processor should be run for these tests test_alerts = (run_options.get('processor') in {'alert', 'all'} or run_options.get('command') == 'live-test') rule_proc_tester = RuleProcessorTester(test_rules) # Run the rule processor for all rules or designated rule set for status, alerts in rule_proc_tester.test_processor(options.rules): # If the alert processor should be tested, pass any alerts to it # and store the status over time if test_alerts: # Update the overall alert processor status with the ongoing status ap_status = AlertProcessorTester(context).test_processor( alerts) and ap_status # Update the overall rule processor status with the ongoing status rp_status = status and rp_status # Report summary information for the alert processor if it was ran if test_alerts: AlertProcessorTester.report_output_summary() if not (rp_status and ap_status): sys.exit(1)
def run_tests(options, context): """Actual protected function for running tests Args: options [namedtuple]: CLI options (debug, processor, etc) context [namedtuple]: A constructed aws context object """ if options.debug: LOGGER_SA.setLevel(logging.DEBUG) LOGGER_SO.setLevel(logging.DEBUG) LOGGER_CLI.setLevel(logging.DEBUG) else: # Add a filter to suppress a few noisy log messages LOGGER_SA.addFilter(TestingSuppressFilter()) # Check if the rule processor should be run for these tests test_rules = (set(run_options.get('processor')).issubset({'rule', 'all'}) or run_options.get('command') == 'live-test') # Check if the alert processor should be run for these tests test_alerts = (set(run_options.get('processor')).issubset({'alert', 'all'}) or run_options.get('command') == 'live-test') rule_proc_tester = RuleProcessorTester(context, test_rules) alert_proc_tester = AlertProcessorTester(context) # Run the rule processor for all rules or designated rule set for alerts in rule_proc_tester.test_processor(options.rules): # If the alert processor should be tested, process any alerts if test_alerts: alert_proc_tester.test_processor(alerts) # Report summary information for the alert processor if it was ran if test_alerts: AlertProcessorTester.report_output_summary() # Print any invalid log messages that we accumulated over this run for message in rule_proc_tester.invalid_log_messages: LOGGER_CLI.error('%s%s%s', COLOR_RED, message, COLOR_RESET) if not (rule_proc_tester.all_tests_passed and alert_proc_tester.all_tests_passed and (not rule_proc_tester.invalid_log_messages)): sys.exit(1)
def run_tests(options, context): """Actual protected function for running tests Args: options (namedtuple): CLI options (debug, processor, etc) context (namedtuple): A constructed aws context object """ # The Rule Processor and Alert Processor need environment variables for many things prefix = config['global']['account']['prefix'] alerts_table = '{}_streamalert_alerts'.format(prefix) os.environ[ 'ALERT_PROCESSOR'] = '{}_streamalert_alert_processor'.format( prefix) os.environ['ALERTS_TABLE'] = alerts_table os.environ['AWS_DEFAULT_REGION'] = config['global']['account'][ 'region'] os.environ['CLUSTER'] = run_options.get('cluster') or '' if not options.debug: # Add a filter to suppress a few noisy log messages LOGGER_SA.addFilter(SuppressNoise()) # Create an in memory logging buffer to be used to caching all error messages log_mem_handler = get_log_memory_handler() # Check if the rule processor should be run for these tests # Using NOT set.isdisjoint will check to see if there are commonalities between # the options in 'processor' and {'rule', 'all'} test_rules = (not run_options.get('processor').isdisjoint( {'rule', 'all'}) if run_options.get('processor') else run_options.get('command') == 'live-test' or run_options.get('command') == 'validate-schemas') # Check if the alert processor should be run for these tests # Using NOT set.isdisjoint will check to see if there are commonalities between # the options in 'processor' and {'alert', 'all'} test_alerts = (not run_options.get('processor').isdisjoint( {'alert', 'all'}) if run_options.get('processor') else run_options.get('command') == 'live-test') validate_schemas = options.command == 'validate-schemas' rules_filter = run_options.get('rules', {}) files_filter = run_options.get('files', {}) # Run the rule processor for all rules or designated rule set if context.mocked: helpers.setup_mock_alerts_table(alerts_table) # Mock S3 bucket for lookup tables testing helpers.mock_s3_bucket(config) rule_proc_tester = RuleProcessorTester(context, config, test_rules) alert_proc_tester = AlertProcessorTester(config, context) for _ in range(run_options.get('repeat', 1)): for alerts in rule_proc_tester.test_processor( rules_filter, files_filter, validate_schemas): # If the alert processor should be tested, process any alerts if test_alerts: alert_proc_tester.test_processor(alerts) # Report summary information for the alert processor if it was ran if test_alerts: AlertProcessorTester.report_output_summary() all_test_rules = None if rules_filter: all_test_rules = helpers.get_rules_from_test_events( TEST_EVENTS_DIR) check_invalid_rules_filters(rules_filter, all_test_rules) # If this is not just a validation run, and rule/file filters are not in place # then warn the user if there are test files without corresponding rules # Also check all of the rule files to make sure they have tests configured if not (validate_schemas or rules_filter or files_filter): all_test_rules = all_test_rules or helpers.get_rules_from_test_events( TEST_EVENTS_DIR) check_untested_files(all_test_rules) check_untested_rules(all_test_rules) if not (rule_proc_tester.all_tests_passed and alert_proc_tester.all_tests_passed): return 1 # will exit with error # If there are any log records in the memory buffer, then errors occurred somewhere if log_mem_handler.buffer: # Release the MemoryHandler so we can do some other logging now logging.getLogger().removeHandler(log_mem_handler) LOGGER_CLI.error( '%sSee %d miscellaneous error(s) below ' 'that were encountered and may need to be addressed%s', COLOR_RED, len(log_mem_handler.buffer), COLOR_RESET) log_mem_handler.setTarget(LOGGER_CLI) log_mem_handler.flush() return 1 # will exit with error return 0 # will exit without error
def run_tests(options, context): """Actual protected function for running tests Args: options (namedtuple): CLI options (debug, processor, etc) context (namedtuple): A constructed aws context object """ if options.debug: # TODO(jack): Currently there is no (clean) way to set # the logger formatter to provide more verbose # output in debug mode. Running basicConfig twice # does not actually change the formatter on the logger object. # This functionality can be added during the logging refactor # Example Steps: # call .shutdown() on the existing logger # debug_formatter = logging.Formatter( # '%(name)s [%(levelname)s]: [%(module)s.%(funcName)s] %(message)s') # set the new logger to the formatter above for streamalert_logger in (LOGGER_SA, LOGGER_SH, LOGGER_SO, LOGGER_CLI): streamalert_logger.setLevel(logging.DEBUG) else: # Add a filter to suppress a few noisy log messages LOGGER_SA.addFilter(SuppressNoise()) # Create an in memory logging buffer to be used to caching all error messages log_mem_hanlder = get_log_memory_hanlder() # Check if the rule processor should be run for these tests test_rules = (set(run_options.get('processor')).issubset( {'rule', 'all'}) if run_options.get('processor') else run_options.get('command') == 'live-test' or run_options.get('command') == 'validate-schemas') # Check if the alert processor should be run for these tests test_alerts = (set(run_options.get('processor')).issubset( {'alert', 'all'}) if run_options.get('processor') else run_options.get('command') == 'live-test') rule_proc_tester = RuleProcessorTester(context, test_rules) alert_proc_tester = AlertProcessorTester(context) validate_schemas = options.command == 'validate-schemas' filters = options.test_files if validate_schemas else options.rules # Run the rule processor for all rules or designated rule set for alerts in rule_proc_tester.test_processor(filters, validate_schemas): # If the alert processor should be tested, process any alerts if test_alerts: alert_proc_tester.test_processor(alerts) # Report summary information for the alert processor if it was ran if test_alerts: AlertProcessorTester.report_output_summary() # Check all of the rule files to make sure they have tests configured check_untested_rules() # If this is not just a validation run, then warn the user # if there are test files without corresponding rules if not validate_schemas: check_untested_files() if not (rule_proc_tester.all_tests_passed and alert_proc_tester.all_tests_passed): sys.exit(1) # If there are any log records in the memory buffer, then errors occured somewhere if log_mem_hanlder.buffer: # Release the MemoryHandler so we can do some other logging now logging.getLogger().removeHandler(log_mem_hanlder) LOGGER_CLI.error( '%sSee %d miscellaneous error(s) below ' 'that were encountered and may need to be addressed%s', COLOR_RED, len(log_mem_hanlder.buffer), COLOR_RESET) log_mem_hanlder.setTarget(LOGGER_CLI) log_mem_hanlder.flush() sys.exit(1)
def run_tests(options, context): """Actual protected function for running tests Args: options (namedtuple): CLI options (debug, processor, etc) context (namedtuple): A constructed aws context object """ # The Rule Processor and Alert Processor need environment variables for many things prefix = config['global']['account']['prefix'] alerts_table = '{}_streamalert_alerts'.format(prefix) os.environ['ALERT_PROCESSOR'] = '{}_streamalert_alert_processor'.format(prefix) os.environ['ALERTS_TABLE'] = alerts_table os.environ['AWS_DEFAULT_REGION'] = config['global']['account']['region'] os.environ['CLUSTER'] = run_options.get('cluster') or '' if options.debug: # TODO(jack): Currently there is no (clean) way to set # the logger formatter to provide more verbose # output in debug mode. Running basicConfig twice # does not actually change the formatter on the logger object. # This functionality can be added during the logging refactor # Example Steps: # call .shutdown() on the existing logger # debug_formatter = logging.Formatter( # '%(name)s [%(levelname)s]: [%(module)s.%(funcName)s] %(message)s') # set the new logger to the formatter above for streamalert_logger in (LOGGER_SA, LOGGER_SH, LOGGER_SO, LOGGER_CLI): streamalert_logger.setLevel(logging.DEBUG) else: # Add a filter to suppress a few noisy log messages LOGGER_SA.addFilter(SuppressNoise()) # Create an in memory logging buffer to be used to caching all error messages log_mem_handler = get_log_memory_handler() # Check if the rule processor should be run for these tests # Using NOT set.isdisjoint will check to see if there are commonalities between # the options in 'processor' and {'rule', 'all'} test_rules = (not run_options.get('processor').isdisjoint({'rule', 'all'}) if run_options.get('processor') else run_options.get('command') == 'live-test' or run_options.get('command') == 'validate-schemas') # Check if the alert processor should be run for these tests # Using NOT set.isdisjoint will check to see if there are commonalities between # the options in 'processor' and {'alert', 'all'} test_alerts = (not run_options.get('processor').isdisjoint({'alert', 'all'}) if run_options.get('processor') else run_options.get('command') == 'live-test') rule_proc_tester = RuleProcessorTester(context, config, test_rules) alert_proc_tester = AlertProcessorTester(config, context) validate_schemas = options.command == 'validate-schemas' rules_filter = run_options.get('rules', {}) files_filter = run_options.get('files', {}) # Run the rule processor for all rules or designated rule set if context.mocked: helpers.setup_mock_alerts_table(alerts_table) for alerts in rule_proc_tester.test_processor(rules_filter, files_filter, validate_schemas): # If the alert processor should be tested, process any alerts if test_alerts: alert_proc_tester.test_processor(alerts) # Report summary information for the alert processor if it was ran if test_alerts: AlertProcessorTester.report_output_summary() all_test_rules = None if rules_filter: all_test_rules = helpers.get_rules_from_test_events(TEST_EVENTS_DIR) check_invalid_rules_filters(rules_filter, all_test_rules) # If this is not just a validation run, and rule/file filters are not in place # then warn the user if there are test files without corresponding rules # Also check all of the rule files to make sure they have tests configured if not (validate_schemas or rules_filter or files_filter): all_test_rules = all_test_rules or helpers.get_rules_from_test_events(TEST_EVENTS_DIR) check_untested_files(all_test_rules) check_untested_rules(all_test_rules) if not (rule_proc_tester.all_tests_passed and alert_proc_tester.all_tests_passed): sys.exit(1) # If there are any log records in the memory buffer, then errors occurred somewhere if log_mem_handler.buffer: # Release the MemoryHandler so we can do some other logging now logging.getLogger().removeHandler(log_mem_handler) LOGGER_CLI.error('%sSee %d miscellaneous error(s) below ' 'that were encountered and may need to be addressed%s', COLOR_RED, len(log_mem_handler.buffer), COLOR_RESET) log_mem_handler.setTarget(LOGGER_CLI) log_mem_handler.flush() sys.exit(1)