def testGeneralFilters(self):
"""test alert delivery with global and local filtering"""
j_in_dq = Journal("j_in_DQ", "data/alert_delivery_test/listener_failure/inject_DQ_alerts.json")
dq_q = get_service(SERVICE_ALERT_DELIVERY_Q)
# Get the AlertListenerJournal journals
listeners = get_service(SERVICE_ALERT_DELIVERY).listeners
for listener in listeners:
name = listener.get_name()
if name == "AllAlerts":
j_out_all = listener.journal
if name == "OnlyAnalyzer1":
j_out_analyzer1 = listener.journal
# inject
j_in_dq.inject_queue(dq_q)
# Create a TEAL alert
create_teal_alert("XXXXXXXX", "no reason at all", "medium well", loc_instance="YYY")
# Get expected values
j_out_all_exp = Journal("all_exp", "data/alert_delivery_test/analyzer_filter/alerts_out_all.json")
j_out_analyzer1_exp = Journal("analyzer1", "data/alert_delivery_test/analyzer_filter/alerts_out_analyzer1.json")
# wait for stuff to come out
self.assertTrue(j_out_all.wait_for_entries(len(j_out_all_exp) + 3))
self.assertTrue(j_out_analyzer1.wait_for_entries(len(j_out_analyzer1_exp)))
# Check that it was what was expected
# Can't really check this because the location is unique for each machine and run
# Make sure only 3 extra
self.assertEqual(len(j_out_all) - len(j_out_all_exp), 3)
# self.assertTrue(j_out_all.deep_match(j_out_all_exp, ignore_delay=True, ignore_times=True))
self.assertTrue(j_out_analyzer1.deep_match(j_out_analyzer1_exp, ignore_delay=True, ignore_times=True))
return
def testDisableDup(self):
''' Test That disable dup works'''
self.teal = Teal('data/common/configurationtest.conf', 'stderr', msgLevel=self.msglevel, commit_alerts=False, commit_checkpoints=False)
am = get_service(SERVICE_ALERT_MGR)
self.assertEqual(len(am.in_mem_alerts), 0)
self.assertEqual(len(am.in_mem_alerts_duplicate), 0)
self.assertEqual(len(am.active_alerts_open), 0)
create_teal_alert('XXXXXXXX', 'no reason at all', 'medium well', loc_instance="YYY")
self.assertEqual(len(am.in_mem_alerts), 1)
self.assertEqual(len(am.in_mem_alerts_duplicate), 0)
self.assertEqual(len(am.active_alerts_open), 1)
create_teal_alert('XXXXXXXX', 'no reason at all', 'medium well', loc_instance="YYY")
self.assertEqual(len(am.in_mem_alerts), 2)
self.assertEqual(len(am.in_mem_alerts_duplicate), 1)
self.assertEqual(len(am.active_alerts_open), 1)
create_teal_alert('XXXXXXXX', 'no reason at all', 'medium well', loc_instance="YYY", disable_dup=True)
self.assertEqual(len(am.in_mem_alerts), 3)
self.assertEqual(len(am.in_mem_alerts_duplicate), 1)
self.assertEqual(len(am.active_alerts_open), 2)
create_teal_alert('XXXXXXXX', 'no reason at all', 'medium well', loc_instance="YYY", disable_dup=True)
self.assertEqual(len(am.in_mem_alerts), 4)
self.assertEqual(len(am.in_mem_alerts_duplicate), 1)
self.assertEqual(len(am.active_alerts_open), 3)
create_teal_alert('XXXXXXXX', 'no reason at all', 'medium well', loc_instance="YYY")
self.assertEqual(len(am.in_mem_alerts), 5)
self.assertEqual(len(am.in_mem_alerts_duplicate), 2)
self.assertEqual(len(am.active_alerts_open), 3)
self.teal.shutdown()
return
def create_listener_alert(self, alert_id, reason, raw_data, src_name=None, severity='I', urgency='N',
loc_instance=None, recommendation='Contact next level of support'):
''' Create an alert for this listener in the alert log and add it to the delivery queue
loc_instance will be added to the end of the TEAL location, if specified and if appropriate to the location type used
src_name will default to 'TEAL:listener:<name>'
'''
if src_name is None:
src_name = 'TEAL:listener:{0}'.format(self.get_name())
create_teal_alert(alert_id, reason, raw_data, src_name=src_name, severity=severity,
urgency=urgency, loc_instance=loc_instance, recommendation=recommendation)
return
def __init__(self,
configFile,
logFile=None,
msgLevel='info',
restart=None,
run_mode=TEAL_RUN_MODE_REALTIME,
commit_alerts=True,
data_only=False,
historic_qry=None,
daemon_mode=False,
extra_log_id='',
commit_checkpoints=None,
use_time_occurred=False):
""" Construct the ELA framework
@param configFile: the TEAL configuration file. This is mandatory
@param logFile: the full pathname of the logging file. If no logging file is
specified, logging will be to stdout
@param msgLevel: the lowest message level that will be logged. The default is
informational messages and above
@param restart: determines how teal will start processing events in realtime mode
@param run_mode: how the monitor will be configured - realtime or historic
@param commit_alerts: Certain run modes may want to not commit alerts if they are created
because the user is debugging rules or trying to determine relationships
@param data_only: do not initialize the processing pipeline Only the data that is
configured for TEAL should be set up and used
@param history_query: The query to use to get the events to do historic analysis on
@param daemon_mode: run in daemon mode
@param extra_log_id: additional string to add to all log entries for this instance of TEAL
@param commit_checkpoints: control whether checkpoints should be committed or not
@param use_time_occurred: Use time occurred instead of time logged for analysis
"""
os.umask(0o002) # Set the umask for files created so group users can access too
log_str_list = []
self.data_only = data_only # Needed for shutdown
# Initialize the registry. This will be used by subsequent initialization
self.init_reg_service()
# Register a shutdown service for users
registry.register_service(SERVICE_SHUTDOWN_MODE, SHUTDOWN_MODE_DEFERRED)
registry.register_service(SERVICE_SHUTDOWN, Shutdown(self.shutdown))
# Setup logging with temporary handler
# Determine if prefix should be used and if so, which one
if run_mode == TEAL_RUN_MODE_HISTORIC:
if commit_alerts == True:
extra_log_id = 'C' + extra_log_id
else:
extra_log_id = 'H' + extra_log_id
self.init_temp_log_service(msgLevel, extra_log_id)
get_logger().info("******* TEAL({0}) Startup initiated on {1}".format(id(self), datetime.now()))
log_str_list.append('\tMessage level: {0}'.format(repr(registry.get_service(SERVICE_MSG_LEVEL))))
get_logger().info(log_str_list[-1])
try:
# Initialize the mode
self.init_run_mode(run_mode)
log_str_list.append('\tRun mode: {0}'.format(registry.get_service(SERVICE_RUN_MODE)))
get_logger().info(log_str_list[-1])
# Initialize the TEAL environment that can't be changed in the configuration file
self.init_non_configurable_environment()
# Read in the configuration files and prepare it for use
config_str = self.init_cfg_service(configFile)
log_str_list.append('\tConfiguration: {0}'.format(config_str))
get_logger().info(log_str_list[-1])
# Initialize the rest of the TEAL environment which can be changed in the configuration file
self.init_configurable_environment(run_mode)
# Setup logging to the actual log
self.init_actual_log_service(logFile)
log_str_list.append('\tLog file: {0}'.format(repr(registry.get_service(SERVICE_LOG_FILE))))
get_logger().info(log_str_list[-1])
if historic_qry is not None and len(historic_qry) > 0:
log_str_list.append('\t Query: {0}'.format(historic_qry))
get_logger().info(log_str_list[-1])
if use_time_occurred == True:
registry.register_service(SERVICE_TIME_MODE, 'time_occurred')
log_str_list.append('\t Time mode = time occurred')
get_logger().info(log_str_list[-1])
else:
registry.register_service(SERVICE_TIME_MODE, 'time_logged')
# Create the location code service
self.init_location_service(run_mode)
# Load the metadata
self.init_metadata_service(run_mode)
# Initialize the DB interface so persistence and monitor can use
self.init_db_interface(daemon_mode, run_mode)
# Initialize the persistence services
self.init_persistence_services(commit_alerts)
# Validate the historic query string
registry.register_service(SERVICE_HISTORIC_QUERY, command.validate_qry_str(qry_info, historic_qry))
if (not data_only):
# Initialize Checkpointing service
self.init_checkpoint_service(commit_checkpoints, restart)
# Build the TEAL event/alert processing pipeline
pipe_str_list = self.init_processing_pipe(run_mode)
if len(pipe_str_list) > 0:
log_str_list.append('\tPipeline plug-ins: {0}'.format(', '.join(pipe_str_list)))
get_logger().info(log_str_list[-1])
else:
log_str_list.append('\tPipeline plug-ins: --None--')
get_logger().info(log_str_list[-1])
# Record startup information
# Note before monitor because monitor may start processing immediately
if ((daemon_mode == True) and not data_only) or (commit_alerts == True and run_mode == TEAL_RUN_MODE_HISTORIC):
# Create TEAL started alert
create_teal_alert(TEAL_ALERT_ID_TEAL_STARTED, 'TEAL started', '; '.join(log_str_list), recommendation='None')
if (not data_only):
# Start the monitor.
self.init_monitor(run_mode)
get_logger().info('TEAL startup complete')
except:
get_logger().exception('TEAL startup failed')
raise
