def test3check_email_addresses(self): """Test if mail addresses are validated as expected.""" ac = self.analysis_context ac.aminer_config.config_properties[DefaultMailNotificationEventHandler.CONFIG_KEY_MAIL_TARGET_ADDRESS] = "*****@*****.**" ac.aminer_config.config_properties[DefaultMailNotificationEventHandler.CONFIG_KEY_MAIL_FROM_ADDRESS] = "*****@*****.**" _ = DefaultMailNotificationEventHandler(ac) ac.aminer_config.config_properties[DefaultMailNotificationEventHandler.CONFIG_KEY_MAIL_TARGET_ADDRESS] = "root@localhost" ac.aminer_config.config_properties[DefaultMailNotificationEventHandler.CONFIG_KEY_MAIL_FROM_ADDRESS] = "root@localhost" _ = DefaultMailNotificationEventHandler(ac) ac.aminer_config.config_properties[DefaultMailNotificationEventHandler.CONFIG_KEY_MAIL_TARGET_ADDRESS] = "domain.user1@localhost" self.assertRaises(Exception, DefaultMailNotificationEventHandler, ac) ac.aminer_config.config_properties[DefaultMailNotificationEventHandler.CONFIG_KEY_MAIL_FROM_ADDRESS] = "domain.user1@localhost" self.assertRaises(Exception, DefaultMailNotificationEventHandler, ac) ac.aminer_config.config_properties[DefaultMailNotificationEventHandler.CONFIG_KEY_MAIL_TARGET_ADDRESS] = "root@notLocalhost" ac.aminer_config.config_properties[DefaultMailNotificationEventHandler.CONFIG_KEY_MAIL_FROM_ADDRESS] = "root@localhost" self.assertRaises(Exception, DefaultMailNotificationEventHandler, ac) ac.aminer_config.config_properties[DefaultMailNotificationEventHandler.CONFIG_KEY_MAIL_TARGET_ADDRESS] = "root@localhost" ac.aminer_config.config_properties[DefaultMailNotificationEventHandler.CONFIG_KEY_MAIL_FROM_ADDRESS] = "root@notLocalhost" self.assertRaises(Exception, DefaultMailNotificationEventHandler, ac)
def test1log_multiple_lines_event(self): """ In this test case multiple lines should be received, before sending an email to root@localhost. Make sure no mail notifications are in /var/spool/mail/root, before running this test. This test case must wait some time to ensure, that the mail can be read. """ description = "Test1DefaultMailNotificationEventHandler" match_context = MatchContext(self.pid) fixed_dme = FixedDataModelElement('s1', self.pid) match_element = fixed_dme.get_match_element("match", match_context) match_context = MatchContext(self.pid) fixed_dme2 = FixedDataModelElement('s2', self.pid) match_element2 = fixed_dme2.get_match_element("match", match_context) default_mail_notification_event_handler = DefaultMailNotificationEventHandler(self.analysis_context) self.analysis_context.register_component(self, description) t = time() log_atom = LogAtom(fixed_dme.fixed_data, ParserMatch(match_element), t, self) default_mail_notification_event_handler.receive_event( self.test % self.__class__.__name__, 'New value for pathes %s, %s: %s' % ( 'match/s1', 'match/s2', repr(match_element.match_object)), [log_atom.raw_data, log_atom.raw_data], None, log_atom, self) t += 600 log_atom = LogAtom(fixed_dme.fixed_data, ParserMatch(match_element), t, self) # set the next_alert_time instead of sleeping 10 seconds default_mail_notification_event_handler.next_alert_time = time() default_mail_notification_event_handler.receive_event( self.test % self.__class__.__name__, 'New value for pathes %s, %s: %s' % ( 'match/s1', 'match/s2', repr(match_element.match_object)), [log_atom.raw_data, log_atom.raw_data], None, log_atom, self) sleep(2) # skipcq: PYL-W1510, BAN-B602 result = subprocess.run(self.mail_call, shell=True, stdout=subprocess.PIPE) # skipcq: PYL-W1510, BAN-B602 subprocess.run(self.mail_delete_call, shell=True, stdout=subprocess.PIPE) if datetime.fromtimestamp(t - 600).strftime(self.datetime_format_string) not in str(result.stdout, 'utf-8'): print("ERROR: %s t-600 not found in mail!" % description, file=sys.stderr) if datetime.fromtimestamp(t).strftime(self.datetime_format_string) not in str(result.stdout, 'utf-8'): print("ERROR: %s t not found in mail!" % description, file=sys.stderr) self.assertTrue(self.__expected_string % ( "" + match_element.get_path() + ", " + match_element2.get_path(), repr(match_element.get_match_object()), self.__class__.__name__, description, 2, match_element.get_match_string().decode() + "\n " + match_element2.get_match_string().decode()) in str(result.stdout, 'utf-8'), msg="%s vs \n %s" % (self.__expected_string % ( match_element.get_path(), repr(match_element.get_match_object()), self.__class__.__name__, description, 1, match_element.get_match_string().decode() + "\n\n"), str(result.stdout, 'utf-8'))) self.assertTrue(self.__expected_string % ( "" + match_element.get_path() + ", " + match_element2.get_path(), repr(match_element.get_match_object()), self.__class__.__name__, description, 2, match_element.get_match_string().decode() + "\n " + match_element2.get_match_string().decode() + "\n\n") in str(result.stdout, 'utf-8'))
def build_analysis_pipeline(analysis_context): """ Define the function to create pipeline for parsing the log data. It has also to define an AtomizerFactory to instruct aminer how to process incoming data streams to create log atoms from them. """ # Build the parsing model: service_children_disk_report = [ FixedDataModelElement( 'Space', b' Current Disk Data is: Filesystem Type Size Used Avail Use%' ), DelimitedDataModelElement('Data', b'%'), AnyByteDataModelElement('Rest') ] service_children_login_details = [ FixedDataModelElement('User', b'User '), DelimitedDataModelElement('Username', b' '), FixedWordlistDataModelElement('Status', [b' logged in', b' logged out']), OptionalMatchModelElement( 'PastTime', SequenceModelElement('Time', [ FixedDataModelElement('Blank', b' '), DecimalIntegerValueModelElement('Minutes'), FixedDataModelElement('Ago', b' minutes ago.') ])) ] service_children_cron_job = [ DateTimeModelElement('DTM', b'%Y-%m-%d %H:%M:%S'), FixedDataModelElement('UNameSpace1', b' '), DelimitedDataModelElement('UName', b' '), FixedDataModelElement('UNameSpace2', b' '), DelimitedDataModelElement('User', b' '), FixedDataModelElement('Cron', b' cron['), DecimalIntegerValueModelElement('JobNumber'), FixedDataModelElement('Details', b']: Job `cron.daily` started.') ] service_children_random_time = [ FixedDataModelElement('Space', b'Random: '), DecimalIntegerValueModelElement('Random') ] service_children_sensors = [ SequenceModelElement('CPUTemp', [ FixedDataModelElement('FixedTemp', b'CPU Temp: '), DecimalIntegerValueModelElement('Temp'), FixedDataModelElement('Degrees', b'\xc2\xb0C') ]), FixedDataModelElement('Space1', b', '), SequenceModelElement('CPUWorkload', [ FixedDataModelElement('FixedWorkload', b'CPUWorkload: '), DecimalIntegerValueModelElement('Workload'), FixedDataModelElement('Percent', b'%') ]), FixedDataModelElement('Space2', b', '), DateTimeModelElement('DTM', b'%Y-%m-%d %H:%M:%S') ] service_children_user_ip_address = [ FixedDataModelElement('User', b'User '), DelimitedDataModelElement('Username', b' '), FixedDataModelElement('Action', b' changed IP address to '), IpAddressDataModelElement('IP') ] service_children_cron_job_announcement = [ DateTimeModelElement('DTM', b'%Y-%m-%d %H:%M:%S'), FixedDataModelElement('Space', b' '), DelimitedDataModelElement('UName', b' '), FixedDataModelElement('Cron', b' cron['), DecimalIntegerValueModelElement('JobNumber'), FixedDataModelElement('Run', b']: Will run job `'), FixedWordlistDataModelElement( 'CronType', [b'cron.daily', b'cron.hourly', b'cron.monthly', b'cron.weekly']), FixedDataModelElement('StartTime', b'\' in 5 min.') ] service_children_cron_job_execution = [ DateTimeModelElement('DTM', b'%Y-%m-%d %H:%M:%S'), FixedDataModelElement('Space1', b' '), DelimitedDataModelElement('UName', b' '), FixedDataModelElement('Cron', b' cron['), DecimalIntegerValueModelElement('JobNumber'), FixedDataModelElement('Job', b']: Job `'), FixedWordlistDataModelElement( 'CronType', [b'cron.daily', b'cron.hourly', b'cron.monthly', b'cron.weekly']), FixedDataModelElement('Started', b'\' started') ] parsing_model = FirstMatchModelElement('model', [ SequenceModelElement('CronAnnouncement', service_children_cron_job_announcement), SequenceModelElement('CronExecution', service_children_cron_job_execution), SequenceModelElement('DailyCron', service_children_cron_job), SequenceModelElement('DiskReport', service_children_disk_report), SequenceModelElement('LoginDetails', service_children_login_details), DecimalIntegerValueModelElement('Random'), SequenceModelElement('RandomTime', service_children_random_time), SequenceModelElement('Sensors', service_children_sensors), SequenceModelElement('IPAddresses', service_children_user_ip_address) ]) # Some generic imports. from aminer.analysis import AtomFilters # Create all global handler lists here and append the real handlers later on. # Use this filter to distribute all atoms to the analysis handlers. atom_filters = AtomFilters.SubhandlerFilter(None) analysis_context.register_component(atom_filters, component_name="AtomFilter") from aminer.analysis.TimestampCorrectionFilters import SimpleMonotonicTimestampAdjust simple_monotonic_timestamp_adjust = SimpleMonotonicTimestampAdjust( [atom_filters]) analysis_context.register_component( simple_monotonic_timestamp_adjust, component_name="SimpleMonotonicTimestampAdjust") from aminer.events.StreamPrinterEventHandler import StreamPrinterEventHandler stream_printer_event_handler = StreamPrinterEventHandler( analysis_context) # skipcq: BAN-B108 from aminer.events.Utils import VolatileLogarithmicBackoffEventHistory volatile_logarithmic_backoff_event_history = VolatileLogarithmicBackoffEventHistory( 100) anomaly_event_handlers = [ stream_printer_event_handler, volatile_logarithmic_backoff_event_history ] analysis_context.register_component( volatile_logarithmic_backoff_event_history, component_name="VolatileLogarithmicBackoffEventHistory") # Now define the AtomizerFactory using the model. A simple line based one is usually sufficient. from aminer.input.SimpleByteStreamLineAtomizerFactory import SimpleByteStreamLineAtomizerFactory analysis_context.atomizer_factory = SimpleByteStreamLineAtomizerFactory( parsing_model, [simple_monotonic_timestamp_adjust], anomaly_event_handlers) # Just report all unparsed atoms to the event handlers. from aminer.analysis.UnparsedAtomHandlers import SimpleUnparsedAtomHandler simple_unparsed_atom_handler = SimpleUnparsedAtomHandler( anomaly_event_handlers) atom_filters.add_handler(simple_unparsed_atom_handler, stop_when_handled_flag=True) analysis_context.register_component(simple_unparsed_atom_handler, component_name="UnparsedHandler") from aminer.analysis.TimestampsUnsortedDetector import TimestampsUnsortedDetector timestamps_unsorted_detector = TimestampsUnsortedDetector( analysis_context.aminer_config, anomaly_event_handlers) atom_filters.add_handler(timestamps_unsorted_detector) analysis_context.register_component( timestamps_unsorted_detector, component_name="TimestampsUnsortedDetector") from aminer.analysis import Rules from aminer.analysis.AllowlistViolationDetector import AllowlistViolationDetector allowlist_rules = [ Rules.OrMatchRule([ Rules.AndMatchRule([ Rules.PathExistsMatchRule( '/model/LoginDetails/PastTime/Time/Minutes'), Rules.NegationMatchRule( Rules.ValueMatchRule('/model/LoginDetails/Username', b'root')) ]), Rules.AndMatchRule([ Rules.NegationMatchRule( Rules.PathExistsMatchRule( '/model/LoginDetails/PastTime/Time/Minutes')), Rules.PathExistsMatchRule('/model/LoginDetails') ]), Rules.NegationMatchRule( Rules.PathExistsMatchRule('/model/LoginDetails')) ]) ] # This rule list should trigger, when the line does not look like: User root (logged in, logged out) # or User 'username' (logged in, logged out) x minutes ago. allowlist_violation_detector = AllowlistViolationDetector( analysis_context.aminer_config, allowlist_rules, anomaly_event_handlers) analysis_context.register_component(allowlist_violation_detector, component_name="Allowlist") atom_filters.add_handler(allowlist_violation_detector) from aminer.analysis.ParserCount import ParserCount parser_count = ParserCount(analysis_context.aminer_config, None, anomaly_event_handlers, 10) analysis_context.register_component(parser_count, component_name="ParserCount") atom_filters.add_handler(parser_count) from aminer.analysis.EventCorrelationDetector import EventCorrelationDetector ecd = EventCorrelationDetector(analysis_context.aminer_config, anomaly_event_handlers, check_rules_flag=True, hypothesis_max_delta_time=1.0, auto_include_flag=True) analysis_context.register_component( ecd, component_name="EventCorrelationDetector") atom_filters.add_handler(ecd) from aminer.analysis.NewMatchPathDetector import NewMatchPathDetector new_match_path_detector = NewMatchPathDetector( analysis_context.aminer_config, anomaly_event_handlers, auto_include_flag=True) analysis_context.register_component(new_match_path_detector, component_name="NewMatchPath") atom_filters.add_handler(new_match_path_detector) def tuple_transformation_function(match_value_list): """Only allow output of the EnhancedNewMatchPathValueComboDetector after every 10000th element.""" extra_data = enhanced_new_match_path_value_combo_detector.known_values_dict.get( tuple(match_value_list)) if extra_data is not None: mod = 10000 if (extra_data[2] + 1) % mod == 0: enhanced_new_match_path_value_combo_detector.auto_include_flag = False else: enhanced_new_match_path_value_combo_detector.auto_include_flag = True return match_value_list from aminer.analysis.EnhancedNewMatchPathValueComboDetector import EnhancedNewMatchPathValueComboDetector enhanced_new_match_path_value_combo_detector = EnhancedNewMatchPathValueComboDetector( analysis_context.aminer_config, ['/model/DailyCron/UName', '/model/DailyCron/JobNumber'], anomaly_event_handlers, auto_include_flag=False, tuple_transformation_function=tuple_transformation_function) analysis_context.register_component( enhanced_new_match_path_value_combo_detector, component_name="EnhancedNewValueCombo") atom_filters.add_handler(enhanced_new_match_path_value_combo_detector) from aminer.analysis.HistogramAnalysis import HistogramAnalysis, LinearNumericBinDefinition, ModuloTimeBinDefinition, \ PathDependentHistogramAnalysis modulo_time_bin_definition = ModuloTimeBinDefinition( 86400, 3600, 0, 1, 24, True) linear_numeric_bin_definition = LinearNumericBinDefinition(50, 5, 20, True) histogram_analysis = HistogramAnalysis( analysis_context.aminer_config, [('/model/RandomTime/Random', modulo_time_bin_definition), ('/model/Random', linear_numeric_bin_definition)], 10, anomaly_event_handlers) analysis_context.register_component(histogram_analysis, component_name="HistogramAnalysis") atom_filters.add_handler(histogram_analysis) path_dependent_histogram_analysis = PathDependentHistogramAnalysis( analysis_context.aminer_config, '/model/RandomTime', modulo_time_bin_definition, 10, anomaly_event_handlers) analysis_context.register_component( path_dependent_histogram_analysis, component_name="PathDependentHistogramAnalysis") atom_filters.add_handler(path_dependent_histogram_analysis) from aminer.analysis.MatchValueAverageChangeDetector import MatchValueAverageChangeDetector match_value_average_change_detector = MatchValueAverageChangeDetector( analysis_context.aminer_config, anomaly_event_handlers, None, ['/model/Random'], 100, 10) analysis_context.register_component( match_value_average_change_detector, component_name="MatchValueAverageChange") atom_filters.add_handler(match_value_average_change_detector) import sys from aminer.analysis.MatchValueStreamWriter import MatchValueStreamWriter match_value_stream_writer = MatchValueStreamWriter(sys.stdout, [ '/model/Sensors/CPUTemp', '/model/Sensors/CPUWorkload', '/model/Sensors/DTM' ], b';', b'') analysis_context.register_component( match_value_stream_writer, component_name="MatchValueStreamWriter") atom_filters.add_handler(match_value_stream_writer) from aminer.analysis.NewMatchPathValueComboDetector import NewMatchPathValueComboDetector new_match_path_value_combo_detector = NewMatchPathValueComboDetector( analysis_context.aminer_config, ['/model/IPAddresses/Username', '/model/IPAddresses/IP'], anomaly_event_handlers, auto_include_flag=False) analysis_context.register_component( new_match_path_value_combo_detector, component_name="NewMatchPathValueCombo") atom_filters.add_handler(new_match_path_value_combo_detector) from aminer.analysis.NewMatchIdValueComboDetector import NewMatchIdValueComboDetector new_match_id_value_combo_detector = NewMatchIdValueComboDetector( analysis_context.aminer_config, ['/model/type/path/name', '/model/type/syscall/syscall'], anomaly_event_handlers, id_path_list=['/model/type/path/id', '/model/type/syscall/id'], min_allowed_time_diff=5, auto_include_flag=True, allow_missing_values_flag=True, output_log_line=True) analysis_context.register_component( new_match_id_value_combo_detector, component_name="NewMatchIdValueComboDetector") atom_filters.add_handler(new_match_id_value_combo_detector) from aminer.analysis.NewMatchPathValueDetector import NewMatchPathValueDetector new_match_path_value_detector = NewMatchPathValueDetector( analysis_context.aminer_config, ['/model/DailyCron/Job Number', '/model/IPAddresses/Username'], anomaly_event_handlers, auto_include_flag=False) analysis_context.register_component(new_match_path_value_detector, component_name="NewMatchPathValue") atom_filters.add_handler(new_match_path_value_detector) from aminer.analysis.MissingMatchPathValueDetector import MissingMatchPathValueDetector missing_match_path_value_detector = MissingMatchPathValueDetector( analysis_context.aminer_config, ['/model/DiskReport/Space'], anomaly_event_handlers, auto_include_flag=False, default_interval=2, realert_interval=5) analysis_context.register_component(missing_match_path_value_detector, component_name="MissingMatch") atom_filters.add_handler(missing_match_path_value_detector) from aminer.analysis.TimeCorrelationDetector import TimeCorrelationDetector time_correlation_detector = TimeCorrelationDetector( analysis_context.aminer_config, anomaly_event_handlers, 2, min_rule_attributes=1, max_rule_attributes=5, record_count_before_event=70000, output_log_line=True) analysis_context.register_component( time_correlation_detector, component_name="TimeCorrelationDetector") atom_filters.add_handler(time_correlation_detector) from aminer.analysis.TimeCorrelationViolationDetector import TimeCorrelationViolationDetector, CorrelationRule, EventClassSelector cron_job_announcement = CorrelationRule( 'CronJobAnnouncement', 5, 6, max_artefacts_a_for_single_b=1, artefact_match_parameters=[('/model/CronAnnouncement/JobNumber', '/model/CronExecution/JobNumber')]) a_class_selector = EventClassSelector('Announcement', [cron_job_announcement], None) b_class_selector = EventClassSelector('Execution', None, [cron_job_announcement]) rules = [ Rules.PathExistsMatchRule('/model/CronAnnouncement/Run', a_class_selector), Rules.PathExistsMatchRule('/model/CronExecution/Job', b_class_selector) ] time_correlation_violation_detector = TimeCorrelationViolationDetector( analysis_context.aminer_config, rules, anomaly_event_handlers) analysis_context.register_component( time_correlation_violation_detector, component_name="TimeCorrelationViolationDetector") atom_filters.add_handler(time_correlation_violation_detector) from aminer.events.DefaultMailNotificationEventHandler import DefaultMailNotificationEventHandler if DefaultMailNotificationEventHandler.CONFIG_KEY_MAIL_TARGET_ADDRESS in analysis_context.aminer_config.config_properties: mail_notification_handler = DefaultMailNotificationEventHandler( analysis_context) analysis_context.register_component(mail_notification_handler, component_name="MailHandler") anomaly_event_handlers.append(mail_notification_handler)
def build_analysis_pipeline(analysis_context): """ Define the function to create pipeline for parsing the log data. It has also to define an AtomizerFactory to instruct aminer how to process incoming data streams to create log atoms from them. """ date_format_string = b'%Y-%m-%d %H:%M:%S' cron = b' cron[' # Build the parsing model: service_children_disk_report = [ FixedDataModelElement( 'Space', b' Current Disk Data is: Filesystem Type Size Used Avail Use%' ), DelimitedDataModelElement('Data', b'%'), AnyByteDataModelElement('Rest') ] service_children_login_details = [ FixedDataModelElement('User', b'User '), DelimitedDataModelElement('Username', b' '), FixedWordlistDataModelElement('Status', [b' logged in', b' logged out']), OptionalMatchModelElement( 'PastTime', SequenceModelElement('Time', [ FixedDataModelElement('Blank', b' '), DecimalIntegerValueModelElement('Minutes'), FixedDataModelElement('Ago', b' minutes ago.') ])) ] service_children_cron_job = [ DateTimeModelElement('DTM', date_format_string), FixedDataModelElement('UNameSpace1', b' '), DelimitedDataModelElement('UName', b' '), FixedDataModelElement('UNameSpace2', b' '), DelimitedDataModelElement('User', b' '), FixedDataModelElement('Cron', cron), DecimalIntegerValueModelElement('JobNumber'), FixedDataModelElement('Details', b']: Job `cron.daily` started.') ] service_children_random_time = [ FixedDataModelElement('Space', b'Random: '), DecimalIntegerValueModelElement('Random') ] service_children_sensors = [ SequenceModelElement('CPUTemp', [ FixedDataModelElement('FixedTemp', b'CPU Temp: '), DecimalIntegerValueModelElement('Temp'), FixedDataModelElement('Degrees', b'\xc2\xb0C') ]), FixedDataModelElement('Space1', b', '), SequenceModelElement('CPUWorkload', [ FixedDataModelElement('Fixed Workload', b'CPU Workload: '), DecimalIntegerValueModelElement('Workload'), FixedDataModelElement('Percent', b'%') ]), FixedDataModelElement('Space2', b', '), DateTimeModelElement('DTM', date_format_string) ] service_children_user_ip_address = [ FixedDataModelElement('User', b'User '), DelimitedDataModelElement('Username', b' '), FixedDataModelElement('Action', b' changed IP address to '), IpAddressDataModelElement('IP') ] service_children_cron_job_announcement = [ DateTimeModelElement('DTM', date_format_string), FixedDataModelElement('Space', b' '), DelimitedDataModelElement('UName', b' '), FixedDataModelElement('Cron', cron), DecimalIntegerValueModelElement('JobNumber'), FixedDataModelElement('Run', b']: Will run job `'), FixedWordlistDataModelElement( 'CronType', [b'cron.daily', b'cron.hourly', b'cron.monthly', b'cron.weekly']), FixedDataModelElement('Start Time', b'\' in 5 min.') ] service_children_cron_job_execution = [ DateTimeModelElement('DTM', date_format_string), FixedDataModelElement('Space1', b' '), DelimitedDataModelElement('UName', b' '), FixedDataModelElement('Cron', cron), DecimalIntegerValueModelElement('JobNumber'), FixedDataModelElement('Job', b']: Job `'), FixedWordlistDataModelElement( 'CronType', [b'cron.daily', b'cron.hourly', b'cron.monthly', b'cron.weekly']), FixedDataModelElement('Started', b'\' started') ] service_children_parsing_model_element = [ DateTimeModelElement('DateTimeModelElement', b'Current DateTime: %d.%m.%Y %H:%M:%S'), DecimalFloatValueModelElement('DecimalFloatValueModelElement', value_sign_type='optional'), DecimalIntegerValueModelElement('DecimalIntegerValueModelElement', value_sign_type='optional', value_pad_type='blank'), SequenceModelElement('', [ DelimitedDataModelElement('DelimitedDataModelElement', b';'), FixedDataModelElement('FixedDataModelElement', b';') ]) ] # ElementValueBranchModelElement fixed_data_me1 = FixedDataModelElement("fixed1", b'match ') fixed_data_me2 = FixedDataModelElement("fixed2", b'fixed String') fixed_wordlist_data_model_element = FixedWordlistDataModelElement( "wordlist", [b'data: ', b'string: ']) decimal_integer_value_model_element = DecimalIntegerValueModelElement( "decimal") service_children_parsing_model_element.append( ElementValueBranchModelElement( 'ElementValueBranchModelElement', FirstMatchModelElement("first", [ SequenceModelElement( "seq1", [fixed_data_me1, fixed_wordlist_data_model_element]), SequenceModelElement("seq2", [ fixed_data_me1, fixed_wordlist_data_model_element, fixed_data_me2 ]) ]), "wordlist", { 0: decimal_integer_value_model_element, 1: fixed_data_me2 })) service_children_parsing_model_element.append( HexStringModelElement('HexStringModelElement')) service_children_parsing_model_element.append( SequenceModelElement('', [ FixedDataModelElement('FixedDataModelElement', b'Gateway IP-Address: '), IpAddressDataModelElement('IpAddressDataModelElement') ])) service_children_parsing_model_element.append( MultiLocaleDateTimeModelElement('MultiLocaleDateTimeModelElement', [(b'%b %d %Y', "de_AT.utf8", None)])) service_children_parsing_model_element.append( RepeatedElementDataModelElement( 'RepeatedElementDataModelElement', SequenceModelElement('SequenceModelElement', [ FixedDataModelElement('FixedDataModelElement', b'drawn number: '), DecimalIntegerValueModelElement( 'DecimalIntegerValueModelElement') ]), 1)) service_children_parsing_model_element.append( VariableByteDataModelElement('VariableByteDataModelElement', b'-@#')) service_children_parsing_model_element.append( SequenceModelElement('', [ WhiteSpaceLimitedDataModelElement( 'WhiteSpaceLimitedDataModelElement'), FixedDataModelElement('', b' ') ])) # The Base64StringModelElement must be just before the AnyByteDataModelElement to avoid unexpected Matches. service_children_parsing_model_element.append( Base64StringModelElement('Base64StringModelElement')) # The OptionalMatchModelElement must be paired with a FirstMatchModelElement because it accepts all data and thus no data gets # to the AnyByteDataModelElement. The AnyByteDataModelElement must be last, because all bytes are accepted. service_children_parsing_model_element.append( OptionalMatchModelElement( 'OptionalMatchModelElement', FirstMatchModelElement('FirstMatchModelElement', [ FixedDataModelElement('FixedDataModelElement', b'The-searched-element-was-found!'), AnyByteDataModelElement('AnyByteDataModelElement') ]))) parsing_model = FirstMatchModelElement('model', [ SequenceModelElement('CronAnnouncement', service_children_cron_job_announcement), SequenceModelElement('CronExecution', service_children_cron_job_execution), SequenceModelElement('DailyCron', service_children_cron_job), SequenceModelElement('DiskReport', service_children_disk_report), SequenceModelElement('LoginDetails', service_children_login_details), DecimalIntegerValueModelElement('Random'), SequenceModelElement('RandomTime', service_children_random_time), SequenceModelElement('Sensors', service_children_sensors), SequenceModelElement('IPAddresses', service_children_user_ip_address), FirstMatchModelElement('ParsingME', service_children_parsing_model_element) ]) # Some generic imports. from aminer.analysis import AtomFilters # Create all global handler lists here and append the real handlers # later on. # Use this filter to distribute all atoms to the analysis handlers. atom_filter = AtomFilters.SubhandlerFilter(None) from aminer.analysis.TimestampCorrectionFilters import SimpleMonotonicTimestampAdjust simple_monotonic_timestamp_adjust = SimpleMonotonicTimestampAdjust( [atom_filter]) analysis_context.register_component( simple_monotonic_timestamp_adjust, component_name="SimpleMonotonicTimestampAdjust") from aminer.events.StreamPrinterEventHandler import StreamPrinterEventHandler stream_printer_event_handler = StreamPrinterEventHandler(analysis_context) from aminer.events.SyslogWriterEventHandler import SyslogWriterEventHandler syslog_event_handler = SyslogWriterEventHandler(analysis_context) from aminer.events.DefaultMailNotificationEventHandler import DefaultMailNotificationEventHandler if DefaultMailNotificationEventHandler.CONFIG_KEY_MAIL_TARGET_ADDRESS in analysis_context.aminer_config.config_properties: mail_notification_handler = DefaultMailNotificationEventHandler( analysis_context) analysis_context.register_component(mail_notification_handler, component_name="MailHandler") anomaly_event_handlers = [ stream_printer_event_handler, syslog_event_handler, mail_notification_handler ] # Now define the AtomizerFactory using the model. A simple line based one is usually sufficient. from aminer.input.SimpleByteStreamLineAtomizerFactory import SimpleByteStreamLineAtomizerFactory analysis_context.atomizer_factory = SimpleByteStreamLineAtomizerFactory( parsing_model, [simple_monotonic_timestamp_adjust], anomaly_event_handlers) # Just report all unparsed atoms to the event handlers. from aminer.analysis.UnparsedAtomHandlers import SimpleUnparsedAtomHandler simple_unparsed_atom_handler = SimpleUnparsedAtomHandler( anomaly_event_handlers) atom_filter.add_handler(simple_unparsed_atom_handler, stop_when_handled_flag=True) analysis_context.register_component(simple_unparsed_atom_handler, component_name="UnparsedHandler") from aminer.analysis.TimestampsUnsortedDetector import TimestampsUnsortedDetector timestamps_unsorted_detector = TimestampsUnsortedDetector( analysis_context.aminer_config, anomaly_event_handlers) atom_filter.add_handler(timestamps_unsorted_detector) analysis_context.register_component( timestamps_unsorted_detector, component_name="TimestampsUnsortedDetector") from aminer.analysis import Rules from aminer.analysis.AllowlistViolationDetector import AllowlistViolationDetector allowlist_rules = [ Rules.OrMatchRule([ Rules.AndMatchRule([ Rules.PathExistsMatchRule( '/model/LoginDetails/PastTime/Time/Minutes'), Rules.NegationMatchRule( Rules.ValueMatchRule('/model/LoginDetails/Username', b'root')) ]), Rules.AndMatchRule([ Rules.NegationMatchRule( Rules.PathExistsMatchRule( '/model/LoginDetails/PastTime/Time/Minutes')), Rules.PathExistsMatchRule('/model/LoginDetails') ]), Rules.NegationMatchRule( Rules.PathExistsMatchRule('/model/LoginDetails')) ]) ] # This rule list should trigger, when the line does not look like: User root (logged in, logged out) # or User 'username' (logged in, logged out) x minutes ago. allowlist_violation_detector = AllowlistViolationDetector( analysis_context.aminer_config, allowlist_rules, anomaly_event_handlers) analysis_context.register_component(allowlist_violation_detector, component_name="Allowlist") atom_filter.add_handler(allowlist_violation_detector) from aminer.analysis.NewMatchPathDetector import NewMatchPathDetector new_match_path_detector = NewMatchPathDetector( analysis_context.aminer_config, anomaly_event_handlers, auto_include_flag=True) analysis_context.register_component(new_match_path_detector, component_name="NewMatchPath") atom_filter.add_handler(new_match_path_detector) def tuple_transformation_function(match_value_list): """Only allow output of the EnhancedNewMatchPathValueComboDetector after every 10000th element.""" extra_data = enhanced_new_match_path_value_combo_detector.known_values_dict.get( tuple(match_value_list)) if extra_data is not None: mod = 10000 if (extra_data[2] + 1) % mod == 0: enhanced_new_match_path_value_combo_detector.auto_include_flag = False else: enhanced_new_match_path_value_combo_detector.auto_include_flag = True return match_value_list from aminer.analysis.EnhancedNewMatchPathValueComboDetector import EnhancedNewMatchPathValueComboDetector enhanced_new_match_path_value_combo_detector = EnhancedNewMatchPathValueComboDetector( analysis_context.aminer_config, ['/model/DailyCron/UName', '/model/DailyCron/JobNumber'], anomaly_event_handlers, auto_include_flag=True, tuple_transformation_function=tuple_transformation_function) analysis_context.register_component( enhanced_new_match_path_value_combo_detector, component_name="EnhancedNewValueCombo") atom_filter.add_handler(enhanced_new_match_path_value_combo_detector) from aminer.analysis.HistogramAnalysis import HistogramAnalysis, LinearNumericBinDefinition, ModuloTimeBinDefinition, \ PathDependentHistogramAnalysis modulo_time_bin_definition = ModuloTimeBinDefinition( 86400, 3600, 0, 1, 24, True) linear_numeric_bin_definition = LinearNumericBinDefinition(50, 5, 20, True) histogram_analysis = HistogramAnalysis( analysis_context.aminer_config, [('/model/RandomTime/Random', modulo_time_bin_definition), ('/model/Random', linear_numeric_bin_definition)], 10, anomaly_event_handlers) analysis_context.register_component(histogram_analysis, component_name="HistogramAnalysis") atom_filter.add_handler(histogram_analysis) path_dependent_histogram_analysis = PathDependentHistogramAnalysis( analysis_context.aminer_config, '/model/RandomTime', modulo_time_bin_definition, 10, anomaly_event_handlers) analysis_context.register_component( path_dependent_histogram_analysis, component_name="PathDependentHistogramAnalysis") atom_filter.add_handler(path_dependent_histogram_analysis) from aminer.analysis.MatchValueAverageChangeDetector import MatchValueAverageChangeDetector match_value_average_change_detector = MatchValueAverageChangeDetector( analysis_context.aminer_config, anomaly_event_handlers, None, ['/model/Random'], 100, 10) analysis_context.register_component( match_value_average_change_detector, component_name="MatchValueAverageChange") atom_filter.add_handler(match_value_average_change_detector) import sys from aminer.analysis.MatchValueStreamWriter import MatchValueStreamWriter match_value_stream_writer = MatchValueStreamWriter(sys.stdout, [ '/model/Sensors/CPUTemp', '/model/Sensors/CPUWorkload', '/model/Sensors/DTM' ], b';', b'') analysis_context.register_component( match_value_stream_writer, component_name="MatchValueStreamWriter") atom_filter.add_handler(match_value_stream_writer) from aminer.analysis.NewMatchPathValueComboDetector import NewMatchPathValueComboDetector new_match_path_value_combo_detector = NewMatchPathValueComboDetector( analysis_context.aminer_config, ['/model/IPAddresses/Username', '/model/IPAddresses/IP'], anomaly_event_handlers, auto_include_flag=True) analysis_context.register_component( new_match_path_value_combo_detector, component_name="NewMatchPathValueCombo") atom_filter.add_handler(new_match_path_value_combo_detector) from aminer.analysis.NewMatchPathValueDetector import NewMatchPathValueDetector new_match_path_value_detector = NewMatchPathValueDetector( analysis_context.aminer_config, ['/model/DailyCron/JobNumber', '/model/IPAddresses/Username'], anomaly_event_handlers, auto_include_flag=True) analysis_context.register_component(new_match_path_value_detector, component_name="NewMatchPathValue") atom_filter.add_handler(new_match_path_value_detector) from aminer.analysis.MissingMatchPathValueDetector import MissingMatchPathValueDetector missing_match_path_value_detector = MissingMatchPathValueDetector( analysis_context.aminer_config, ['/model/DiskReport/Space'], anomaly_event_handlers, auto_include_flag=True, default_interval=2, realert_interval=5) analysis_context.register_component(missing_match_path_value_detector, component_name="MissingMatch") atom_filter.add_handler(missing_match_path_value_detector) from aminer.analysis.TimeCorrelationDetector import TimeCorrelationDetector time_correlation_detector = TimeCorrelationDetector( analysis_context.aminer_config, anomaly_event_handlers, 2, min_rule_attributes=1, max_rule_attributes=5, record_count_before_event=70000, output_log_line=True) analysis_context.register_component( time_correlation_detector, component_name="TimeCorrelationDetector") atom_filter.add_handler(time_correlation_detector) from aminer.analysis.TimeCorrelationViolationDetector import TimeCorrelationViolationDetector, CorrelationRule, EventClassSelector cron_job_announcement = CorrelationRule( 'CronJobAnnouncement', 5, 6, max_artefacts_a_for_single_b=1, artefact_match_parameters=[('/model/CronAnnouncement/JobNumber', '/model/CronExecution/JobNumber')]) a_class_selector = EventClassSelector('Announcement', [cron_job_announcement], None) b_class_selector = EventClassSelector('Execution', None, [cron_job_announcement]) rules = [ Rules.PathExistsMatchRule('/model/CronAnnouncement/Run', a_class_selector), Rules.PathExistsMatchRule('/model/CronExecution/Job', b_class_selector) ] time_correlation_violation_detector = TimeCorrelationViolationDetector( analysis_context.aminer_config, rules, anomaly_event_handlers) analysis_context.register_component( time_correlation_violation_detector, component_name="TimeCorrelationViolationDetector") atom_filter.add_handler(time_correlation_violation_detector)
def test2do_timer(self): """In this test case the functionality of the timer is tested. The eventCollectTime must not be 0.""" description = "Test2DefaultMailNotificationEventHandler" default_mail_notification_event_handler = DefaultMailNotificationEventHandler( self.analysis_context) self.analysis_context.register_component(self, description) t = time() match_context = MatchContext(self.pid) fixed_dme = FixedDataModelElement('s3', self.pid) match_element = fixed_dme.get_match_element("match", match_context) log_atom = LogAtom(fixed_dme.fixed_data, ParserMatch(match_element), t, self) default_mail_notification_event_handler.receive_event( self.test % self.__class__.__name__, 'New value for pathes %s: %s' % ('match/s3', repr(match_element.match_object)), [log_atom.raw_data], None, log_atom, self) t = 0 default_mail_notification_event_handler.do_timer(t) # skipcq: PYL-W1510, BAN-B602 result = subprocess.run(self.mail_call, shell=True, stdout=subprocess.PIPE) self.assertFalse( self.__expected_string % (match_element.get_path(), match_element.get_match_object(), self.__class__.__name__, description, 1, match_element.get_match_string().decode() + "\n\n") in str(result.stdout, 'utf-8')) t = time() default_mail_notification_event_handler.next_alert_time = t + 500 default_mail_notification_event_handler.do_timer(t) # skipcq: PYL-W1510, BAN-B602 result = subprocess.run(self.mail_call, shell=True, stdout=subprocess.PIPE) self.assertFalse( self.__expected_string % (match_element.get_path(), match_element.get_match_object(), self.__class__.__name__, description, 1, match_element.get_match_string().decode() + "\n\n") in str(result.stdout, 'utf-8')) default_mail_notification_event_handler.next_alert_time = t default_mail_notification_event_handler.do_timer(t) sleep(2) # skipcq: PYL-W1510, BAN-B602 result = subprocess.run(self.mail_call, shell=True, stdout=subprocess.PIPE) # skipcq: PYL-W1510, BAN-B602 subprocess.run(self.mail_delete_call, shell=True, stdout=subprocess.PIPE) if datetime.fromtimestamp(t).strftime( self.datetime_format_string) not in str( result.stdout, 'utf-8'): print("ERROR: %s t not found in mail!" % description, file=sys.stderr) self.assertTrue( self.__expected_string % (match_element.get_path(), match_element.get_match_object(), self.__class__.__name__, description, 1, match_element.get_match_string().decode() + "\n\n") in str(result.stdout, 'utf-8'), msg="%s vs \n %s" % (self.__expected_string % (match_element.get_path(), match_element.get_match_object(), self.__class__.__name__, description, 1, match_element.get_match_string().decode() + "\n\n"), str(result.stdout, 'utf-8')))
def build_analysis_pipeline(analysis_context): """ Define the function to create pipeline for parsing the log data. It has also to define an AtomizerFactory to instruct aminer how to process incoming data streams to create log atoms from them. """ # Build the parsing model: from aminer.parsing.FirstMatchModelElement import FirstMatchModelElement from aminer.parsing.SequenceModelElement import SequenceModelElement from aminer.parsing.DateTimeModelElement import DateTimeModelElement from aminer.parsing.FixedDataModelElement import FixedDataModelElement from aminer.parsing.DelimitedDataModelElement import DelimitedDataModelElement from aminer.parsing.AnyByteDataModelElement import AnyByteDataModelElement service_children_disk_upgrade = [ DateTimeModelElement('DTM', b'%Y-%m-%d %H:%M:%S'), FixedDataModelElement('UNameSpace1', b' '), DelimitedDataModelElement('UName', b' '), FixedDataModelElement('UNameSpace2', b' '), DelimitedDataModelElement('User', b' '), FixedDataModelElement('HDRepair', b' System rebooted for hard disk upgrade')] service_children_home_path = [ FixedDataModelElement('Pwd', b'The Path of the home directory shown by pwd of the user '), DelimitedDataModelElement('Username', b' '), FixedDataModelElement('Is', b' is: '), AnyByteDataModelElement('Path')] parsing_model = FirstMatchModelElement('model', [ SequenceModelElement('DiskUpgrade', service_children_disk_upgrade), SequenceModelElement('HomePath', service_children_home_path)]) # Some generic imports. from aminer.analysis import AtomFilters # Create all global handler lists here and append the real handlers later on. # Use this filter to distribute all atoms to the analysis handlers. atom_filter = AtomFilters.SubhandlerFilter(None) from aminer.events.StreamPrinterEventHandler import StreamPrinterEventHandler stream_printer_event_handler = StreamPrinterEventHandler(analysis_context) anomaly_event_handlers = [stream_printer_event_handler] # Now define the AtomizerFactory using the model. A simple line based one is usually sufficient. from aminer.input.SimpleByteStreamLineAtomizerFactory import SimpleByteStreamLineAtomizerFactory analysis_context.atomizer_factory = SimpleByteStreamLineAtomizerFactory(parsing_model, [atom_filter], anomaly_event_handlers) # Just report all unparsed atoms to the event handlers. from aminer.input.SimpleUnparsedAtomHandler import SimpleUnparsedAtomHandler simple_unparsed_atom_handler = SimpleUnparsedAtomHandler(anomaly_event_handlers) atom_filter.add_handler(simple_unparsed_atom_handler, stop_when_handled_flag=True) analysis_context.register_component(simple_unparsed_atom_handler, component_name="UnparsedHandler") from aminer.analysis.NewMatchPathDetector import NewMatchPathDetector new_match_path_detector = NewMatchPathDetector(analysis_context.aminer_config, anomaly_event_handlers, auto_include_flag=True) analysis_context.register_component(new_match_path_detector, component_name="NewPath") atom_filter.add_handler(new_match_path_detector) from aminer.analysis.NewMatchPathValueComboDetector import NewMatchPathValueComboDetector new_match_path_value_combo_detector = NewMatchPathValueComboDetector(analysis_context.aminer_config, [ '/model/HomePath/Username', '/model/HomePath/Path'], anomaly_event_handlers, auto_include_flag=True) analysis_context.register_component(new_match_path_value_combo_detector, component_name="NewValueCombo") atom_filter.add_handler(new_match_path_value_combo_detector) # Include the e-mail notification handler only if the configuration parameter was set. from aminer.events.DefaultMailNotificationEventHandler import DefaultMailNotificationEventHandler if DefaultMailNotificationEventHandler.CONFIG_KEY_MAIL_TARGET_ADDRESS in analysis_context.aminer_config.config_properties: mail_notification_handler = DefaultMailNotificationEventHandler(analysis_context) analysis_context.register_component(mail_notification_handler, component_name="MailHandler") anomaly_event_handlers.append(mail_notification_handler)