def test3get_match_element_valid_match(self): """Parse matching substring from MatchContext and check if the MatchContext was updated with all characters.""" element_value_branch_me = ElementValueBranchModelElement( self.id_, self.value_model, None, { "path: ": self.path_me, "data: ": self.data_me }) data = b"path: /model" match_context = DummyMatchContext(data) match_element = element_value_branch_me.get_match_element( self.path, match_context) self.compare_match_results( data, match_element, match_context, self.id_, self.path, data, data, [ MatchElement("path/value_branch/branch/path", self.path_path, self.path_path, None), MatchElement("path/value_branch/value_model", self.path_fixed_string, self.path_fixed_string, None) ]) data = b"data: this is some random data: 255." match_context = DummyMatchContext(data) match_element = element_value_branch_me.get_match_element( self.path, match_context) self.compare_match_results( data, match_element, match_context, self.id_, self.path, data, data, [ MatchElement("path/value_branch/branch/data", self.data_path, self.data_path, None), MatchElement("path/value_branch/value_model", self.data_fixed_string, self.data_fixed_string, None) ])
def test1get_id(self): """Test if get_id works properly.""" element_value_branch_me = ElementValueBranchModelElement( self.id_, self.value_model, None, { "path: ": self.path_me, "data: ": self.data_me }) self.assertEqual(element_value_branch_me.get_id(), self.id_)
def test2get_child_elements(self): """Test if get_child_elements returns None.""" element_value_branch_me = ElementValueBranchModelElement( self.id_, self.value_model, None, { "path: ": self.path_me, "data: ": self.data_me }) self.assertEqual(element_value_branch_me.get_child_elements(), self.children)
def test3_get_child_elements(self): """In this test case the functionality of the get_child_elements-method is tested.""" element_value_branch_model_element = ElementValueBranchModelElement( "id", self.first_match_me, "wordlist", { 0: self.decimal_integer_value_model_element, 1: self.fixed_data_me2 }) self.assertEqual( element_value_branch_model_element.get_child_elements(), [ self.first_match_me, self.decimal_integer_value_model_element, self.fixed_data_me2 ])
def test1_match_element_found(self): """In this test case different possible parameters are used to obtain a MatchElement successfully.""" match_context = MatchContext(self.match_data) element_value_branch_model_element = ElementValueBranchModelElement( "id", self.first_match_me, "wordlist", {0: self.decimal_integer_value_model_element, 1: self.fixed_data_me2}) match_element = element_value_branch_model_element.get_match_element("elementValueBranchME match", match_context) self.assertEqual(match_element.get_path(), 'elementValueBranchME match/id') self.assertEqual(match_element.get_match_string(), self.match_data) self.assertEqual(match_element.get_match_object(), self.match_data) self.assertEqual(match_element.get_children()[0].get_path(), 'elementValueBranchME match/id/first/seq1') self.assertEqual(match_element.get_children()[0].get_match_string(), b'match data: ') self.assertEqual(match_element.get_children()[0].get_match_object(), b'match data: ') self.assertEqual(match_element.get_children()[0].get_children()[0].get_path(), 'elementValueBranchME match/id/first/seq1/fixed 1') self.assertEqual(match_element.get_children()[0].get_children()[0].get_match_string(), self.match) self.assertEqual(match_element.get_children()[0].get_children()[0].get_match_object(), self.match) self.assertEqual(match_element.get_children()[0].get_children()[0].get_children(), None) self.assertEqual(match_element.get_children()[0].get_children()[1].get_path(), 'elementValueBranchME match/id/first/seq1/wordlist') self.assertEqual(match_element.get_children()[0].get_children()[1].get_match_string(), b'data: ') self.assertEqual(match_element.get_children()[0].get_children()[1].get_match_object(), 0) self.assertEqual(match_element.get_children()[0].get_children()[1].get_children(), None) self.assertEqual(match_element.get_children()[1].get_path(), 'elementValueBranchME match/id/decimal') self.assertEqual(match_element.get_children()[1].get_match_string(), b'25000') self.assertEqual(match_element.get_children()[1].get_match_object(), 25000) self.assertEqual(match_element.get_children()[1].get_children(), None) match_context = MatchContext(self.match_string) match_element = element_value_branch_model_element.get_match_element("elementValueBranchME match", match_context) self.assertEqual(match_element.get_path(), 'elementValueBranchME match/id') self.assertEqual(match_element.get_match_string(), self.match_string) self.assertEqual(match_element.get_match_object(), self.match_string) self.assertEqual(match_element.get_children()[0].get_path(), 'elementValueBranchME match/id/first/seq1') self.assertEqual(match_element.get_children()[0].get_match_string(), b'match string: ') self.assertEqual(match_element.get_children()[0].get_match_object(), b'match string: ') self.assertEqual(match_element.get_children()[0].get_children()[0].get_path(), 'elementValueBranchME match/id/first/seq1/fixed 1') self.assertEqual(match_element.get_children()[0].get_children()[0].get_match_string(), self.match) self.assertEqual(match_element.get_children()[0].get_children()[0].get_match_object(), self.match) self.assertEqual(match_element.get_children()[0].get_children()[0].get_children(), None) self.assertEqual(match_element.get_children()[0].get_children()[1].get_path(), 'elementValueBranchME match/id/first/seq1/wordlist') self.assertEqual(match_element.get_children()[0].get_children()[1].get_match_string(), b'string: ') self.assertEqual(match_element.get_children()[0].get_children()[1].get_match_object(), 1) self.assertEqual(match_element.get_children()[0].get_children()[1].get_children(), None) self.assertEqual(match_element.get_children()[1].get_path(), 'elementValueBranchME match/id/fixed 2') self.assertEqual(match_element.get_children()[1].get_match_string(), self.fixed_string) self.assertEqual(match_element.get_children()[1].get_match_object(), self.fixed_string) self.assertEqual(match_element.get_children()[1].get_children(), None)
def test10get_match_element_match_context_input_validation(self): """Check if an exception is raised, when other classes than MatchContext are used in get_match_element.""" model_element = ElementValueBranchModelElement( self.id_, self.value_model, None, { "path: ": self.path_me, "data: ": self.data_me }) data = b"abcdefghijklmnopqrstuvwxyz.!?" model_element.get_match_element(self.path, DummyMatchContext(data)) model_element.get_match_element(self.path, MatchContext(data)) self.assertRaises(AttributeError, model_element.get_match_element, self.path, MatchElement(None, data, None, None)) self.assertRaises(AttributeError, model_element.get_match_element, self.path, data) self.assertRaises(AttributeError, model_element.get_match_element, self.path, data.decode()) self.assertRaises(AttributeError, model_element.get_match_element, self.path, 123) self.assertRaises(AttributeError, model_element.get_match_element, self.path, 123.22) self.assertRaises(AttributeError, model_element.get_match_element, self.path, True) self.assertRaises(AttributeError, model_element.get_match_element, self.path, None) self.assertRaises(AttributeError, model_element.get_match_element, self.path, []) self.assertRaises(AttributeError, model_element.get_match_element, self.path, {"key": MatchContext(data)}) self.assertRaises(AttributeError, model_element.get_match_element, self.path, set()) self.assertRaises(AttributeError, model_element.get_match_element, self.path, ()) self.assertRaises(AttributeError, model_element.get_match_element, self.path, model_element)
def test4get_match_element_no_match(self): """Parse not matching substring from MatchContext and check if the MatchContext was not changed.""" element_value_branch_me = ElementValueBranchModelElement( self.id_, self.value_model, None, { "path: ": self.path_me, "data: ": self.data_me }) data = b"path: /random" match_context = DummyMatchContext(data) match_element = element_value_branch_me.get_match_element( self.path, match_context) self.compare_no_match_results(data, match_element, match_context) data = b"path: this is some random data: 255." match_context = DummyMatchContext(data) match_element = element_value_branch_me.get_match_element( self.path, match_context) self.compare_no_match_results(data, match_element, match_context) data = b"data: /model" match_context = DummyMatchContext(data) match_element = element_value_branch_me.get_match_element( self.path, match_context) self.compare_no_match_results(data, match_element, match_context) data = b"path: " match_context = DummyMatchContext(data) match_element = element_value_branch_me.get_match_element( self.path, match_context) self.compare_no_match_results(data, match_element, match_context) data = b"data: " match_context = DummyMatchContext(data) match_element = element_value_branch_me.get_match_element( self.path, match_context) self.compare_no_match_results(data, match_element, match_context)
def build_analysis_pipeline(analysis_context): """ Define the function to create pipeline for parsing the log data. It has also to define an AtomizerFactory to instruct aminer how to process incoming data streams to create log atoms from them. """ date_format_string = b'%Y-%m-%d %H:%M:%S' cron = b' cron[' # Build the parsing model: service_children_disk_report = [ FixedDataModelElement('Space', b' Current Disk Data is: Filesystem Type Size Used Avail Use%'), DelimitedDataModelElement('Data', b'%'), AnyByteDataModelElement('Rest')] service_children_login_details = [ FixedDataModelElement('User/LoginDetails', b'User '), DelimitedDataModelElement('Username', b' '), FixedWordlistDataModelElement('Status', [b' logged in', b' logged out']), OptionalMatchModelElement('PastTime', SequenceModelElement('Time', [ FixedDataModelElement('Blank', b' '), DecimalIntegerValueModelElement('Minutes'), FixedDataModelElement('Ago', b' minutes ago.')]))] service_children_cron_job = [ DateTimeModelElement('DTM', date_format_string), FixedDataModelElement('UNameSpace1', b' '), DelimitedDataModelElement('UName', b' '), FixedDataModelElement('UNameSpace2', b' '), DelimitedDataModelElement('User', b' '), FixedDataModelElement('Cron', cron), DecimalIntegerValueModelElement('JobNumber'), FixedDataModelElement('Details', b']: Job `cron.daily` started.')] service_children_random_time = [FixedDataModelElement('Space', b'Random: '), DecimalIntegerValueModelElement('Random')] service_children_sensors = [SequenceModelElement('CPUTemp', [ FixedDataModelElement('FixedTemp', b'CPU Temp: '), DecimalIntegerValueModelElement('Temp'), FixedDataModelElement('Degrees', b'\xc2\xb0C')]), FixedDataModelElement('Space1', b', '), SequenceModelElement('CPUWorkload', [ FixedDataModelElement('FixedWorkload', b'CPU Workload: '), DecimalIntegerValueModelElement('Workload'), FixedDataModelElement('Percent', b'%')]), FixedDataModelElement('Space2', b', '), DateTimeModelElement('DTM', date_format_string)] service_children_user_ip_address = [ FixedDataModelElement('User/UserIPAddress', b'User '), DelimitedDataModelElement('Username', b' '), FixedDataModelElement('Action', b' changed IP address to '), IpAddressDataModelElement('IP')] service_children_cron_job_announcement = [ DateTimeModelElement('DTM', date_format_string), FixedDataModelElement('Space', b' '), DelimitedDataModelElement('UName', b' '), FixedDataModelElement('Cron', cron), DecimalIntegerValueModelElement('JobNumber'), FixedDataModelElement('Run', b']: Will run job `'), FixedWordlistDataModelElement('CronType', [b'cron.daily', b'cron.hourly', b'cron.monthly', b'cron.weekly']), FixedDataModelElement('StartTime', b'\' in 5 min.')] service_children_cron_job_execution = [ DateTimeModelElement('DTM', date_format_string), FixedDataModelElement('Space1', b' '), DelimitedDataModelElement('UName', b' '), FixedDataModelElement('Cron', cron), DecimalIntegerValueModelElement('JobNumber'), FixedDataModelElement('Job', b']: Job `'), FixedWordlistDataModelElement('CronType', [b'cron.daily', b'cron.hourly', b'cron.monthly', b'cron.weekly']), FixedDataModelElement('Started', b'\' started')] service_children_audit = [SequenceModelElement('path', [ FixedDataModelElement('type', b'type=PATH '), FixedDataModelElement('msg_audit', b'msg=audit('), DelimitedDataModelElement('msg', b':'), FixedDataModelElement('placeholder', b':'), DecimalIntegerValueModelElement('id'), FixedDataModelElement('item_string', b'): item='), DecimalIntegerValueModelElement('item'), FixedDataModelElement('name_string', b' name="'), DelimitedDataModelElement('name', b'"'), FixedDataModelElement('inode_string', b'" inode='), DecimalIntegerValueModelElement('inode'), FixedDataModelElement('dev_string', b' dev='), DelimitedDataModelElement('dev', b' '), FixedDataModelElement('mode_string', b' mode='), DecimalIntegerValueModelElement('mode', value_pad_type=DecimalIntegerValueModelElement.PAD_TYPE_ZERO), FixedDataModelElement('ouid_string', b' ouid='), DecimalIntegerValueModelElement('ouid'), FixedDataModelElement('ogid_string', b' ogid='), DecimalIntegerValueModelElement('ogid'), FixedDataModelElement('rdev_string', b' rdev='), DelimitedDataModelElement('rdev', b' '), FixedDataModelElement('nametype_string', b' nametype='), FixedWordlistDataModelElement('nametype', [b'NORMAL', b'ERROR'])]), SequenceModelElement('syscall', [ FixedDataModelElement('type', b'type=SYSCALL '), FixedDataModelElement('msg_audit', b'msg=audit('), DelimitedDataModelElement('msg', b':'), FixedDataModelElement('placeholder', b':'), DecimalIntegerValueModelElement('id'), FixedDataModelElement('arch_string', b'): arch='), DelimitedDataModelElement('arch', b' '), FixedDataModelElement('syscall_string', b' syscall='), DecimalIntegerValueModelElement('syscall'), FixedDataModelElement('success_string', b' success='), FixedWordlistDataModelElement('success', [b'yes', b'no']), FixedDataModelElement('exit_string', b' exit='), DecimalIntegerValueModelElement('exit'), AnyByteDataModelElement('remainding_data')])] service_children_parsing_model_element = [ DateTimeModelElement('DateTimeModelElement', b'Current DateTime: %d.%m.%Y %H:%M:%S'), DecimalFloatValueModelElement('DecimalFloatValueModelElement', value_sign_type='optional'), DecimalIntegerValueModelElement('DecimalIntegerValueModelElement', value_sign_type='optional', value_pad_type='blank'), SequenceModelElement('se', [ DelimitedDataModelElement('DelimitedDataModelElement', b';'), FixedDataModelElement('FixedDataModelElement', b';')])] # ElementValueBranchModelElement fixed_data_me1 = FixedDataModelElement("fixed1", b'match ') fixed_data_me2 = FixedDataModelElement("fixed2", b'fixed String') fixed_wordlist_data_model_element = FixedWordlistDataModelElement("wordlist", [b'data: ', b'string: ']) decimal_integer_value_model_element = DecimalIntegerValueModelElement("decimal") service_children_parsing_model_element.append( ElementValueBranchModelElement('ElementValueBranchModelElement', FirstMatchModelElement("first", [ SequenceModelElement("seq1", [fixed_data_me1, fixed_wordlist_data_model_element]), SequenceModelElement("seq2", [fixed_data_me1, fixed_wordlist_data_model_element, fixed_data_me2])]), "wordlist", {0: decimal_integer_value_model_element, 1: fixed_data_me2})) service_children_parsing_model_element.append(HexStringModelElement('HexStringModelElement')) service_children_parsing_model_element.append(SequenceModelElement('se2', [ FixedDataModelElement('FixedDataModelElement', b'Gateway IP-Address: '), IpAddressDataModelElement('IpAddressDataModelElement')])) import locale loc = locale.getlocale() if loc == (None, None): loc = ('en_US', 'utf8') service_children_parsing_model_element.append( MultiLocaleDateTimeModelElement('MultiLocaleDateTimeModelElement', [(b'%b %d %Y', None, '%s.%s' % loc)])) service_children_parsing_model_element.append( RepeatedElementDataModelElement('RepeatedElementDataModelElement', SequenceModelElement('SequenceModelElement', [ FixedDataModelElement('FixedDataModelElement', b'[drawn number]: '), DecimalIntegerValueModelElement('DecimalIntegerValueModelElement')]), 1)) service_children_parsing_model_element.append(VariableByteDataModelElement('VariableByteDataModelElement', b'-@#')) service_children_parsing_model_element.append(SequenceModelElement('se', [ WhiteSpaceLimitedDataModelElement('WhiteSpaceLimitedDataModelElement'), FixedDataModelElement('fixed', b' ')])) # The Base64StringModelElement must be just before the AnyByteDataModelElement to avoid unexpected Matches. service_children_parsing_model_element.append(Base64StringModelElement('Base64StringModelElement')) # The OptionalMatchModelElement must be paired with a FirstMatchModelElement because it accepts all data and thus no data gets # to the AnyByteDataModelElement. The AnyByteDataModelElement must be last, because all bytes are accepted. service_children_parsing_model_element.append( OptionalMatchModelElement('/', FirstMatchModelElement('FirstMatchModelElement//optional', [ FixedDataModelElement('FixedDataModelElement', b'The-searched-element-was-found!'), SequenceModelElement('se', [ FixedDataModelElement('FixedDME', b'Any:'), AnyByteDataModelElement('AnyByteDataModelElement')])]))) alphabet = b'ghijkl' service_children_ecd = [] for _, char in enumerate(alphabet): char = bytes([char]) service_children_ecd.append(FixedDataModelElement(char.decode(), char)) parsing_model = FirstMatchModelElement('model', [ SequenceModelElement('CronAnnouncement', service_children_cron_job_announcement), SequenceModelElement('CronExecution', service_children_cron_job_execution), SequenceModelElement('DailyCron', service_children_cron_job), SequenceModelElement('DiskReport', service_children_disk_report), SequenceModelElement('LoginDetails', service_children_login_details), DecimalIntegerValueModelElement('Random'), SequenceModelElement('RandomTime', service_children_random_time), SequenceModelElement('Sensors', service_children_sensors), SequenceModelElement('IPAddresses', service_children_user_ip_address), FirstMatchModelElement('type', service_children_audit), FirstMatchModelElement('ECD', service_children_ecd), FirstMatchModelElement('ParsingME', service_children_parsing_model_element)]) # Some generic imports. from aminer.analysis import AtomFilters # Create all global handler lists here and append the real handlers later on. # Use this filter to distribute all atoms to the analysis handlers. atom_filter = AtomFilters.SubhandlerFilter(None) from aminer.analysis.TimestampCorrectionFilters import SimpleMonotonicTimestampAdjust simple_monotonic_timestamp_adjust = SimpleMonotonicTimestampAdjust([atom_filter]) analysis_context.register_component(simple_monotonic_timestamp_adjust, component_name="SimpleMonotonicTimestampAdjust") from aminer.events.StreamPrinterEventHandler import StreamPrinterEventHandler from aminer.events.JsonConverterHandler import JsonConverterHandler stream_printer_event_handler = StreamPrinterEventHandler(analysis_context) json_converter_handler = JsonConverterHandler([stream_printer_event_handler], analysis_context) anomaly_event_handlers = [json_converter_handler] # Now define the AtomizerFactory using the model. A simple line based one is usually sufficient. from aminer.input.SimpleByteStreamLineAtomizerFactory import SimpleByteStreamLineAtomizerFactory analysis_context.atomizer_factory = SimpleByteStreamLineAtomizerFactory(parsing_model, [simple_monotonic_timestamp_adjust], anomaly_event_handlers) # Just report all unparsed atoms to the event handlers. from aminer.analysis.UnparsedAtomHandlers import SimpleUnparsedAtomHandler, VerboseUnparsedAtomHandler simple_unparsed_atom_handler = SimpleUnparsedAtomHandler(anomaly_event_handlers) atom_filter.add_handler(simple_unparsed_atom_handler, stop_when_handled_flag=False) analysis_context.register_component(simple_unparsed_atom_handler, component_name="SimpleUnparsedHandler") verbose_unparsed_atom_handler = VerboseUnparsedAtomHandler(anomaly_event_handlers, parsing_model) atom_filter.add_handler(verbose_unparsed_atom_handler, stop_when_handled_flag=True) analysis_context.register_component(verbose_unparsed_atom_handler, component_name="VerboseUnparsedHandler") from aminer.analysis.TimestampsUnsortedDetector import TimestampsUnsortedDetector timestamps_unsorted_detector = TimestampsUnsortedDetector(analysis_context.aminer_config, anomaly_event_handlers) atom_filter.add_handler(timestamps_unsorted_detector) analysis_context.register_component(timestamps_unsorted_detector, component_name="TimestampsUnsortedDetector") from aminer.analysis import Rules from aminer.analysis.AllowlistViolationDetector import AllowlistViolationDetector allowlist_rules = [ Rules.OrMatchRule([ Rules.AndMatchRule([ Rules.PathExistsMatchRule('/model/LoginDetails/PastTime/Time/Minutes'), Rules.NegationMatchRule(Rules.ValueMatchRule('/model/LoginDetails/Username', b'root')), Rules.DebugMatchRule(debug_match_result=True)]), Rules.AndMatchRule([ Rules.NegationMatchRule(Rules.PathExistsMatchRule('/model/LoginDetails/PastTime/Time/Minutes')), Rules.PathExistsMatchRule('/model/LoginDetails'), Rules.DebugMatchRule(debug_match_result=True)]), Rules.NegationMatchRule(Rules.PathExistsMatchRule('/model/LoginDetails'))])] # This rule list should trigger, when the line does not look like: User root (logged in, logged out) # or User 'username' (logged in, logged out) x minutes ago. allowlist_violation_detector = AllowlistViolationDetector(analysis_context.aminer_config, allowlist_rules, anomaly_event_handlers, output_log_line=True) analysis_context.register_component(allowlist_violation_detector, component_name="Allowlist") atom_filter.add_handler(allowlist_violation_detector) from aminer.analysis.ParserCount import ParserCount parser_count = ParserCount(analysis_context.aminer_config, None, anomaly_event_handlers, 10) analysis_context.register_component(parser_count, component_name="ParserCount") atom_filter.add_handler(parser_count) from aminer.analysis.EventTypeDetector import EventTypeDetector etd = EventTypeDetector(analysis_context.aminer_config, anomaly_event_handlers) analysis_context.register_component(etd, component_name="EventTypeDetector") atom_filter.add_handler(etd) from aminer.analysis.VariableTypeDetector import VariableTypeDetector vtd = VariableTypeDetector(analysis_context.aminer_config, anomaly_event_handlers, etd, silence_output_except_indicator=False, output_log_line=False, ignore_list=["/model/RandomTime"]) analysis_context.register_component(vtd, component_name="VariableTypeDetector") atom_filter.add_handler(vtd) from aminer.analysis.VariableCorrelationDetector import VariableCorrelationDetector vtd = VariableCorrelationDetector(analysis_context.aminer_config, anomaly_event_handlers, etd, disc_div_thres=0.5, ignore_list=["/model/RandomTime"]) analysis_context.register_component(vtd, component_name="VariableCorrelationDetector") atom_filter.add_handler(vtd) from aminer.analysis.EventCorrelationDetector import EventCorrelationDetector ecd = EventCorrelationDetector(analysis_context.aminer_config, anomaly_event_handlers, check_rules_flag=True, hypothesis_max_delta_time=1.0) analysis_context.register_component(ecd, component_name="EventCorrelationDetector") atom_filter.add_handler(ecd) from aminer.analysis.EventFrequencyDetector import EventFrequencyDetector efd = EventFrequencyDetector(analysis_context.aminer_config, anomaly_event_handlers, window_size=0.1) analysis_context.register_component(efd, component_name="EventFrequencyDetector") atom_filter.add_handler(efd) from aminer.analysis.EventSequenceDetector import EventSequenceDetector esd = EventSequenceDetector(analysis_context.aminer_config, anomaly_event_handlers, ['/model/ParsingME'], ignore_list=[ '/model/ECD/g', '/model/ECD/h', '/model/ECD/i', '/model/ECD/j', '/model/ECD/k', '/model/ECD/l', '/model/Random', '/model/RandomTime', '/model/DailyCron']) analysis_context.register_component(esd, component_name="EventSequenceDetector") atom_filter.add_handler(esd) from aminer.analysis.MatchFilter import MatchFilter match_filter = MatchFilter(analysis_context.aminer_config, ['/model/Random'], anomaly_event_handlers, target_value_list=[ 1, 10, 100], output_log_line=True) analysis_context.register_component(match_filter, component_name="MatchFilter") atom_filter.add_handler(match_filter) from aminer.analysis.NewMatchPathDetector import NewMatchPathDetector new_match_path_detector = NewMatchPathDetector(analysis_context.aminer_config, anomaly_event_handlers, auto_include_flag=True, output_log_line=True) analysis_context.register_component(new_match_path_detector, component_name="NewMatchPath") atom_filter.add_handler(new_match_path_detector) def tuple_transformation_function(match_value_list): """Only allow output of the EnhancedNewMatchPathValueComboDetector after every 10th element.""" extra_data = enhanced_new_match_path_value_combo_detector.known_values_dict.get(tuple(match_value_list)) if extra_data is not None: mod = 10 if (extra_data[2] + 1) % mod == 0: enhanced_new_match_path_value_combo_detector.auto_include_flag = False else: enhanced_new_match_path_value_combo_detector.auto_include_flag = True return match_value_list from aminer.analysis.EnhancedNewMatchPathValueComboDetector import EnhancedNewMatchPathValueComboDetector enhanced_new_match_path_value_combo_detector = EnhancedNewMatchPathValueComboDetector(analysis_context.aminer_config, [ '/model/DailyCron/UName', '/model/DailyCron/JobNumber'], anomaly_event_handlers, auto_include_flag=True, tuple_transformation_function=tuple_transformation_function, output_log_line=True) analysis_context.register_component(enhanced_new_match_path_value_combo_detector, component_name="EnhancedNewValueCombo") atom_filter.add_handler(enhanced_new_match_path_value_combo_detector) import re ip_match_action = Rules.EventGenerationMatchAction( "Analysis.Rules.IPv4InRFC1918MatchRule", "Private IP address occurred!", anomaly_event_handlers) vdmt = Rules.ValueDependentModuloTimeMatchRule(None, 3, ["/model/ECD/j", "/model/ECD/k", "/model/ECD/l"], {b"e": [0, 2.95]}, [0, 3]) mt = Rules.ModuloTimeMatchRule(None, 3, 0, 3, None) time_allowlist_rules = [ Rules.AndMatchRule([ Rules.ParallelMatchRule([ Rules.ValueDependentDelegatedMatchRule([ '/model/ECD/g', '/model/ECD/h', '/model/ECD/i', '/model/ECD/j', '/model/ECD/k', '/model/ECD/l'], { (b"a",): mt, (b"b",): mt, (b"c",): mt, (b"d",): vdmt, (b"e",): vdmt, (b"f",): vdmt, None: mt}, mt), Rules.IPv4InRFC1918MatchRule("/model/ParsingME/se2/IpAddressDataModelElement", ip_match_action), Rules.DebugHistoryMatchRule(debug_match_result=True) ]), # IP addresses 8.8.8.8, 8.8.4.4 and 10.0.0.0 - 10.255.255.255 are not allowed Rules.NegationMatchRule(Rules.ValueListMatchRule("/model/ParsingME/se2/IpAddressDataModelElement", [134744072, 134743044])), Rules.NegationMatchRule(Rules.ValueRangeMatchRule("/model/ParsingME/se2/IpAddressDataModelElement", 167772160, 184549375)), Rules.NegationMatchRule(Rules.StringRegexMatchRule("/model/type/syscall/success", re.compile(b"^no$"))) ]) ] time_allowlist_violation_detector = AllowlistViolationDetector( analysis_context.aminer_config, time_allowlist_rules, anomaly_event_handlers, output_log_line=True) analysis_context.register_component(time_allowlist_violation_detector, component_name="TimeAllowlist") atom_filter.add_handler(time_allowlist_violation_detector) from aminer.analysis.HistogramAnalysis import HistogramAnalysis, LinearNumericBinDefinition, ModuloTimeBinDefinition, \ PathDependentHistogramAnalysis modulo_time_bin_definition = ModuloTimeBinDefinition(86400, 3600, 0, 1, 24, True) linear_numeric_bin_definition = LinearNumericBinDefinition(50, 5, 20, True) histogram_analysis = HistogramAnalysis(analysis_context.aminer_config, [ ('/model/RandomTime/Random', modulo_time_bin_definition), ('/model/Random', linear_numeric_bin_definition)], 10, anomaly_event_handlers, output_log_line=True) analysis_context.register_component(histogram_analysis, component_name="HistogramAnalysis") atom_filter.add_handler(histogram_analysis) path_dependent_histogram_analysis = PathDependentHistogramAnalysis( analysis_context.aminer_config, '/model/RandomTime', modulo_time_bin_definition, 10, anomaly_event_handlers, output_log_line=True) analysis_context.register_component(path_dependent_histogram_analysis, component_name="PathDependentHistogramAnalysis") atom_filter.add_handler(path_dependent_histogram_analysis) from aminer.analysis.MatchValueAverageChangeDetector import MatchValueAverageChangeDetector match_value_average_change_detector = MatchValueAverageChangeDetector(analysis_context.aminer_config, anomaly_event_handlers, None, [ '/model/Random'], 100, 10, output_log_line=True) analysis_context.register_component(match_value_average_change_detector, component_name="MatchValueAverageChange") atom_filter.add_handler(match_value_average_change_detector) import sys from aminer.analysis.MatchValueStreamWriter import MatchValueStreamWriter match_value_stream_writer = MatchValueStreamWriter( sys.stdout, ['/model/Sensors/CPUTemp', '/model/Sensors/CPUWorkload', '/model/Sensors/DTM'], b';', b'') analysis_context.register_component(match_value_stream_writer, component_name="MatchValueStreamWriter") atom_filter.add_handler(match_value_stream_writer) from aminer.analysis.NewMatchPathValueComboDetector import NewMatchPathValueComboDetector new_match_path_value_combo_detector = NewMatchPathValueComboDetector( analysis_context.aminer_config, ['/model/IPAddresses/Username', '/model/IPAddresses/IP'], anomaly_event_handlers, output_log_line=True) analysis_context.register_component(new_match_path_value_combo_detector, component_name="NewMatchPathValueCombo") atom_filter.add_handler(new_match_path_value_combo_detector) from aminer.analysis.NewMatchIdValueComboDetector import NewMatchIdValueComboDetector new_match_id_value_combo_detector = NewMatchIdValueComboDetector(analysis_context.aminer_config, [ '/model/type/path/name', '/model/type/syscall/syscall'], anomaly_event_handlers, id_path_list=[ '/model/type/path/id', '/model/type/syscall/id'], min_allowed_time_diff=5, auto_include_flag=True, allow_missing_values_flag=True, output_log_line=True) analysis_context.register_component(new_match_id_value_combo_detector, component_name="NewMatchIdValueComboDetector") atom_filter.add_handler(new_match_id_value_combo_detector) from aminer.analysis.NewMatchPathValueDetector import NewMatchPathValueDetector new_match_path_value_detector = NewMatchPathValueDetector(analysis_context.aminer_config, [ '/model/DailyCron/JobNumber', '/model/IPAddresses/Username'], anomaly_event_handlers, auto_include_flag=True, output_log_line=True) analysis_context.register_component(new_match_path_value_detector, component_name="NewMatchPathValue") atom_filter.add_handler(new_match_path_value_detector) from aminer.analysis.MissingMatchPathValueDetector import MissingMatchPathValueDetector missing_match_path_value_detector = MissingMatchPathValueDetector( analysis_context.aminer_config, ['/model/DiskReport/Space'], anomaly_event_handlers, auto_include_flag=True, default_interval=2, realert_interval=5, output_log_line=True) analysis_context.register_component(missing_match_path_value_detector, component_name="MissingMatch") atom_filter.add_handler(missing_match_path_value_detector) from aminer.analysis.TimeCorrelationDetector import TimeCorrelationDetector time_correlation_detector = TimeCorrelationDetector( analysis_context.aminer_config, anomaly_event_handlers, 2, min_rule_attributes=1, max_rule_attributes=5, record_count_before_event=10000, output_log_line=True) analysis_context.register_component(time_correlation_detector, component_name="TimeCorrelationDetector") atom_filter.add_handler(time_correlation_detector) from aminer.analysis.TimeCorrelationViolationDetector import TimeCorrelationViolationDetector, CorrelationRule, EventClassSelector cron_job_announcement = CorrelationRule('CronJobAnnouncement', 5, 6, max_artefacts_a_for_single_b=1, artefact_match_parameters=[ ('/model/CronAnnouncement/JobNumber', '/model/CronExecution/JobNumber')]) a_class_selector = EventClassSelector('Announcement', [cron_job_announcement], None) b_class_selector = EventClassSelector('Execution', None, [cron_job_announcement]) rules = [Rules.PathExistsMatchRule('/model/CronAnnouncement/Run', a_class_selector), Rules.PathExistsMatchRule('/model/CronExecution/Job', b_class_selector)] time_correlation_violation_detector = TimeCorrelationViolationDetector(analysis_context.aminer_config, rules, anomaly_event_handlers, output_log_line=True) analysis_context.register_component(time_correlation_violation_detector, component_name="TimeCorrelationViolationDetector") atom_filter.add_handler(time_correlation_violation_detector)
def build_analysis_pipeline(analysis_context): """ Define the function to create pipeline for parsing the log data. It has also to define an AtomizerFactory to instruct aminer how to process incoming data streams to create log atoms from them. """ date_format_string = b'%Y-%m-%d %H:%M:%S' cron = b' cron[' # Build the parsing model: service_children_disk_report = [ FixedDataModelElement( 'Space', b' Current Disk Data is: Filesystem Type Size Used Avail Use%' ), DelimitedDataModelElement('Data', b'%'), AnyByteDataModelElement('Rest') ] service_children_login_details = [ FixedDataModelElement('User', b'User '), DelimitedDataModelElement('Username', b' '), FixedWordlistDataModelElement('Status', [b' logged in', b' logged out']), OptionalMatchModelElement( 'PastTime', SequenceModelElement('Time', [ FixedDataModelElement('Blank', b' '), DecimalIntegerValueModelElement('Minutes'), FixedDataModelElement('Ago', b' minutes ago.') ])) ] service_children_cron_job = [ DateTimeModelElement('DTM', date_format_string), FixedDataModelElement('UNameSpace1', b' '), DelimitedDataModelElement('UName', b' '), FixedDataModelElement('UNameSpace2', b' '), DelimitedDataModelElement('User', b' '), FixedDataModelElement('Cron', cron), DecimalIntegerValueModelElement('JobNumber'), FixedDataModelElement('Details', b']: Job `cron.daily` started.') ] service_children_random_time = [ FixedDataModelElement('Space', b'Random: '), DecimalIntegerValueModelElement('Random') ] service_children_sensors = [ SequenceModelElement('CPUTemp', [ FixedDataModelElement('FixedTemp', b'CPU Temp: '), DecimalIntegerValueModelElement('Temp'), FixedDataModelElement('Degrees', b'\xc2\xb0C') ]), FixedDataModelElement('Space1', b', '), SequenceModelElement('CPUWorkload', [ FixedDataModelElement('Fixed Workload', b'CPU Workload: '), DecimalIntegerValueModelElement('Workload'), FixedDataModelElement('Percent', b'%') ]), FixedDataModelElement('Space2', b', '), DateTimeModelElement('DTM', date_format_string) ] service_children_user_ip_address = [ FixedDataModelElement('User', b'User '), DelimitedDataModelElement('Username', b' '), FixedDataModelElement('Action', b' changed IP address to '), IpAddressDataModelElement('IP') ] service_children_cron_job_announcement = [ DateTimeModelElement('DTM', date_format_string), FixedDataModelElement('Space', b' '), DelimitedDataModelElement('UName', b' '), FixedDataModelElement('Cron', cron), DecimalIntegerValueModelElement('JobNumber'), FixedDataModelElement('Run', b']: Will run job `'), FixedWordlistDataModelElement( 'CronType', [b'cron.daily', b'cron.hourly', b'cron.monthly', b'cron.weekly']), FixedDataModelElement('Start Time', b'\' in 5 min.') ] service_children_cron_job_execution = [ DateTimeModelElement('DTM', date_format_string), FixedDataModelElement('Space1', b' '), DelimitedDataModelElement('UName', b' '), FixedDataModelElement('Cron', cron), DecimalIntegerValueModelElement('JobNumber'), FixedDataModelElement('Job', b']: Job `'), FixedWordlistDataModelElement( 'CronType', [b'cron.daily', b'cron.hourly', b'cron.monthly', b'cron.weekly']), FixedDataModelElement('Started', b'\' started') ] service_children_parsing_model_element = [ DateTimeModelElement('DateTimeModelElement', b'Current DateTime: %d.%m.%Y %H:%M:%S'), DecimalFloatValueModelElement('DecimalFloatValueModelElement', value_sign_type='optional'), DecimalIntegerValueModelElement('DecimalIntegerValueModelElement', value_sign_type='optional', value_pad_type='blank'), SequenceModelElement('', [ DelimitedDataModelElement('DelimitedDataModelElement', b';'), FixedDataModelElement('FixedDataModelElement', b';') ]) ] # ElementValueBranchModelElement fixed_data_me1 = FixedDataModelElement("fixed1", b'match ') fixed_data_me2 = FixedDataModelElement("fixed2", b'fixed String') fixed_wordlist_data_model_element = FixedWordlistDataModelElement( "wordlist", [b'data: ', b'string: ']) decimal_integer_value_model_element = DecimalIntegerValueModelElement( "decimal") service_children_parsing_model_element.append( ElementValueBranchModelElement( 'ElementValueBranchModelElement', FirstMatchModelElement("first", [ SequenceModelElement( "seq1", [fixed_data_me1, fixed_wordlist_data_model_element]), SequenceModelElement("seq2", [ fixed_data_me1, fixed_wordlist_data_model_element, fixed_data_me2 ]) ]), "wordlist", { 0: decimal_integer_value_model_element, 1: fixed_data_me2 })) service_children_parsing_model_element.append( HexStringModelElement('HexStringModelElement')) service_children_parsing_model_element.append( SequenceModelElement('', [ FixedDataModelElement('FixedDataModelElement', b'Gateway IP-Address: '), IpAddressDataModelElement('IpAddressDataModelElement') ])) service_children_parsing_model_element.append( MultiLocaleDateTimeModelElement('MultiLocaleDateTimeModelElement', [(b'%b %d %Y', "de_AT.utf8", None)])) service_children_parsing_model_element.append( RepeatedElementDataModelElement( 'RepeatedElementDataModelElement', SequenceModelElement('SequenceModelElement', [ FixedDataModelElement('FixedDataModelElement', b'drawn number: '), DecimalIntegerValueModelElement( 'DecimalIntegerValueModelElement') ]), 1)) service_children_parsing_model_element.append( VariableByteDataModelElement('VariableByteDataModelElement', b'-@#')) service_children_parsing_model_element.append( SequenceModelElement('', [ WhiteSpaceLimitedDataModelElement( 'WhiteSpaceLimitedDataModelElement'), FixedDataModelElement('', b' ') ])) # The Base64StringModelElement must be just before the AnyByteDataModelElement to avoid unexpected Matches. service_children_parsing_model_element.append( Base64StringModelElement('Base64StringModelElement')) # The OptionalMatchModelElement must be paired with a FirstMatchModelElement because it accepts all data and thus no data gets # to the AnyByteDataModelElement. The AnyByteDataModelElement must be last, because all bytes are accepted. service_children_parsing_model_element.append( OptionalMatchModelElement( 'OptionalMatchModelElement', FirstMatchModelElement('FirstMatchModelElement', [ FixedDataModelElement('FixedDataModelElement', b'The-searched-element-was-found!'), AnyByteDataModelElement('AnyByteDataModelElement') ]))) parsing_model = FirstMatchModelElement('model', [ SequenceModelElement('CronAnnouncement', service_children_cron_job_announcement), SequenceModelElement('CronExecution', service_children_cron_job_execution), SequenceModelElement('DailyCron', service_children_cron_job), SequenceModelElement('DiskReport', service_children_disk_report), SequenceModelElement('LoginDetails', service_children_login_details), DecimalIntegerValueModelElement('Random'), SequenceModelElement('RandomTime', service_children_random_time), SequenceModelElement('Sensors', service_children_sensors), SequenceModelElement('IPAddresses', service_children_user_ip_address), FirstMatchModelElement('ParsingME', service_children_parsing_model_element) ]) # Some generic imports. from aminer.analysis import AtomFilters # Create all global handler lists here and append the real handlers # later on. # Use this filter to distribute all atoms to the analysis handlers. atom_filter = AtomFilters.SubhandlerFilter(None) from aminer.analysis.TimestampCorrectionFilters import SimpleMonotonicTimestampAdjust simple_monotonic_timestamp_adjust = SimpleMonotonicTimestampAdjust( [atom_filter]) analysis_context.register_component( simple_monotonic_timestamp_adjust, component_name="SimpleMonotonicTimestampAdjust") from aminer.events.StreamPrinterEventHandler import StreamPrinterEventHandler stream_printer_event_handler = StreamPrinterEventHandler(analysis_context) from aminer.events.SyslogWriterEventHandler import SyslogWriterEventHandler syslog_event_handler = SyslogWriterEventHandler(analysis_context) from aminer.events.DefaultMailNotificationEventHandler import DefaultMailNotificationEventHandler if DefaultMailNotificationEventHandler.CONFIG_KEY_MAIL_TARGET_ADDRESS in analysis_context.aminer_config.config_properties: mail_notification_handler = DefaultMailNotificationEventHandler( analysis_context) analysis_context.register_component(mail_notification_handler, component_name="MailHandler") anomaly_event_handlers = [ stream_printer_event_handler, syslog_event_handler, mail_notification_handler ] # Now define the AtomizerFactory using the model. A simple line based one is usually sufficient. from aminer.input.SimpleByteStreamLineAtomizerFactory import SimpleByteStreamLineAtomizerFactory analysis_context.atomizer_factory = SimpleByteStreamLineAtomizerFactory( parsing_model, [simple_monotonic_timestamp_adjust], anomaly_event_handlers) # Just report all unparsed atoms to the event handlers. from aminer.analysis.UnparsedAtomHandlers import SimpleUnparsedAtomHandler simple_unparsed_atom_handler = SimpleUnparsedAtomHandler( anomaly_event_handlers) atom_filter.add_handler(simple_unparsed_atom_handler, stop_when_handled_flag=True) analysis_context.register_component(simple_unparsed_atom_handler, component_name="UnparsedHandler") from aminer.analysis.TimestampsUnsortedDetector import TimestampsUnsortedDetector timestamps_unsorted_detector = TimestampsUnsortedDetector( analysis_context.aminer_config, anomaly_event_handlers) atom_filter.add_handler(timestamps_unsorted_detector) analysis_context.register_component( timestamps_unsorted_detector, component_name="TimestampsUnsortedDetector") from aminer.analysis import Rules from aminer.analysis.AllowlistViolationDetector import AllowlistViolationDetector allowlist_rules = [ Rules.OrMatchRule([ Rules.AndMatchRule([ Rules.PathExistsMatchRule( '/model/LoginDetails/PastTime/Time/Minutes'), Rules.NegationMatchRule( Rules.ValueMatchRule('/model/LoginDetails/Username', b'root')) ]), Rules.AndMatchRule([ Rules.NegationMatchRule( Rules.PathExistsMatchRule( '/model/LoginDetails/PastTime/Time/Minutes')), Rules.PathExistsMatchRule('/model/LoginDetails') ]), Rules.NegationMatchRule( Rules.PathExistsMatchRule('/model/LoginDetails')) ]) ] # This rule list should trigger, when the line does not look like: User root (logged in, logged out) # or User 'username' (logged in, logged out) x minutes ago. allowlist_violation_detector = AllowlistViolationDetector( analysis_context.aminer_config, allowlist_rules, anomaly_event_handlers) analysis_context.register_component(allowlist_violation_detector, component_name="Allowlist") atom_filter.add_handler(allowlist_violation_detector) from aminer.analysis.NewMatchPathDetector import NewMatchPathDetector new_match_path_detector = NewMatchPathDetector( analysis_context.aminer_config, anomaly_event_handlers, auto_include_flag=True) analysis_context.register_component(new_match_path_detector, component_name="NewMatchPath") atom_filter.add_handler(new_match_path_detector) def tuple_transformation_function(match_value_list): """Only allow output of the EnhancedNewMatchPathValueComboDetector after every 10000th element.""" extra_data = enhanced_new_match_path_value_combo_detector.known_values_dict.get( tuple(match_value_list)) if extra_data is not None: mod = 10000 if (extra_data[2] + 1) % mod == 0: enhanced_new_match_path_value_combo_detector.auto_include_flag = False else: enhanced_new_match_path_value_combo_detector.auto_include_flag = True return match_value_list from aminer.analysis.EnhancedNewMatchPathValueComboDetector import EnhancedNewMatchPathValueComboDetector enhanced_new_match_path_value_combo_detector = EnhancedNewMatchPathValueComboDetector( analysis_context.aminer_config, ['/model/DailyCron/UName', '/model/DailyCron/JobNumber'], anomaly_event_handlers, auto_include_flag=True, tuple_transformation_function=tuple_transformation_function) analysis_context.register_component( enhanced_new_match_path_value_combo_detector, component_name="EnhancedNewValueCombo") atom_filter.add_handler(enhanced_new_match_path_value_combo_detector) from aminer.analysis.HistogramAnalysis import HistogramAnalysis, LinearNumericBinDefinition, ModuloTimeBinDefinition, \ PathDependentHistogramAnalysis modulo_time_bin_definition = ModuloTimeBinDefinition( 86400, 3600, 0, 1, 24, True) linear_numeric_bin_definition = LinearNumericBinDefinition(50, 5, 20, True) histogram_analysis = HistogramAnalysis( analysis_context.aminer_config, [('/model/RandomTime/Random', modulo_time_bin_definition), ('/model/Random', linear_numeric_bin_definition)], 10, anomaly_event_handlers) analysis_context.register_component(histogram_analysis, component_name="HistogramAnalysis") atom_filter.add_handler(histogram_analysis) path_dependent_histogram_analysis = PathDependentHistogramAnalysis( analysis_context.aminer_config, '/model/RandomTime', modulo_time_bin_definition, 10, anomaly_event_handlers) analysis_context.register_component( path_dependent_histogram_analysis, component_name="PathDependentHistogramAnalysis") atom_filter.add_handler(path_dependent_histogram_analysis) from aminer.analysis.MatchValueAverageChangeDetector import MatchValueAverageChangeDetector match_value_average_change_detector = MatchValueAverageChangeDetector( analysis_context.aminer_config, anomaly_event_handlers, None, ['/model/Random'], 100, 10) analysis_context.register_component( match_value_average_change_detector, component_name="MatchValueAverageChange") atom_filter.add_handler(match_value_average_change_detector) import sys from aminer.analysis.MatchValueStreamWriter import MatchValueStreamWriter match_value_stream_writer = MatchValueStreamWriter(sys.stdout, [ '/model/Sensors/CPUTemp', '/model/Sensors/CPUWorkload', '/model/Sensors/DTM' ], b';', b'') analysis_context.register_component( match_value_stream_writer, component_name="MatchValueStreamWriter") atom_filter.add_handler(match_value_stream_writer) from aminer.analysis.NewMatchPathValueComboDetector import NewMatchPathValueComboDetector new_match_path_value_combo_detector = NewMatchPathValueComboDetector( analysis_context.aminer_config, ['/model/IPAddresses/Username', '/model/IPAddresses/IP'], anomaly_event_handlers, auto_include_flag=True) analysis_context.register_component( new_match_path_value_combo_detector, component_name="NewMatchPathValueCombo") atom_filter.add_handler(new_match_path_value_combo_detector) from aminer.analysis.NewMatchPathValueDetector import NewMatchPathValueDetector new_match_path_value_detector = NewMatchPathValueDetector( analysis_context.aminer_config, ['/model/DailyCron/JobNumber', '/model/IPAddresses/Username'], anomaly_event_handlers, auto_include_flag=True) analysis_context.register_component(new_match_path_value_detector, component_name="NewMatchPathValue") atom_filter.add_handler(new_match_path_value_detector) from aminer.analysis.MissingMatchPathValueDetector import MissingMatchPathValueDetector missing_match_path_value_detector = MissingMatchPathValueDetector( analysis_context.aminer_config, ['/model/DiskReport/Space'], anomaly_event_handlers, auto_include_flag=True, default_interval=2, realert_interval=5) analysis_context.register_component(missing_match_path_value_detector, component_name="MissingMatch") atom_filter.add_handler(missing_match_path_value_detector) from aminer.analysis.TimeCorrelationDetector import TimeCorrelationDetector time_correlation_detector = TimeCorrelationDetector( analysis_context.aminer_config, anomaly_event_handlers, 2, min_rule_attributes=1, max_rule_attributes=5, record_count_before_event=70000, output_log_line=True) analysis_context.register_component( time_correlation_detector, component_name="TimeCorrelationDetector") atom_filter.add_handler(time_correlation_detector) from aminer.analysis.TimeCorrelationViolationDetector import TimeCorrelationViolationDetector, CorrelationRule, EventClassSelector cron_job_announcement = CorrelationRule( 'CronJobAnnouncement', 5, 6, max_artefacts_a_for_single_b=1, artefact_match_parameters=[('/model/CronAnnouncement/JobNumber', '/model/CronExecution/JobNumber')]) a_class_selector = EventClassSelector('Announcement', [cron_job_announcement], None) b_class_selector = EventClassSelector('Execution', None, [cron_job_announcement]) rules = [ Rules.PathExistsMatchRule('/model/CronAnnouncement/Run', a_class_selector), Rules.PathExistsMatchRule('/model/CronExecution/Job', b_class_selector) ] time_correlation_violation_detector = TimeCorrelationViolationDetector( analysis_context.aminer_config, rules, anomaly_event_handlers) analysis_context.register_component( time_correlation_violation_detector, component_name="TimeCorrelationViolationDetector") atom_filter.add_handler(time_correlation_violation_detector)
def test2_match_element_not_found(self): """In this test case all possible ways of not getting a MatchElement successfully are tested.""" # no modelMatch element_value_branch_model_element = ElementValueBranchModelElement( "id", self.first_match_me, "wordlist", { 0: self.decimal_integer_value_model_element, 1: self.fixed_data_me2 }) match_context = MatchContext(self.path_to_match) self.assertEqual( element_value_branch_model_element.get_match_element( 'match', match_context), None) self.assertEqual(match_context.match_data, self.path_to_match) # no matchChildren element_value_branch_model_element = ElementValueBranchModelElement( "id", self.fixed_data_me3, "wordlist", { 0: self.decimal_integer_value_model_element, 1: self.fixed_data_me2 }) match_context = MatchContext(self.path_to_match) self.assertEqual( element_value_branch_model_element.get_match_element( 'match', match_context), None) self.assertEqual(match_context.match_data, self.path_to_match) # no branchModel element_value_branch_model_element = ElementValueBranchModelElement( "id", self.first_match_me, "wordlist", {0: self.decimal_integer_value_model_element}) match_context = MatchContext(self.string_path_to_match) self.assertEqual( element_value_branch_model_element.get_match_element( 'match', match_context), None) self.assertEqual(match_context.match_data, self.string_path_to_match) # no branchMatch element_value_branch_model_element = ElementValueBranchModelElement( "id", self.first_match_me, "wordlist", { 0: self.decimal_integer_value_model_element, 1: self.fixed_data_me2 }) match_context = MatchContext(self.string_path_to_match) self.assertEqual( element_value_branch_model_element.get_match_element( 'match', match_context), None) self.assertEqual(match_context.match_data, self.string_path_to_match)
def get_model(): """Return a model to parse a audispd message logged via syslog after any standard logging preamble, e.g. from syslog.""" class ExecArgumentDataModelElement(): """This is a helper class for parsing the (encoded) exec argument strings found within audit logs.""" def __init__(self, element_id): self.element_id = element_id @staticmethod def get_child_elements(): """Get the children of this element (none).""" return None def get_match_element(self, path, match_context): """ Find the maximum number of bytes belonging to an exec argument. @return a match when at least two bytes were found including the delimiters. """ data = match_context.match_data match_len = 0 match_value = b'' if data[0] == ord(b'"'): match_len = data.find(b'"', 1) if match_len == -1: return None match_value = data[1:match_len] match_len += 1 elif data.startswith(b'(null)'): match_len = 6 match_value = None else: # Must be upper case hex encoded: next_value = -1 for d_byte in data: if 0x30 <= d_byte <= 0x39: d_byte -= 0x30 elif 0x41 <= d_byte <= 0x46: d_byte -= 0x37 else: break if next_value == -1: next_value = (d_byte << 4) else: match_value += bytearray(((next_value | d_byte), )) next_value = -1 match_len += 1 if next_value != -1: return None match_data = data[:match_len] match_context.update(match_data) return MatchElement("%s/%s" % (path, self.element_id), match_data, match_value, None) pam_status_word_list = FixedWordlistDataModelElement( 'status', [b'failed', b'success']) pid = b' pid=' uid = b' uid=' auid = b' auid=' ses = b' ses=' exe = b' exe="' hostname = b'" hostname=' addr = b' addr=' terminal = b' terminal=' success = b' res=success\'' res = b' res=' exe1 = b'" exe="' type_branches = { 'ADD_USER': SequenceModelElement('adduser', [ FixedDataModelElement('s0', pid), DecimalIntegerValueModelElement('pid'), FixedDataModelElement('s1', uid), DecimalIntegerValueModelElement('uid'), FixedDataModelElement('s2', auid), DecimalIntegerValueModelElement('auid'), FixedDataModelElement('s3', ses), DecimalIntegerValueModelElement('ses'), FixedDataModelElement('s4', b' msg=\'op=adding user id='), DecimalIntegerValueModelElement('newuserid'), FixedDataModelElement('s5', exe), DelimitedDataModelElement('exec', b'"'), FixedDataModelElement('s6', hostname), DelimitedDataModelElement('clientname', b' '), FixedDataModelElement('s7', addr), DelimitedDataModelElement('clientip', b' '), FixedDataModelElement('s8', terminal), WhiteSpaceLimitedDataModelElement('terminal'), FixedDataModelElement('s9', success) ]), 'BPRM_FCAPS': SequenceModelElement('bprmfcaps', [ FixedDataModelElement('s0', b' fver=0 fp='), HexStringModelElement('fp'), FixedDataModelElement('s1', b' fi='), HexStringModelElement('fi'), FixedDataModelElement('s2', b' fe='), HexStringModelElement('fe'), FixedDataModelElement('s3', b' old_pp='), HexStringModelElement('pp-old'), FixedDataModelElement('s4', b' old_pi='), HexStringModelElement('pi-old'), FixedDataModelElement('s5', b' old_pe='), HexStringModelElement('pe-old'), FixedDataModelElement('s6', b' new_pp='), HexStringModelElement('pp-new'), FixedDataModelElement('s7', b' new_pi='), HexStringModelElement('pi-new'), FixedDataModelElement('s8', b' new_pe='), HexStringModelElement('pe-new') ]), 'CONFIG_CHANGE': SequenceModelElement('conf-change', [ FixedDataModelElement('s0', auid), DecimalIntegerValueModelElement('auid'), FixedDataModelElement('s1', ses), DecimalIntegerValueModelElement('ses'), FixedDataModelElement('s2', b' op="add rule" key=(null) list='), DecimalIntegerValueModelElement('list'), FixedDataModelElement('s3', res), DecimalIntegerValueModelElement('result') ]), 'CRED_ACQ': SequenceModelElement('credacq', [ FixedDataModelElement('s0', pid), DecimalIntegerValueModelElement('pid'), FixedDataModelElement('s1', uid), DecimalIntegerValueModelElement('uid'), FixedDataModelElement('s2', auid), DecimalIntegerValueModelElement('auid'), FixedDataModelElement('s3', ses), DecimalIntegerValueModelElement('ses'), FixedDataModelElement('s4', b' msg=\'op=PAM:setcred acct="'), DelimitedDataModelElement('username', b'"'), FixedDataModelElement('s5', exe1), DelimitedDataModelElement('exec', b'"'), FixedDataModelElement('s6', hostname), DelimitedDataModelElement('clientname', b' '), FixedDataModelElement('s7', addr), DelimitedDataModelElement('clientip', b' '), FixedDataModelElement('s8', terminal), WhiteSpaceLimitedDataModelElement('terminal'), FixedDataModelElement('s9', success) ]), 'CRED_DISP': SequenceModelElement('creddisp', [ FixedDataModelElement('s0', pid), DecimalIntegerValueModelElement('pid'), FixedDataModelElement('s1', uid), DecimalIntegerValueModelElement('uid'), FixedDataModelElement('s2', auid), DecimalIntegerValueModelElement('auid'), FixedDataModelElement('s3', ses), DecimalIntegerValueModelElement('ses'), FixedDataModelElement('s4', b' msg=\'op=PAM:setcred acct="'), DelimitedDataModelElement('username', b'"'), FixedDataModelElement('s5', exe1), DelimitedDataModelElement('exec', b'"'), FixedDataModelElement('s6', hostname), DelimitedDataModelElement('clientname', b' '), FixedDataModelElement('s7', addr), DelimitedDataModelElement('clientip', b' '), FixedDataModelElement('s8', terminal), WhiteSpaceLimitedDataModelElement('terminal'), FixedDataModelElement('s9', success) ]), 'CRED_REFR': SequenceModelElement('creddisp', [ FixedDataModelElement('s0', pid), DecimalIntegerValueModelElement('pid'), FixedDataModelElement('s1', uid), DecimalIntegerValueModelElement('uid'), FixedDataModelElement('s2', auid), DecimalIntegerValueModelElement('auid'), FixedDataModelElement('s3', ses), DecimalIntegerValueModelElement('ses'), FixedDataModelElement( 's4', b' msg=\'op=PAM:setcred acct="root" exe="/usr/sbin/sshd" hostname=' ), IpAddressDataModelElement('clientname'), FixedDataModelElement('s5', addr), IpAddressDataModelElement('clientip'), FixedDataModelElement('s6', b' terminal=ssh res=success\'') ]), 'CWD': SequenceModelElement('cwd', [ FixedDataModelElement('s0', b' cwd='), ExecArgumentDataModelElement('cwd') ]), 'EOE': OptionalMatchModelElement('eoe', FixedDataModelElement('s0', b'')) } # We need a type branch here also, but there is no additional # data in EOE records after Ubuntu Trusty any more. exec_arg_model = SequenceModelElement('execarg', [ FixedDataModelElement('s0', b' a'), DecimalIntegerValueModelElement('argn'), FixedDataModelElement('s1', b'='), ExecArgumentDataModelElement('argval') ]) type_branches['EXECVE'] = SequenceModelElement('execve', [ FixedDataModelElement('s0', b' argc='), DecimalIntegerValueModelElement('argc'), RepeatedElementDataModelElement('arg', exec_arg_model) ]) type_branches['FD_PAIR'] = SequenceModelElement('fdpair', [ FixedDataModelElement('s0', b' fd0='), DecimalIntegerValueModelElement('fd0'), FixedDataModelElement('s1', b' fd1='), DecimalIntegerValueModelElement('fd1') ]) # This message differs on Ubuntu 32/64 bit variants. type_branches['LOGIN'] = SequenceModelElement('login', [ FixedDataModelElement('s0', pid), DecimalIntegerValueModelElement('pid'), FixedDataModelElement('s1', uid), DecimalIntegerValueModelElement('uid'), FixedWordlistDataModelElement('s2', [b' old auid=', b' old-auid=']), DecimalIntegerValueModelElement('auid-old'), FixedWordlistDataModelElement('s3', [b' new auid=', auid]), DecimalIntegerValueModelElement('auid-new'), FixedWordlistDataModelElement('s4', [b' old ses=', b' old-ses=']), DecimalIntegerValueModelElement('ses-old'), FixedWordlistDataModelElement('s5', [b' new ses=', ses]), DecimalIntegerValueModelElement('ses-new'), FixedDataModelElement('s6', res), DecimalIntegerValueModelElement('result') ]) inode_info_model_element = SequenceModelElement( 'inodeinfo', [ FixedDataModelElement('s0', b' inode='), DecimalIntegerValueModelElement('inode'), FixedDataModelElement('s1', b' dev='), # A special major/minor device element could be better here. VariableByteDataModelElement('dev', b'0123456789abcdef:'), FixedDataModelElement('s2', b' mode='), # is octal DecimalIntegerValueModelElement('mode'), FixedDataModelElement('s3', b' ouid='), DecimalIntegerValueModelElement('ouid'), FixedDataModelElement('s4', b' ogid='), DecimalIntegerValueModelElement('ogid'), FixedDataModelElement('s5', b' rdev='), # A special major/minor device element could be better here (see above). VariableByteDataModelElement('rdev', b'0123456789abcdef:'), FixedDataModelElement('s6', b' nametype=') ]) type_branches['NETFILTER_CFG'] = SequenceModelElement( 'conf-change', [ FixedDataModelElement('s0', b' table='), FixedWordlistDataModelElement('table', [b'filter', b'mangle', b'nat']), FixedDataModelElement('s1', b' family='), DecimalIntegerValueModelElement('family'), FixedDataModelElement('s2', b' entries='), DecimalIntegerValueModelElement('entries') ]) type_branches['OBJ_PID'] = SequenceModelElement('objpid', [ FixedDataModelElement('s0', b' opid='), DecimalIntegerValueModelElement('opid'), FixedDataModelElement('s1', b' oauid='), DecimalIntegerValueModelElement( 'oauid', value_sign_type=DecimalIntegerValueModelElement.SIGN_TYPE_OPTIONAL ), FixedDataModelElement('s2', b' ouid='), DecimalIntegerValueModelElement('ouid'), FixedDataModelElement('s3', b' oses='), DecimalIntegerValueModelElement( 'oses', value_sign_type=DecimalIntegerValueModelElement.SIGN_TYPE_OPTIONAL ), FixedDataModelElement('s4', b' ocomm='), ExecArgumentDataModelElement('ocomm') ]) type_branches['PATH'] = SequenceModelElement('path', [ FixedDataModelElement('s0', b' item='), DecimalIntegerValueModelElement('item'), FixedDataModelElement('s1', b' name='), ExecArgumentDataModelElement('name'), FirstMatchModelElement('fsinfo', [ inode_info_model_element, FixedDataModelElement('noinfo', b' nametype=') ]), FixedWordlistDataModelElement( 'nametype', [b'CREATE', b'DELETE', b'NORMAL', b'PARENT', b'UNKNOWN']) ]) type_branches['PROCTITLE'] = SequenceModelElement('proctitle', [ FixedDataModelElement('s1', b' proctitle='), ExecArgumentDataModelElement('proctitle') ]) type_branches['SERVICE_START'] = SequenceModelElement( 'service', [ FixedDataModelElement('s0', pid), DecimalIntegerValueModelElement('pid'), FixedDataModelElement('s1', uid), DecimalIntegerValueModelElement('uid'), FixedDataModelElement('s2', auid), DecimalIntegerValueModelElement('auid'), FixedDataModelElement('s3', ses), DecimalIntegerValueModelElement('ses'), FixedDataModelElement('s4', b' msg=\'unit='), DelimitedDataModelElement('unit', b' '), FixedDataModelElement('s5', b' comm="systemd" exe="'), DelimitedDataModelElement('exec', b'"'), FixedDataModelElement('s6', hostname), DelimitedDataModelElement('clientname', b' '), FixedDataModelElement('s7', addr), DelimitedDataModelElement('clientip', b' '), FixedDataModelElement('s8', terminal), WhiteSpaceLimitedDataModelElement('terminal'), FixedDataModelElement('s9', res), pam_status_word_list, FixedDataModelElement('s10', b'\'') ]) type_branches['SERVICE_STOP'] = type_branches['SERVICE_START'] type_branches['SOCKADDR'] = SequenceModelElement('sockaddr', [ FixedDataModelElement('s0', b' saddr='), HexStringModelElement('sockaddr', upper_case=True) ]) type_branches['SYSCALL'] = SequenceModelElement('syscall', [ FixedDataModelElement('s0', b' arch='), HexStringModelElement('arch'), FixedDataModelElement('s1', b' syscall='), DecimalIntegerValueModelElement('syscall'), OptionalMatchModelElement( 'personality', SequenceModelElement('pseq', [ FixedDataModelElement('s0', b' per='), DecimalIntegerValueModelElement('personality') ])), OptionalMatchModelElement( 'result', SequenceModelElement('rseq', [ FixedDataModelElement('s2', b' success='), FixedWordlistDataModelElement('succes', [b'no', b'yes']), FixedDataModelElement('s3', b' exit='), DecimalIntegerValueModelElement( 'exit', value_sign_type=DecimalIntegerValueModelElement. SIGN_TYPE_OPTIONAL) ])), FixedDataModelElement('s4', b' a0='), HexStringModelElement('arg0'), FixedDataModelElement('s5', b' a1='), HexStringModelElement('arg1'), FixedDataModelElement('s6', b' a2='), HexStringModelElement('arg2'), FixedDataModelElement('s7', b' a3='), HexStringModelElement('arg3'), FixedDataModelElement('s8', b' items='), DecimalIntegerValueModelElement('items'), FixedDataModelElement('s9', b' ppid='), DecimalIntegerValueModelElement('ppid'), FixedDataModelElement('s10', pid), DecimalIntegerValueModelElement('pid'), FixedDataModelElement('s11', auid), DecimalIntegerValueModelElement('auid'), FixedDataModelElement('s12', uid), DecimalIntegerValueModelElement('uid'), FixedDataModelElement('s13', b' gid='), DecimalIntegerValueModelElement('gid'), FixedDataModelElement('s14', b' euid='), DecimalIntegerValueModelElement('euid'), FixedDataModelElement('s15', b' suid='), DecimalIntegerValueModelElement('suid'), FixedDataModelElement('s16', b' fsuid='), DecimalIntegerValueModelElement('fsuid'), FixedDataModelElement('s17', b' egid='), DecimalIntegerValueModelElement('egid'), FixedDataModelElement('s18', b' sgid='), DecimalIntegerValueModelElement('sgid'), FixedDataModelElement('s19', b' fsgid='), DecimalIntegerValueModelElement('fsgid'), FixedDataModelElement('s20', b' tty='), DelimitedDataModelElement('tty', b' '), FixedDataModelElement('s21', ses), DecimalIntegerValueModelElement('sesid'), FixedDataModelElement('s22', b' comm='), ExecArgumentDataModelElement('command'), FixedDataModelElement('s23', exe), DelimitedDataModelElement('executable', b'"'), FixedDataModelElement('s24', b'" key='), AnyByteDataModelElement('key') ]) # The UNKNOWN type is used then audispd does not know the type of the event, usually because the kernel is more recent than # audispd, thus emiting yet unknown event types. # * type=1327: procitle: see https://www.redhat.com/archives/linux-audit/2014-February/msg00047.html type_branches['UNKNOWN[1327]'] = SequenceModelElement( 'unknown-proctitle', [ FixedDataModelElement('s0', b' proctitle='), ExecArgumentDataModelElement('proctitle') ]) type_branches['USER_ACCT'] = SequenceModelElement('useracct', [ FixedDataModelElement('s0', pid), DecimalIntegerValueModelElement('pid'), FixedDataModelElement('s1', uid), DecimalIntegerValueModelElement('uid'), FixedDataModelElement('s2', auid), DecimalIntegerValueModelElement('auid'), FixedDataModelElement('s3', ses), DecimalIntegerValueModelElement('ses'), FixedDataModelElement('s4', b' msg=\'op=PAM:accounting acct="'), DelimitedDataModelElement('username', b'"'), FixedDataModelElement('s5', exe1), DelimitedDataModelElement('exec', b'"'), FixedDataModelElement('s6', hostname), DelimitedDataModelElement('clientname', b' '), FixedDataModelElement('s7', addr), DelimitedDataModelElement('clientip', b' '), FixedDataModelElement('s8', terminal), WhiteSpaceLimitedDataModelElement('terminal'), FixedDataModelElement('s9', success) ]) type_branches['USER_AUTH'] = SequenceModelElement('userauth', [ FixedDataModelElement('s0', pid), DecimalIntegerValueModelElement('pid'), FixedDataModelElement('s1', uid), DecimalIntegerValueModelElement('uid'), FixedDataModelElement('s2', auid), DecimalIntegerValueModelElement('auid'), FixedDataModelElement('s3', ses), DecimalIntegerValueModelElement('ses'), FixedDataModelElement('s4', b' msg=\'op=PAM:authentication acct="'), DelimitedDataModelElement('username', b'"'), FixedDataModelElement('s5', exe1), DelimitedDataModelElement('exec', b'"'), FixedDataModelElement('s6', hostname), DelimitedDataModelElement('clientname', b' '), FixedDataModelElement('s7', addr), DelimitedDataModelElement('clientip', b' '), FixedDataModelElement('s8', terminal), WhiteSpaceLimitedDataModelElement('terminal'), FixedDataModelElement('s9', success) ]) type_branches['USER_START'] = SequenceModelElement('userstart', [ FixedDataModelElement('s0', pid), DecimalIntegerValueModelElement('pid'), FixedDataModelElement('s1', uid), DecimalIntegerValueModelElement('uid'), FixedDataModelElement('s2', auid), DecimalIntegerValueModelElement('auid'), FixedDataModelElement('s3', ses), DecimalIntegerValueModelElement('ses'), FixedDataModelElement('s4', b' msg=\'op=PAM:session_open acct="'), DelimitedDataModelElement('username', b'"'), FixedDataModelElement('s5', exe1), DelimitedDataModelElement('exec', b'"'), FixedDataModelElement('s6', hostname), DelimitedDataModelElement('clientname', b' '), FixedDataModelElement('s7', addr), DelimitedDataModelElement('clientip', b' '), FixedDataModelElement('s8', terminal), WhiteSpaceLimitedDataModelElement('terminal'), FixedDataModelElement('s9', success) ]) type_branches['USER_END'] = SequenceModelElement('userend', [ FixedDataModelElement('s0', pid), DecimalIntegerValueModelElement('pid'), FixedDataModelElement('s1', uid), DecimalIntegerValueModelElement('uid'), FixedDataModelElement('s2', auid), DecimalIntegerValueModelElement('auid'), FixedDataModelElement('s3', ses), DecimalIntegerValueModelElement('ses'), FixedDataModelElement('s4', b' msg=\'op=PAM:session_close acct="'), DelimitedDataModelElement('username', b'"'), FixedDataModelElement('s5', exe1), DelimitedDataModelElement('exec', b'"'), FixedDataModelElement('s6', hostname), DelimitedDataModelElement('clientname', b' '), FixedDataModelElement('s7', addr), DelimitedDataModelElement('clientip', b' '), FixedDataModelElement('s8', terminal), WhiteSpaceLimitedDataModelElement('terminal'), FixedDataModelElement('s9', success) ]) type_branches['USER_ERR'] = SequenceModelElement('usererr', [ FixedDataModelElement('s0', pid), DecimalIntegerValueModelElement('pid'), FixedDataModelElement('s1', uid), DecimalIntegerValueModelElement('uid'), FixedDataModelElement('s2', auid), DecimalIntegerValueModelElement('auid'), FixedDataModelElement('s3', ses), DecimalIntegerValueModelElement('ses'), FixedDataModelElement('s4', b' msg=\'op=PAM:bad_ident acct="?" exe="'), DelimitedDataModelElement('exec', b'"'), FixedDataModelElement('s5', hostname), DelimitedDataModelElement('clientname', b' '), FixedDataModelElement('s6', addr), DelimitedDataModelElement('clientip', b' '), FixedDataModelElement('s7', terminal), WhiteSpaceLimitedDataModelElement('terminal'), FixedDataModelElement('s8', b' res=failed\'') ]) type_branches['USER_LOGIN'] = SequenceModelElement('userlogin', [ FixedDataModelElement('s0', pid), DecimalIntegerValueModelElement('pid'), FixedDataModelElement('s1', uid), DecimalIntegerValueModelElement('uid'), FixedDataModelElement('s2', auid), DecimalIntegerValueModelElement('auid'), FixedDataModelElement('s3', ses), DecimalIntegerValueModelElement('ses'), FixedDataModelElement('s4', b' msg=\'op=login '), FirstMatchModelElement('msgtype', [ FixedDataModelElement('loginok', b'id=0'), SequenceModelElement('loginfail', [ FixedDataModelElement('s0', b'acct='), ExecArgumentDataModelElement('account') ]) ]), FixedDataModelElement('s5', exe), DelimitedDataModelElement('exec', b'"'), FixedDataModelElement('s6', hostname), DelimitedDataModelElement('clientname', b' '), FixedDataModelElement('s7', addr), DelimitedDataModelElement('clientip', b' '), FixedDataModelElement('s8', terminal), WhiteSpaceLimitedDataModelElement('terminal'), FixedDataModelElement('s9', res), pam_status_word_list, FixedDataModelElement('s10', b'\'') ]) model = SequenceModelElement('audispd', [ FixedDataModelElement('sname', b'audispd: '), FirstMatchModelElement('msg', [ ElementValueBranchModelElement( 'record', SequenceModelElement('preamble', [ FixedDataModelElement('s0', b'type='), WhiteSpaceLimitedDataModelElement('type'), FixedDataModelElement('s1', b' msg=audit('), DecimalIntegerValueModelElement('time'), FixedDataModelElement('s0', b'.'), DecimalIntegerValueModelElement('ms'), FixedDataModelElement('s1', b':'), DecimalIntegerValueModelElement('seq'), FixedDataModelElement('s2', b'):') ]), 'type', type_branches, default_branch=None), FixedDataModelElement('queue-full', b'queue is full - dropping event') ]) ]) return model