def test1get_id(self): """Test if get_id works properly.""" ip_addr_dme = IpAddressDataModelElement(self.id_) self.assertEqual(ip_addr_dme.get_id(), self.id_)
def test2get_child_elements(self): """Test if get_child_elements returns None.""" ip_addr_dme = IpAddressDataModelElement(self.id_) self.assertEqual(ip_addr_dme.get_child_elements(), None)
def build_analysis_pipeline(analysis_context): """ Define the function to create pipeline for parsing the log data. It has also to define an AtomizerFactory to instruct aminer how to process incoming data streams to create log atoms from them. """ date_format_string = b'%Y-%m-%d %H:%M:%S' cron = b' cron[' # Build the parsing model: service_children_disk_report = [ FixedDataModelElement( 'Space', b' Current Disk Data is: Filesystem Type Size Used Avail Use%' ), DelimitedDataModelElement('Data', b'%'), AnyByteDataModelElement('Rest') ] service_children_login_details = [ FixedDataModelElement('User/LoginDetails', b'User '), DelimitedDataModelElement('Username', b' '), FixedWordlistDataModelElement('Status', [b' logged in', b' logged out']), OptionalMatchModelElement( 'PastTime', SequenceModelElement('Time', [ FixedDataModelElement('Blank', b' '), DecimalIntegerValueModelElement('Minutes'), FixedDataModelElement('Ago', b' minutes ago.') ])) ] service_children_cron_job = [ DateTimeModelElement('DTM', date_format_string), FixedDataModelElement('UNameSpace1', b' '), DelimitedDataModelElement('UName', b' '), FixedDataModelElement('UNameSpace2', b' '), DelimitedDataModelElement('User', b' '), FixedDataModelElement('Cron', cron), DecimalIntegerValueModelElement('JobNumber'), FixedDataModelElement('Details', b']: Job `cron.daily` started.') ] service_children_random_time = [ FixedDataModelElement('Space', b'Random: '), DecimalIntegerValueModelElement('Random') ] service_children_sensors = [ SequenceModelElement('CPUTemp', [ FixedDataModelElement('FixedTemp', b'CPU Temp: '), DecimalIntegerValueModelElement('Temp'), FixedDataModelElement('Degrees', b'\xc2\xb0C') ]), FixedDataModelElement('Space1', b', '), SequenceModelElement('CPUWorkload', [ FixedDataModelElement('FixedWorkload', b'CPU Workload: '), DecimalIntegerValueModelElement('Workload'), FixedDataModelElement('Percent', b'%') ]), FixedDataModelElement('Space2', b', '), DateTimeModelElement('DTM', date_format_string) ] service_children_user_ip_address = [ FixedDataModelElement('User/UserIPAddress', b'User '), DelimitedDataModelElement('Username', b' '), FixedDataModelElement('Action', b' changed IP address to '), IpAddressDataModelElement('IP') ] service_children_cron_job_announcement = [ DateTimeModelElement('DTM', date_format_string), FixedDataModelElement('Space', b' '), DelimitedDataModelElement('UName', b' '), FixedDataModelElement('Cron', cron), DecimalIntegerValueModelElement('JobNumber'), FixedDataModelElement('Run', b']: Will run job `'), FixedWordlistDataModelElement( 'CronType', [b'cron.daily', b'cron.hourly', b'cron.monthly', b'cron.weekly']), FixedDataModelElement('StartTime', b'\' in 5 min.') ] service_children_cron_job_execution = [ DateTimeModelElement('DTM', date_format_string), FixedDataModelElement('Space1', b' '), DelimitedDataModelElement('UName', b' '), FixedDataModelElement('Cron', cron), DecimalIntegerValueModelElement('JobNumber'), FixedDataModelElement('Job', b']: Job `'), FixedWordlistDataModelElement( 'CronType', [b'cron.daily', b'cron.hourly', b'cron.monthly', b'cron.weekly']), FixedDataModelElement('Started', b'\' started') ] service_children_audit = [ SequenceModelElement('path', [ FixedDataModelElement('type', b'type=PATH '), FixedDataModelElement('msg_audit', b'msg=audit('), DelimitedDataModelElement('msg', b':'), FixedDataModelElement('placeholder', b':'), DecimalIntegerValueModelElement('id'), FixedDataModelElement('item_string', b'): item='), DecimalIntegerValueModelElement('item'), FixedDataModelElement('name_string', b' name="'), DelimitedDataModelElement('name', b'"'), FixedDataModelElement('inode_string', b'" inode='), DecimalIntegerValueModelElement('inode'), FixedDataModelElement('dev_string', b' dev='), DelimitedDataModelElement('dev', b' '), FixedDataModelElement('mode_string', b' mode='), DecimalIntegerValueModelElement( 'mode', value_pad_type=DecimalIntegerValueModelElement.PAD_TYPE_ZERO), FixedDataModelElement('ouid_string', b' ouid='), DecimalIntegerValueModelElement('ouid'), FixedDataModelElement('ogid_string', b' ogid='), DecimalIntegerValueModelElement('ogid'), FixedDataModelElement('rdev_string', b' rdev='), DelimitedDataModelElement('rdev', b' '), FixedDataModelElement('nametype_string', b' nametype='), FixedWordlistDataModelElement('nametype', [b'NORMAL', b'ERROR']) ]), SequenceModelElement('syscall', [ FixedDataModelElement('type', b'type=SYSCALL '), FixedDataModelElement('msg_audit', b'msg=audit('), DelimitedDataModelElement('msg', b':'), FixedDataModelElement('placeholder', b':'), DecimalIntegerValueModelElement('id'), FixedDataModelElement('arch_string', b'): arch='), DelimitedDataModelElement('arch', b' '), FixedDataModelElement('syscall_string', b' syscall='), DecimalIntegerValueModelElement('syscall'), FixedDataModelElement('success_string', b' success='), FixedWordlistDataModelElement('success', [b'yes', b'no']), FixedDataModelElement('exit_string', b' exit='), DecimalIntegerValueModelElement('exit'), AnyByteDataModelElement('remainding_data') ]) ] service_children_parsing_model_element = [ DateTimeModelElement('DateTimeModelElement', b'Current DateTime: %d.%m.%Y %H:%M:%S'), DecimalFloatValueModelElement('DecimalFloatValueModelElement', value_sign_type='optional'), DecimalIntegerValueModelElement('DecimalIntegerValueModelElement', value_sign_type='optional', value_pad_type='blank'), SequenceModelElement('se', [ DelimitedDataModelElement('DelimitedDataModelElement', b';'), FixedDataModelElement('FixedDataModelElement', b';') ]) ] # ElementValueBranchModelElement fixed_data_me1 = FixedDataModelElement("fixed1", b'match ') fixed_data_me2 = FixedDataModelElement("fixed2", b'fixed String') fixed_wordlist_data_model_element = FixedWordlistDataModelElement( "wordlist", [b'data: ', b'string: ']) decimal_integer_value_model_element = DecimalIntegerValueModelElement( "decimal") service_children_parsing_model_element.append( ElementValueBranchModelElement( 'ElementValueBranchModelElement', FirstMatchModelElement("first", [ SequenceModelElement( "seq1", [fixed_data_me1, fixed_wordlist_data_model_element]), SequenceModelElement("seq2", [ fixed_data_me1, fixed_wordlist_data_model_element, fixed_data_me2 ]) ]), "wordlist", { 0: decimal_integer_value_model_element, 1: fixed_data_me2 })) service_children_parsing_model_element.append( HexStringModelElement('HexStringModelElement')) service_children_parsing_model_element.append( SequenceModelElement('se2', [ FixedDataModelElement('FixedDataModelElement', b'Gateway IP-Address: '), IpAddressDataModelElement('IpAddressDataModelElement') ])) import locale loc = locale.getlocale() if loc == (None, None): loc = ('en_US', 'utf8') service_children_parsing_model_element.append( MultiLocaleDateTimeModelElement('MultiLocaleDateTimeModelElement', [(b'%b %d %Y', None, '%s.%s' % loc)])) service_children_parsing_model_element.append( RepeatedElementDataModelElement( 'RepeatedElementDataModelElement', SequenceModelElement('SequenceModelElement', [ FixedDataModelElement('FixedDataModelElement', b'[drawn number]: '), DecimalIntegerValueModelElement( 'DecimalIntegerValueModelElement') ]), 1)) service_children_parsing_model_element.append( VariableByteDataModelElement('VariableByteDataModelElement', b'-@#')) service_children_parsing_model_element.append( SequenceModelElement('se', [ WhiteSpaceLimitedDataModelElement( 'WhiteSpaceLimitedDataModelElement'), FixedDataModelElement('fixed', b' ') ])) # The Base64StringModelElement must be just before the AnyByteDataModelElement to avoid unexpected Matches. service_children_parsing_model_element.append( Base64StringModelElement('Base64StringModelElement')) # The OptionalMatchModelElement must be paired with a FirstMatchModelElement because it accepts all data and thus no data gets # to the AnyByteDataModelElement. The AnyByteDataModelElement must be last, because all bytes are accepted. service_children_parsing_model_element.append( OptionalMatchModelElement( '/', FirstMatchModelElement('FirstMatchModelElement//optional', [ FixedDataModelElement('FixedDataModelElement', b'The-searched-element-was-found!'), SequenceModelElement('se', [ FixedDataModelElement('FixedDME', b'Any:'), AnyByteDataModelElement('AnyByteDataModelElement') ]) ]))) alphabet = b'ghijkl' service_children_ecd = [] for _, char in enumerate(alphabet): char = bytes([char]) service_children_ecd.append(FixedDataModelElement(char.decode(), char)) parsing_model = FirstMatchModelElement('model', [ SequenceModelElement('CronAnnouncement', service_children_cron_job_announcement), SequenceModelElement('CronExecution', service_children_cron_job_execution), SequenceModelElement('DailyCron', service_children_cron_job), SequenceModelElement('DiskReport', service_children_disk_report), SequenceModelElement('LoginDetails', service_children_login_details), DecimalIntegerValueModelElement('Random'), SequenceModelElement('RandomTime', service_children_random_time), SequenceModelElement('Sensors', service_children_sensors), SequenceModelElement('IPAddresses', service_children_user_ip_address), FirstMatchModelElement('type', service_children_audit), FirstMatchModelElement('ECD', service_children_ecd), FirstMatchModelElement('ParsingME', service_children_parsing_model_element) ]) # Some generic imports. from aminer.analysis import AtomFilters # Create all global handler lists here and append the real handlers later on. # Use this filter to distribute all atoms to the analysis handlers. atom_filter = AtomFilters.SubhandlerFilter(None) from aminer.analysis.TimestampCorrectionFilters import SimpleMonotonicTimestampAdjust simple_monotonic_timestamp_adjust = SimpleMonotonicTimestampAdjust( [atom_filter]) analysis_context.register_component( simple_monotonic_timestamp_adjust, component_name="SimpleMonotonicTimestampAdjust") from aminer.events.StreamPrinterEventHandler import StreamPrinterEventHandler from aminer.events.JsonConverterHandler import JsonConverterHandler stream_printer_event_handler = StreamPrinterEventHandler(analysis_context) json_converter_handler = JsonConverterHandler( [stream_printer_event_handler], analysis_context) anomaly_event_handlers = [json_converter_handler] # Now define the AtomizerFactory using the model. A simple line based one is usually sufficient. from aminer.input.SimpleByteStreamLineAtomizerFactory import SimpleByteStreamLineAtomizerFactory analysis_context.atomizer_factory = SimpleByteStreamLineAtomizerFactory( parsing_model, [simple_monotonic_timestamp_adjust], anomaly_event_handlers) # Just report all unparsed atoms to the event handlers. from aminer.input.SimpleUnparsedAtomHandler import SimpleUnparsedAtomHandler from aminer.input.VerboseUnparsedAtomHandler import VerboseUnparsedAtomHandler simple_unparsed_atom_handler = SimpleUnparsedAtomHandler( anomaly_event_handlers) atom_filter.add_handler(simple_unparsed_atom_handler, stop_when_handled_flag=False) analysis_context.register_component(simple_unparsed_atom_handler, component_name="SimpleUnparsedHandler") verbose_unparsed_atom_handler = VerboseUnparsedAtomHandler( anomaly_event_handlers, parsing_model) atom_filter.add_handler(verbose_unparsed_atom_handler, stop_when_handled_flag=True) analysis_context.register_component( verbose_unparsed_atom_handler, component_name="VerboseUnparsedHandler") from aminer.analysis.TimestampsUnsortedDetector import TimestampsUnsortedDetector timestamps_unsorted_detector = TimestampsUnsortedDetector( analysis_context.aminer_config, anomaly_event_handlers) atom_filter.add_handler(timestamps_unsorted_detector) analysis_context.register_component( timestamps_unsorted_detector, component_name="TimestampsUnsortedDetector") from aminer.analysis import Rules from aminer.analysis.AllowlistViolationDetector import AllowlistViolationDetector allowlist_rules = [ Rules.OrMatchRule([ Rules.AndMatchRule([ Rules.PathExistsMatchRule( '/model/LoginDetails/PastTime/Time/Minutes'), Rules.NegationMatchRule( Rules.ValueMatchRule('/model/LoginDetails/Username', b'root')), Rules.DebugMatchRule(debug_match_result=True) ]), Rules.AndMatchRule([ Rules.NegationMatchRule( Rules.PathExistsMatchRule( '/model/LoginDetails/PastTime/Time/Minutes')), Rules.PathExistsMatchRule('/model/LoginDetails'), Rules.DebugMatchRule(debug_match_result=True) ]), Rules.NegationMatchRule( Rules.PathExistsMatchRule('/model/LoginDetails')) ]) ] # This rule list should trigger, when the line does not look like: User root (logged in, logged out) # or User 'username' (logged in, logged out) x minutes ago. allowlist_violation_detector = AllowlistViolationDetector( analysis_context.aminer_config, allowlist_rules, anomaly_event_handlers, output_log_line=True) analysis_context.register_component(allowlist_violation_detector, component_name="Allowlist") atom_filter.add_handler(allowlist_violation_detector) from aminer.analysis.ParserCount import ParserCount parser_count = ParserCount(analysis_context.aminer_config, None, anomaly_event_handlers, 10) analysis_context.register_component(parser_count, component_name="ParserCount") atom_filter.add_handler(parser_count) from aminer.analysis.EventTypeDetector import EventTypeDetector etd = EventTypeDetector(analysis_context.aminer_config, anomaly_event_handlers) analysis_context.register_component(etd, component_name="EventTypeDetector") atom_filter.add_handler(etd) from aminer.analysis.VariableTypeDetector import VariableTypeDetector vtd = VariableTypeDetector(analysis_context.aminer_config, anomaly_event_handlers, etd, silence_output_except_indicator=False, output_log_line=False) analysis_context.register_component(vtd, component_name="VariableTypeDetector") atom_filter.add_handler(vtd) from aminer.analysis.VariableCorrelationDetector import VariableCorrelationDetector vtd = VariableCorrelationDetector(analysis_context.aminer_config, anomaly_event_handlers, etd, disc_div_thres=0.5) analysis_context.register_component( vtd, component_name="VariableCorrelationDetector") atom_filter.add_handler(vtd) from aminer.analysis.EventCorrelationDetector import EventCorrelationDetector ecd = EventCorrelationDetector(analysis_context.aminer_config, anomaly_event_handlers, check_rules_flag=True, hypothesis_max_delta_time=1.0) analysis_context.register_component( ecd, component_name="EventCorrelationDetector") atom_filter.add_handler(ecd) from aminer.analysis.EventFrequencyDetector import EventFrequencyDetector efd = EventFrequencyDetector(analysis_context.aminer_config, anomaly_event_handlers, window_size=0.1) analysis_context.register_component( efd, component_name="EventFrequencyDetector") atom_filter.add_handler(efd) from aminer.analysis.EventSequenceDetector import EventSequenceDetector esd = EventSequenceDetector(analysis_context.aminer_config, anomaly_event_handlers, ['/model/ParsingME'], ignore_list=[ '/model/ECD/g', '/model/ECD/h', '/model/ECD/i', '/model/ECD/j', '/model/ECD/k', '/model/ECD/l', '/model/Random', '/model/RandomTime', '/model/DailyCron' ]) analysis_context.register_component(esd, component_name="EventSequenceDetector") atom_filter.add_handler(esd) from aminer.analysis.MatchFilter import MatchFilter match_filter = MatchFilter(analysis_context.aminer_config, ['/model/Random'], anomaly_event_handlers, target_value_list=[1, 10, 100], output_log_line=True) analysis_context.register_component(match_filter, component_name="MatchFilter") atom_filter.add_handler(match_filter) from aminer.analysis.NewMatchPathDetector import NewMatchPathDetector new_match_path_detector = NewMatchPathDetector( analysis_context.aminer_config, anomaly_event_handlers, auto_include_flag=True, output_log_line=True) analysis_context.register_component(new_match_path_detector, component_name="NewMatchPath") atom_filter.add_handler(new_match_path_detector) def tuple_transformation_function(match_value_list): """Only allow output of the EnhancedNewMatchPathValueComboDetector after every 10th element.""" extra_data = enhanced_new_match_path_value_combo_detector.known_values_dict.get( tuple(match_value_list)) if extra_data is not None: mod = 10 if (extra_data[2] + 1) % mod == 0: enhanced_new_match_path_value_combo_detector.auto_include_flag = False else: enhanced_new_match_path_value_combo_detector.auto_include_flag = True return match_value_list from aminer.analysis.EnhancedNewMatchPathValueComboDetector import EnhancedNewMatchPathValueComboDetector enhanced_new_match_path_value_combo_detector = EnhancedNewMatchPathValueComboDetector( analysis_context.aminer_config, ['/model/DailyCron/UName', '/model/DailyCron/JobNumber'], anomaly_event_handlers, auto_include_flag=True, tuple_transformation_function=tuple_transformation_function, output_log_line=True) analysis_context.register_component( enhanced_new_match_path_value_combo_detector, component_name="EnhancedNewValueCombo") atom_filter.add_handler(enhanced_new_match_path_value_combo_detector) import re ip_match_action = Rules.EventGenerationMatchAction( "Analysis.Rules.IPv4InRFC1918MatchRule", "Private IP address occurred!", anomaly_event_handlers) vdmt = Rules.ValueDependentModuloTimeMatchRule( None, 3, ["/model/ECD/j", "/model/ECD/k", "/model/ECD/l"], {b"e": [0, 2.95]}, [0, 3]) mt = Rules.ModuloTimeMatchRule(None, 3, 0, 3, None) time_allowlist_rules = [ Rules.AndMatchRule([ Rules.ParallelMatchRule([ Rules.ValueDependentDelegatedMatchRule( [ '/model/ECD/g', '/model/ECD/h', '/model/ECD/i', '/model/ECD/j', '/model/ECD/k', '/model/ECD/l' ], { (b"a", ): mt, (b"b", ): mt, (b"c", ): mt, (b"d", ): vdmt, (b"e", ): vdmt, (b"f", ): vdmt, None: mt }, mt), Rules.IPv4InRFC1918MatchRule( "/model/ParsingME/se2/IpAddressDataModelElement", ip_match_action), Rules.DebugHistoryMatchRule(debug_match_result=True) ]), # IP addresses 8.8.8.8, 8.8.4.4 and 10.0.0.0 - 10.255.255.255 are not allowed Rules.NegationMatchRule( Rules.ValueListMatchRule( "/model/ParsingME/se2/IpAddressDataModelElement", [134744072, 134743044])), Rules.NegationMatchRule( Rules.ValueRangeMatchRule( "/model/ParsingME/se2/IpAddressDataModelElement", 167772160, 184549375)), Rules.NegationMatchRule( Rules.StringRegexMatchRule("/model/type/syscall/success", re.compile(b"^no$"))) ]) ] time_allowlist_violation_detector = AllowlistViolationDetector( analysis_context.aminer_config, time_allowlist_rules, anomaly_event_handlers, output_log_line=True) analysis_context.register_component(time_allowlist_violation_detector, component_name="TimeAllowlist") atom_filter.add_handler(time_allowlist_violation_detector) from aminer.analysis.HistogramAnalysis import HistogramAnalysis, LinearNumericBinDefinition, ModuloTimeBinDefinition, \ PathDependentHistogramAnalysis modulo_time_bin_definition = ModuloTimeBinDefinition( 86400, 3600, 0, 1, 24, True) linear_numeric_bin_definition = LinearNumericBinDefinition(50, 5, 20, True) histogram_analysis = HistogramAnalysis( analysis_context.aminer_config, [('/model/RandomTime/Random', modulo_time_bin_definition), ('/model/Random', linear_numeric_bin_definition)], 10, anomaly_event_handlers, output_log_line=True) analysis_context.register_component(histogram_analysis, component_name="HistogramAnalysis") atom_filter.add_handler(histogram_analysis) path_dependent_histogram_analysis = PathDependentHistogramAnalysis( analysis_context.aminer_config, '/model/RandomTime', modulo_time_bin_definition, 10, anomaly_event_handlers, output_log_line=True) analysis_context.register_component( path_dependent_histogram_analysis, component_name="PathDependentHistogramAnalysis") atom_filter.add_handler(path_dependent_histogram_analysis) from aminer.analysis.MatchValueAverageChangeDetector import MatchValueAverageChangeDetector match_value_average_change_detector = MatchValueAverageChangeDetector( analysis_context.aminer_config, anomaly_event_handlers, None, ['/model/Random'], 100, 10, output_log_line=True) analysis_context.register_component( match_value_average_change_detector, component_name="MatchValueAverageChange") atom_filter.add_handler(match_value_average_change_detector) import sys from aminer.analysis.MatchValueStreamWriter import MatchValueStreamWriter match_value_stream_writer = MatchValueStreamWriter(sys.stdout, [ '/model/Sensors/CPUTemp', '/model/Sensors/CPUWorkload', '/model/Sensors/DTM' ], b';', b'') analysis_context.register_component( match_value_stream_writer, component_name="MatchValueStreamWriter") atom_filter.add_handler(match_value_stream_writer) from aminer.analysis.NewMatchPathValueComboDetector import NewMatchPathValueComboDetector new_match_path_value_combo_detector = NewMatchPathValueComboDetector( analysis_context.aminer_config, ['/model/IPAddresses/Username', '/model/IPAddresses/IP'], anomaly_event_handlers, output_log_line=True) analysis_context.register_component( new_match_path_value_combo_detector, component_name="NewMatchPathValueCombo") atom_filter.add_handler(new_match_path_value_combo_detector) from aminer.analysis.NewMatchIdValueComboDetector import NewMatchIdValueComboDetector new_match_id_value_combo_detector = NewMatchIdValueComboDetector( analysis_context.aminer_config, ['/model/type/path/name', '/model/type/syscall/syscall'], anomaly_event_handlers, id_path_list=['/model/type/path/id', '/model/type/syscall/id'], min_allowed_time_diff=5, auto_include_flag=True, allow_missing_values_flag=True, output_log_line=True) analysis_context.register_component( new_match_id_value_combo_detector, component_name="NewMatchIdValueComboDetector") atom_filter.add_handler(new_match_id_value_combo_detector) from aminer.analysis.NewMatchPathValueDetector import NewMatchPathValueDetector new_match_path_value_detector = NewMatchPathValueDetector( analysis_context.aminer_config, ['/model/DailyCron/JobNumber', '/model/IPAddresses/Username'], anomaly_event_handlers, auto_include_flag=True, output_log_line=True) analysis_context.register_component(new_match_path_value_detector, component_name="NewMatchPathValue") atom_filter.add_handler(new_match_path_value_detector) from aminer.analysis.MissingMatchPathValueDetector import MissingMatchPathValueDetector missing_match_path_value_detector = MissingMatchPathValueDetector( analysis_context.aminer_config, ['/model/DiskReport/Space'], anomaly_event_handlers, auto_include_flag=True, default_interval=2, realert_interval=5, output_log_line=True) analysis_context.register_component(missing_match_path_value_detector, component_name="MissingMatch") atom_filter.add_handler(missing_match_path_value_detector) from aminer.analysis.TimeCorrelationDetector import TimeCorrelationDetector time_correlation_detector = TimeCorrelationDetector( analysis_context.aminer_config, anomaly_event_handlers, 2, min_rule_attributes=1, max_rule_attributes=5, record_count_before_event=10000, output_log_line=True) analysis_context.register_component( time_correlation_detector, component_name="TimeCorrelationDetector") atom_filter.add_handler(time_correlation_detector) from aminer.analysis.TimeCorrelationViolationDetector import TimeCorrelationViolationDetector, CorrelationRule, EventClassSelector cron_job_announcement = CorrelationRule( 'CronJobAnnouncement', 5, 6, max_artefacts_a_for_single_b=1, artefact_match_parameters=[('/model/CronAnnouncement/JobNumber', '/model/CronExecution/JobNumber')]) a_class_selector = EventClassSelector('Announcement', [cron_job_announcement], None) b_class_selector = EventClassSelector('Execution', None, [cron_job_announcement]) rules = [ Rules.PathExistsMatchRule('/model/CronAnnouncement/Run', a_class_selector), Rules.PathExistsMatchRule('/model/CronExecution/Job', b_class_selector) ] time_correlation_violation_detector = TimeCorrelationViolationDetector( analysis_context.aminer_config, rules, anomaly_event_handlers, output_log_line=True) analysis_context.register_component( time_correlation_violation_detector, component_name="TimeCorrelationViolationDetector") atom_filter.add_handler(time_correlation_violation_detector)
def build_analysis_pipeline(analysis_context): """ Define the function to create pipeline for parsing the log data. It has also to define an AtomizerFactory to instruct aminer how to process incoming data streams to create log atoms from them. """ date_format_string = b'%Y-%m-%d %H:%M:%S' cron = b' cron[' # Build the parsing model: service_children_disk_report = [ FixedDataModelElement( 'Space', b' Current Disk Data is: Filesystem Type Size Used Avail Use%' ), DelimitedDataModelElement('Data', b'%'), AnyByteDataModelElement('Rest') ] service_children_login_details = [ FixedDataModelElement('User', b'User '), DelimitedDataModelElement('Username', b' '), FixedWordlistDataModelElement('Status', [b' logged in', b' logged out']), OptionalMatchModelElement( 'PastTime', SequenceModelElement('Time', [ FixedDataModelElement('Blank', b' '), DecimalIntegerValueModelElement('Minutes'), FixedDataModelElement('Ago', b' minutes ago.') ])) ] service_children_cron_job = [ DateTimeModelElement('DTM', date_format_string), FixedDataModelElement('UNameSpace1', b' '), DelimitedDataModelElement('UName', b' '), FixedDataModelElement('UNameSpace2', b' '), DelimitedDataModelElement('User', b' '), FixedDataModelElement('Cron', cron), DecimalIntegerValueModelElement('JobNumber'), FixedDataModelElement('Details', b']: Job `cron.daily` started.') ] service_children_random_time = [ FixedDataModelElement('Space', b'Random: '), DecimalIntegerValueModelElement('Random') ] service_children_sensors = [ SequenceModelElement('CPUTemp', [ FixedDataModelElement('FixedTemp', b'CPU Temp: '), DecimalIntegerValueModelElement('Temp'), FixedDataModelElement('Degrees', b'\xc2\xb0C') ]), FixedDataModelElement('Space1', b', '), SequenceModelElement('CPUWorkload', [ FixedDataModelElement('Fixed Workload', b'CPU Workload: '), DecimalIntegerValueModelElement('Workload'), FixedDataModelElement('Percent', b'%') ]), FixedDataModelElement('Space2', b', '), DateTimeModelElement('DTM', date_format_string) ] service_children_user_ip_address = [ FixedDataModelElement('User', b'User '), DelimitedDataModelElement('Username', b' '), FixedDataModelElement('Action', b' changed IP address to '), IpAddressDataModelElement('IP') ] service_children_cron_job_announcement = [ DateTimeModelElement('DTM', date_format_string), FixedDataModelElement('Space', b' '), DelimitedDataModelElement('UName', b' '), FixedDataModelElement('Cron', cron), DecimalIntegerValueModelElement('JobNumber'), FixedDataModelElement('Run', b']: Will run job `'), FixedWordlistDataModelElement( 'CronType', [b'cron.daily', b'cron.hourly', b'cron.monthly', b'cron.weekly']), FixedDataModelElement('Start Time', b'\' in 5 min.') ] service_children_cron_job_execution = [ DateTimeModelElement('DTM', date_format_string), FixedDataModelElement('Space1', b' '), DelimitedDataModelElement('UName', b' '), FixedDataModelElement('Cron', cron), DecimalIntegerValueModelElement('JobNumber'), FixedDataModelElement('Job', b']: Job `'), FixedWordlistDataModelElement( 'CronType', [b'cron.daily', b'cron.hourly', b'cron.monthly', b'cron.weekly']), FixedDataModelElement('Started', b'\' started') ] service_children_parsing_model_element = [ DateTimeModelElement('DateTimeModelElement', b'Current DateTime: %d.%m.%Y %H:%M:%S'), DecimalFloatValueModelElement('DecimalFloatValueModelElement', value_sign_type='optional'), DecimalIntegerValueModelElement('DecimalIntegerValueModelElement', value_sign_type='optional', value_pad_type='blank'), SequenceModelElement('', [ DelimitedDataModelElement('DelimitedDataModelElement', b';'), FixedDataModelElement('FixedDataModelElement', b';') ]) ] # ElementValueBranchModelElement fixed_data_me1 = FixedDataModelElement("fixed1", b'match ') fixed_data_me2 = FixedDataModelElement("fixed2", b'fixed String') fixed_wordlist_data_model_element = FixedWordlistDataModelElement( "wordlist", [b'data: ', b'string: ']) decimal_integer_value_model_element = DecimalIntegerValueModelElement( "decimal") service_children_parsing_model_element.append( ElementValueBranchModelElement( 'ElementValueBranchModelElement', FirstMatchModelElement("first", [ SequenceModelElement( "seq1", [fixed_data_me1, fixed_wordlist_data_model_element]), SequenceModelElement("seq2", [ fixed_data_me1, fixed_wordlist_data_model_element, fixed_data_me2 ]) ]), "wordlist", { 0: decimal_integer_value_model_element, 1: fixed_data_me2 })) service_children_parsing_model_element.append( HexStringModelElement('HexStringModelElement')) service_children_parsing_model_element.append( SequenceModelElement('', [ FixedDataModelElement('FixedDataModelElement', b'Gateway IP-Address: '), IpAddressDataModelElement('IpAddressDataModelElement') ])) service_children_parsing_model_element.append( MultiLocaleDateTimeModelElement('MultiLocaleDateTimeModelElement', [(b'%b %d %Y', "de_AT.utf8", None)])) service_children_parsing_model_element.append( RepeatedElementDataModelElement( 'RepeatedElementDataModelElement', SequenceModelElement('SequenceModelElement', [ FixedDataModelElement('FixedDataModelElement', b'drawn number: '), DecimalIntegerValueModelElement( 'DecimalIntegerValueModelElement') ]), 1)) service_children_parsing_model_element.append( VariableByteDataModelElement('VariableByteDataModelElement', b'-@#')) service_children_parsing_model_element.append( SequenceModelElement('', [ WhiteSpaceLimitedDataModelElement( 'WhiteSpaceLimitedDataModelElement'), FixedDataModelElement('', b' ') ])) # The Base64StringModelElement must be just before the AnyByteDataModelElement to avoid unexpected Matches. service_children_parsing_model_element.append( Base64StringModelElement('Base64StringModelElement')) # The OptionalMatchModelElement must be paired with a FirstMatchModelElement because it accepts all data and thus no data gets # to the AnyByteDataModelElement. The AnyByteDataModelElement must be last, because all bytes are accepted. service_children_parsing_model_element.append( OptionalMatchModelElement( 'OptionalMatchModelElement', FirstMatchModelElement('FirstMatchModelElement', [ FixedDataModelElement('FixedDataModelElement', b'The-searched-element-was-found!'), AnyByteDataModelElement('AnyByteDataModelElement') ]))) parsing_model = FirstMatchModelElement('model', [ SequenceModelElement('CronAnnouncement', service_children_cron_job_announcement), SequenceModelElement('CronExecution', service_children_cron_job_execution), SequenceModelElement('DailyCron', service_children_cron_job), SequenceModelElement('DiskReport', service_children_disk_report), SequenceModelElement('LoginDetails', service_children_login_details), DecimalIntegerValueModelElement('Random'), SequenceModelElement('RandomTime', service_children_random_time), SequenceModelElement('Sensors', service_children_sensors), SequenceModelElement('IPAddresses', service_children_user_ip_address), FirstMatchModelElement('ParsingME', service_children_parsing_model_element) ]) # Some generic imports. from aminer.analysis import AtomFilters # Create all global handler lists here and append the real handlers # later on. # Use this filter to distribute all atoms to the analysis handlers. atom_filter = AtomFilters.SubhandlerFilter(None) from aminer.analysis.TimestampCorrectionFilters import SimpleMonotonicTimestampAdjust simple_monotonic_timestamp_adjust = SimpleMonotonicTimestampAdjust( [atom_filter]) analysis_context.register_component( simple_monotonic_timestamp_adjust, component_name="SimpleMonotonicTimestampAdjust") from aminer.events.StreamPrinterEventHandler import StreamPrinterEventHandler stream_printer_event_handler = StreamPrinterEventHandler(analysis_context) from aminer.events.SyslogWriterEventHandler import SyslogWriterEventHandler syslog_event_handler = SyslogWriterEventHandler(analysis_context) from aminer.events.DefaultMailNotificationEventHandler import DefaultMailNotificationEventHandler if DefaultMailNotificationEventHandler.CONFIG_KEY_MAIL_TARGET_ADDRESS in analysis_context.aminer_config.config_properties: mail_notification_handler = DefaultMailNotificationEventHandler( analysis_context) analysis_context.register_component(mail_notification_handler, component_name="MailHandler") anomaly_event_handlers = [ stream_printer_event_handler, syslog_event_handler, mail_notification_handler ] # Now define the AtomizerFactory using the model. A simple line based one is usually sufficient. from aminer.input.SimpleByteStreamLineAtomizerFactory import SimpleByteStreamLineAtomizerFactory analysis_context.atomizer_factory = SimpleByteStreamLineAtomizerFactory( parsing_model, [simple_monotonic_timestamp_adjust], anomaly_event_handlers) # Just report all unparsed atoms to the event handlers. from aminer.input.SimpleUnparsedAtomHandler import SimpleUnparsedAtomHandler simple_unparsed_atom_handler = SimpleUnparsedAtomHandler( anomaly_event_handlers) atom_filter.add_handler(simple_unparsed_atom_handler, stop_when_handled_flag=True) analysis_context.register_component(simple_unparsed_atom_handler, component_name="UnparsedHandler") from aminer.analysis.TimestampsUnsortedDetector import TimestampsUnsortedDetector timestamps_unsorted_detector = TimestampsUnsortedDetector( analysis_context.aminer_config, anomaly_event_handlers) atom_filter.add_handler(timestamps_unsorted_detector) analysis_context.register_component( timestamps_unsorted_detector, component_name="TimestampsUnsortedDetector") from aminer.analysis import Rules from aminer.analysis.AllowlistViolationDetector import AllowlistViolationDetector allowlist_rules = [ Rules.OrMatchRule([ Rules.AndMatchRule([ Rules.PathExistsMatchRule( '/model/LoginDetails/PastTime/Time/Minutes'), Rules.NegationMatchRule( Rules.ValueMatchRule('/model/LoginDetails/Username', b'root')) ]), Rules.AndMatchRule([ Rules.NegationMatchRule( Rules.PathExistsMatchRule( '/model/LoginDetails/PastTime/Time/Minutes')), Rules.PathExistsMatchRule('/model/LoginDetails') ]), Rules.NegationMatchRule( Rules.PathExistsMatchRule('/model/LoginDetails')) ]) ] # This rule list should trigger, when the line does not look like: User root (logged in, logged out) # or User 'username' (logged in, logged out) x minutes ago. allowlist_violation_detector = AllowlistViolationDetector( analysis_context.aminer_config, allowlist_rules, anomaly_event_handlers) analysis_context.register_component(allowlist_violation_detector, component_name="Allowlist") atom_filter.add_handler(allowlist_violation_detector) from aminer.analysis.NewMatchPathDetector import NewMatchPathDetector new_match_path_detector = NewMatchPathDetector( analysis_context.aminer_config, anomaly_event_handlers, auto_include_flag=True) analysis_context.register_component(new_match_path_detector, component_name="NewMatchPath") atom_filter.add_handler(new_match_path_detector) def tuple_transformation_function(match_value_list): """Only allow output of the EnhancedNewMatchPathValueComboDetector after every 10000th element.""" extra_data = enhanced_new_match_path_value_combo_detector.known_values_dict.get( tuple(match_value_list)) if extra_data is not None: mod = 10000 if (extra_data[2] + 1) % mod == 0: enhanced_new_match_path_value_combo_detector.auto_include_flag = False else: enhanced_new_match_path_value_combo_detector.auto_include_flag = True return match_value_list from aminer.analysis.EnhancedNewMatchPathValueComboDetector import EnhancedNewMatchPathValueComboDetector enhanced_new_match_path_value_combo_detector = EnhancedNewMatchPathValueComboDetector( analysis_context.aminer_config, ['/model/DailyCron/UName', '/model/DailyCron/JobNumber'], anomaly_event_handlers, auto_include_flag=True, tuple_transformation_function=tuple_transformation_function) analysis_context.register_component( enhanced_new_match_path_value_combo_detector, component_name="EnhancedNewValueCombo") atom_filter.add_handler(enhanced_new_match_path_value_combo_detector) from aminer.analysis.HistogramAnalysis import HistogramAnalysis, LinearNumericBinDefinition, ModuloTimeBinDefinition, \ PathDependentHistogramAnalysis modulo_time_bin_definition = ModuloTimeBinDefinition( 86400, 3600, 0, 1, 24, True) linear_numeric_bin_definition = LinearNumericBinDefinition(50, 5, 20, True) histogram_analysis = HistogramAnalysis( analysis_context.aminer_config, [('/model/RandomTime/Random', modulo_time_bin_definition), ('/model/Random', linear_numeric_bin_definition)], 10, anomaly_event_handlers) analysis_context.register_component(histogram_analysis, component_name="HistogramAnalysis") atom_filter.add_handler(histogram_analysis) path_dependent_histogram_analysis = PathDependentHistogramAnalysis( analysis_context.aminer_config, '/model/RandomTime', modulo_time_bin_definition, 10, anomaly_event_handlers) analysis_context.register_component( path_dependent_histogram_analysis, component_name="PathDependentHistogramAnalysis") atom_filter.add_handler(path_dependent_histogram_analysis) from aminer.analysis.MatchValueAverageChangeDetector import MatchValueAverageChangeDetector match_value_average_change_detector = MatchValueAverageChangeDetector( analysis_context.aminer_config, anomaly_event_handlers, None, ['/model/Random'], 100, 10) analysis_context.register_component( match_value_average_change_detector, component_name="MatchValueAverageChange") atom_filter.add_handler(match_value_average_change_detector) import sys from aminer.analysis.MatchValueStreamWriter import MatchValueStreamWriter match_value_stream_writer = MatchValueStreamWriter(sys.stdout, [ '/model/Sensors/CPUTemp', '/model/Sensors/CPUWorkload', '/model/Sensors/DTM' ], b';', b'') analysis_context.register_component( match_value_stream_writer, component_name="MatchValueStreamWriter") atom_filter.add_handler(match_value_stream_writer) from aminer.analysis.NewMatchPathValueComboDetector import NewMatchPathValueComboDetector new_match_path_value_combo_detector = NewMatchPathValueComboDetector( analysis_context.aminer_config, ['/model/IPAddresses/Username', '/model/IPAddresses/IP'], anomaly_event_handlers, auto_include_flag=True) analysis_context.register_component( new_match_path_value_combo_detector, component_name="NewMatchPathValueCombo") atom_filter.add_handler(new_match_path_value_combo_detector) from aminer.analysis.NewMatchPathValueDetector import NewMatchPathValueDetector new_match_path_value_detector = NewMatchPathValueDetector( analysis_context.aminer_config, ['/model/DailyCron/JobNumber', '/model/IPAddresses/Username'], anomaly_event_handlers, auto_include_flag=True) analysis_context.register_component(new_match_path_value_detector, component_name="NewMatchPathValue") atom_filter.add_handler(new_match_path_value_detector) from aminer.analysis.MissingMatchPathValueDetector import MissingMatchPathValueDetector missing_match_path_value_detector = MissingMatchPathValueDetector( analysis_context.aminer_config, ['/model/DiskReport/Space'], anomaly_event_handlers, auto_include_flag=True, default_interval=2, realert_interval=5) analysis_context.register_component(missing_match_path_value_detector, component_name="MissingMatch") atom_filter.add_handler(missing_match_path_value_detector) from aminer.analysis.TimeCorrelationDetector import TimeCorrelationDetector time_correlation_detector = TimeCorrelationDetector( analysis_context.aminer_config, anomaly_event_handlers, 2, min_rule_attributes=1, max_rule_attributes=5, record_count_before_event=70000, output_log_line=True) analysis_context.register_component( time_correlation_detector, component_name="TimeCorrelationDetector") atom_filter.add_handler(time_correlation_detector) from aminer.analysis.TimeCorrelationViolationDetector import TimeCorrelationViolationDetector, CorrelationRule, EventClassSelector cron_job_announcement = CorrelationRule( 'CronJobAnnouncement', 5, 6, max_artefacts_a_for_single_b=1, artefact_match_parameters=[('/model/CronAnnouncement/JobNumber', '/model/CronExecution/JobNumber')]) a_class_selector = EventClassSelector('Announcement', [cron_job_announcement], None) b_class_selector = EventClassSelector('Execution', None, [cron_job_announcement]) rules = [ Rules.PathExistsMatchRule('/model/CronAnnouncement/Run', a_class_selector), Rules.PathExistsMatchRule('/model/CronExecution/Job', b_class_selector) ] time_correlation_violation_detector = TimeCorrelationViolationDetector( analysis_context.aminer_config, rules, anomaly_event_handlers) analysis_context.register_component( time_correlation_violation_detector, component_name="TimeCorrelationViolationDetector") atom_filter.add_handler(time_correlation_violation_detector)
def get_model(): """Return a model to parse Exim logs from the AIT-LDS.""" alphabet = b'!"#$%&\'()*+,-./0123456789:;<>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\\^_`abcdefghijklmnopqrstuvwxyz{|}~=[]' model = SequenceModelElement('model', [ DateTimeModelElement('time', b'%Y-%m-%d %H:%M:%S'), FixedDataModelElement('sp', b' '), FirstMatchModelElement('fm', [ SequenceModelElement('start', [ FixedDataModelElement('start', b'Start queue run: pid='), DecimalIntegerValueModelElement('pid'), ]), SequenceModelElement('end', [ FixedDataModelElement('end', b'End queue run: pid='), DecimalIntegerValueModelElement('pid'), ]), SequenceModelElement('no_host_found', [ FixedDataModelElement('no_host_found_str', b'no host name found for IP address '), IpAddressDataModelElement('ip'), ]), SequenceModelElement('vrfy_failed', [ FixedDataModelElement('vrfy_failed_str', b'VRFY failed for '), DelimitedDataModelElement('mail', b' '), FixedDataModelElement('h_str', b' H='), DelimitedDataModelElement('h', b' '), FixedDataModelElement('sp1', b' ['), IpAddressDataModelElement('ip'), FixedDataModelElement('sp2', b']') ]), SequenceModelElement('mail', [ DelimitedDataModelElement('id', b' '), FirstMatchModelElement('dir', [ SequenceModelElement('dir_in', [ FixedDataModelElement('in', b' <= '), FirstMatchModelElement('fm', [ SequenceModelElement('seq1', [ FixedDataModelElement('brack', b'<> '), FirstMatchModelElement('fm', [ SequenceModelElement( 'r', [ FixedDataModelElement( 'r_str', b'R='), DelimitedDataModelElement( 'r', b' '), FixedDataModelElement( 'u_str', b' U='), DelimitedDataModelElement( 'u', b' '), ]), SequenceModelElement( 'h', [ FixedDataModelElement( 'h_str', b'H='), DelimitedDataModelElement( 'h', b' '), FixedDataModelElement( 'sp1', b' ['), IpAddressDataModelElement('ip'), FixedDataModelElement('sp1', b']'), ]) ]), FixedDataModelElement('sp2', b' P='), DelimitedDataModelElement('p', b' '), FixedDataModelElement('sp2', b' S='), DecimalIntegerValueModelElement('s'), ]), SequenceModelElement('seq2', [ DelimitedDataModelElement('mail', b' '), FixedDataModelElement('user_str', b' U='), DelimitedDataModelElement('user', b' '), FixedDataModelElement('p_str', b' P='), DelimitedDataModelElement('p', b' '), FixedDataModelElement('s_str', b' S='), DecimalIntegerValueModelElement('s'), OptionalMatchModelElement( 'id', SequenceModelElement( 'id', [ FixedDataModelElement( 'id_str', b' id='), AnyByteDataModelElement('id') ])) ]) ]) ]), SequenceModelElement('dir_out', [ FixedDataModelElement('in', b' => '), DelimitedDataModelElement('name', b' '), FixedDataModelElement('sp1', b' '), OptionalMatchModelElement( 'mail_opt', SequenceModelElement('mail', [ FixedDataModelElement('brack1', b'('), DelimitedDataModelElement('brack_mail', b')'), FixedDataModelElement('brack2', b') '), ])), FixedDataModelElement('sp2', b'<'), DelimitedDataModelElement('mail', b'>'), FixedDataModelElement('r_str', b'> R='), DelimitedDataModelElement('r', b' '), FixedDataModelElement('t_str', b' T='), VariableByteDataModelElement('t', alphabet), ]), SequenceModelElement('aster', [ FixedDataModelElement('aster', b' ** '), DelimitedDataModelElement('command', b' '), FixedDataModelElement( 'headers_str', b' Too many "Received" headers - suspected mail loop' ) ]), FixedDataModelElement('completed', b' Completed'), FixedDataModelElement('frozen', b' Message is frozen'), FixedDataModelElement('frozen', b' Frozen (delivery error message)') ]) ]) ]) ]) return model
def get_model(): """Return a model to parse Suricata Event logs from the AIT-LDS.""" conn = SequenceModelElement('conn', [ FixedDataModelElement('src_ip_str', b'"src_ip":"'), FirstMatchModelElement('ip', [ SequenceModelElement('ipv4', [ IpAddressDataModelElement('src_ip'), FixedDataModelElement('src_port_str', b'","src_port":'), DecimalIntegerValueModelElement('src_port'), FixedDataModelElement('dest_ip_str', b',"dest_ip":"'), IpAddressDataModelElement('dest_ip'), FixedDataModelElement('dest_port_str', b'","dest_port":'), DecimalIntegerValueModelElement('dest_port'), FixedDataModelElement('proto_str', b',"proto":"'), DelimitedDataModelElement('proto', b'"'), FixedDataModelElement('quote', b'"') ]), SequenceModelElement('ipv6', [ DelimitedDataModelElement('src_ip', b'"'), FixedDataModelElement('dest_ip_str', b'","dest_ip":"'), DelimitedDataModelElement('dest_ip', b'"'), FixedDataModelElement('proto_str', b'","proto":"'), DelimitedDataModelElement('proto', b'"'), FixedDataModelElement('icmp_type_str', b'","icmp_type":'), DecimalIntegerValueModelElement('icmp_type'), FixedDataModelElement('icmp_code_str', b',"icmp_code":'), DecimalIntegerValueModelElement('icmp_code'), ]), ]) ]) http = SequenceModelElement('http', [ FixedDataModelElement('hostname_str', b',"http":{"hostname":"'), DelimitedDataModelElement('hostname', b'"'), FixedDataModelElement('url_str', b'","url":"'), DelimitedDataModelElement('url', b'"', escape=b'\\'), FixedDataModelElement('http_user_agent_str', b'","http_user_agent":"'), DelimitedDataModelElement('http_user_agent', b'"'), OptionalMatchModelElement( 'content_type', SequenceModelElement('content_type', [ FixedDataModelElement('http_content_type_str', b'","http_content_type":"'), DelimitedDataModelElement('http_content_type', b'"'), ])), OptionalMatchModelElement( 'http_refer', SequenceModelElement('http_refer', [ FixedDataModelElement('http_refer_str', b'","http_refer":"'), DelimitedDataModelElement('http_refer', b'"'), ])), FixedDataModelElement('http_method_str', b'","http_method":"'), DelimitedDataModelElement('http_method', b'"'), FixedDataModelElement('protocol_str', b'","protocol":"'), DelimitedDataModelElement('protocol', b'"'), FixedDataModelElement('quote_str', b'"'), OptionalMatchModelElement( 'status', SequenceModelElement('status', [ FixedDataModelElement('status_str', b',"status":'), DecimalIntegerValueModelElement('status'), ])), OptionalMatchModelElement( 'redirect', SequenceModelElement('redirect', [ FixedDataModelElement('redirect_str', b',"redirect":"'), DelimitedDataModelElement('redirect', b'"'), FixedDataModelElement('quote_str', b'"') ])), FixedDataModelElement('length_str', b',"length":'), DecimalIntegerValueModelElement('length'), FixedDataModelElement('brack_str', b'}') ]) model = SequenceModelElement('model', [ FixedDataModelElement('time_str', b'{"timestamp":"'), DateTimeModelElement('time', b'%Y-%m-%dT%H:%M:%S.%f'), FixedDataModelElement('plus_sign', b'+'), DecimalIntegerValueModelElement('tz'), FixedDataModelElement('comma_str', b'",'), OptionalMatchModelElement( 'flow_id', SequenceModelElement('flow_id', [ FixedDataModelElement('flow_id_str', b'"flow_id":'), DecimalIntegerValueModelElement('flow_id'), FixedDataModelElement('comma_str', b',') ])), OptionalMatchModelElement( 'in_iface', SequenceModelElement('in_iface', [ FixedDataModelElement('in_iface_str', b'"in_iface":"'), DelimitedDataModelElement('in_iface', b'"'), FixedDataModelElement('comma_str', b'",') ])), FixedDataModelElement('event_type_str', b'"event_type":"'), FirstMatchModelElement('event_type', [ SequenceModelElement('dns', [ FixedDataModelElement('dns_str', b'dns",'), conn, SequenceModelElement('dns', [ FixedDataModelElement('type_str', b',"dns":{"type":"'), DelimitedDataModelElement('type', b'"'), FixedDataModelElement('id_str', b'","id":'), DecimalIntegerValueModelElement('id'), OptionalMatchModelElement( 'rcode', SequenceModelElement('rcode', [ FixedDataModelElement('rcode_str', b',"rcode":"'), DelimitedDataModelElement('rcode', b'"'), FixedDataModelElement('quote_str', b'"') ])), FixedDataModelElement('rrname_str', b',"rrname":"'), DelimitedDataModelElement('rrname', b'"'), OptionalMatchModelElement( 'rrtype', SequenceModelElement('rrtype', [ FixedDataModelElement('rrtype_str', b'","rrtype":"'), DelimitedDataModelElement('rrtype', b'"') ])), FixedDataModelElement('quote', b'"'), OptionalMatchModelElement( 'tx_id', SequenceModelElement('tx_id', [ FixedDataModelElement('tx_id_str', b',"tx_id":'), DecimalIntegerValueModelElement('tx_id') ])), OptionalMatchModelElement( 'ttl', SequenceModelElement('ttl', [ FixedDataModelElement('ttl_str', b',"ttl":'), DecimalIntegerValueModelElement('ttl') ])), OptionalMatchModelElement( 'rdata', SequenceModelElement('rdata', [ FixedDataModelElement('rdata_str', b',"rdata":"'), DelimitedDataModelElement('rdata', b'"'), FixedDataModelElement('quote_str', b'"') ])), FixedDataModelElement('brack_str', b'}}') ]), ]), SequenceModelElement('flow', [ FixedDataModelElement('flow_str', b'flow",'), conn, OptionalMatchModelElement( 'app_proto', SequenceModelElement('app_proto', [ FixedDataModelElement('app_proto_str', b',"app_proto":"'), DelimitedDataModelElement('app_proto', b'"'), FixedDataModelElement('quote_str', b'"') ])), OptionalMatchModelElement( 'app_proto_tc', SequenceModelElement('app_proto_tc', [ FixedDataModelElement('app_proto_tc_str', b',"app_proto_tc":"'), DelimitedDataModelElement('app_proto_tc', b'"'), FixedDataModelElement('quote_str', b'"') ])), SequenceModelElement('flow', [ FixedDataModelElement('pkts_toserver_str', b',"flow":{"pkts_toserver":'), DecimalIntegerValueModelElement('pkts_toserver'), FixedDataModelElement('pkts_toclient_str', b',"pkts_toclient":'), DecimalIntegerValueModelElement('pkts_toclient'), FixedDataModelElement('bytes_toserver_str', b',"bytes_toserver":'), DecimalIntegerValueModelElement('bytes_toserver'), FixedDataModelElement('bytes_toclient_str', b',"bytes_toclient":'), DecimalIntegerValueModelElement('bytes_toclient'), FixedDataModelElement('start_str', b',"start":"'), DelimitedDataModelElement('start', b'"'), FixedDataModelElement('end_str', b'","end":"'), DelimitedDataModelElement('end', b'"'), FixedDataModelElement('age_str', b'","age":'), DecimalIntegerValueModelElement('age'), FixedDataModelElement('state_str', b',"state":"'), DelimitedDataModelElement('state', b'"'), FixedDataModelElement('reason_str', b'","reason":"'), DelimitedDataModelElement('reason', b'"'), FixedDataModelElement('alerted_str', b'","alerted":'), FixedWordlistDataModelElement('alerted', [b'true', b'false']), FixedDataModelElement('brack_str1', b'}'), OptionalMatchModelElement( 'tcp', SequenceModelElement('tcp', [ FixedDataModelElement('tcp_flags_str', b',"tcp":{"tcp_flags":"'), HexStringModelElement('tcp_flags'), FixedDataModelElement('tcp_flags_ts_str', b'","tcp_flags_ts":"'), HexStringModelElement('tcp_flags_ts'), FixedDataModelElement('tcp_flags_tc_str', b'","tcp_flags_tc":"'), HexStringModelElement('tcp_flags_tc'), OptionalMatchModelElement( 'flags', SequenceModelElement('flags', [ FixedDataModelElement( 'syn_str', b'","syn":'), FixedWordlistDataModelElement( 'syn', [b'true', b'false']), OptionalMatchModelElement( 'fin', SequenceModelElement( 'fin', [ FixedDataModelElement( 'fin_str', b',"fin":'), FixedWordlistDataModelElement( 'fin', [b'true', b'false']), ])), OptionalMatchModelElement( 'rst', SequenceModelElement( 'rst', [ FixedDataModelElement( 'rst_str', b',"rst":'), FixedWordlistDataModelElement( 'rst', [b'true', b'false']), ])), OptionalMatchModelElement( 'psh', SequenceModelElement( 'psh', [ FixedDataModelElement( 'psh_str', b',"psh":'), FixedWordlistDataModelElement( 'psh', [b'true', b'false']), ])), FixedDataModelElement( 'ack_str', b',"ack":'), FixedWordlistDataModelElement( 'ack', [b'true', b'false']), FixedDataModelElement( 'tcp_state_str', b',"state":"'), DelimitedDataModelElement( 'tcp_state', b'"'), ])), FixedDataModelElement('tcp_brack_str', b'"}'), ])), FixedDataModelElement('brack_str2', b'}') ]), ]), SequenceModelElement('http', [ FixedDataModelElement('http_str', b'http",'), conn, FixedDataModelElement('tx_id_str', b',"tx_id":'), DecimalIntegerValueModelElement('tx_id'), http, FixedDataModelElement('brack_str', b'}') ]), SequenceModelElement('fileinfo', [ FixedDataModelElement('fileinfo_str', b'fileinfo",'), conn, http, FixedDataModelElement('app_proto_str', b',"app_proto":"'), DelimitedDataModelElement('app_proto', b'"'), SequenceModelElement('fileinfo', [ FixedDataModelElement('fileinfo_str', b'","fileinfo":{'), OptionalMatchModelElement( 'filename', SequenceModelElement('filename', [ FixedDataModelElement('filename_str', b'"filename":"'), DelimitedDataModelElement('filename', b'"'), FixedDataModelElement('quote_str', b'",') ])), FixedDataModelElement('state_str', b'"state":"'), DelimitedDataModelElement('state', b'"'), FixedDataModelElement('stored_str', b'","stored":'), FixedWordlistDataModelElement('stored', [b'true', b'false']), FixedDataModelElement('size_str', b',"size":'), DecimalIntegerValueModelElement('size'), FixedDataModelElement('tx_id_str', b',"tx_id":'), DecimalIntegerValueModelElement('tx_id'), FixedDataModelElement('brack_str', b'}}') ]), ]), SequenceModelElement('stats', [ FixedDataModelElement('stats_str', b'stats",'), FixedDataModelElement('uptime_str', b'"stats":{"uptime":'), DecimalIntegerValueModelElement('uptime'), SequenceModelElement('capture', [ FixedDataModelElement('capture_str', b',"capture":{'), FixedDataModelElement('kernel_packets_str', b'"kernel_packets":'), DecimalIntegerValueModelElement('kernel_packets'), FixedDataModelElement('kernel_drops_str', b',"kernel_drops":'), DecimalIntegerValueModelElement('kernel_drops'), FixedDataModelElement('brack_str', b'}') ]), SequenceModelElement('decoder', [ FixedDataModelElement('pkts_str', b',"decoder":{"pkts":'), DecimalIntegerValueModelElement('pkts'), FixedDataModelElement('bytes_str', b',"bytes":'), DecimalIntegerValueModelElement('bytes'), FixedDataModelElement('invalid_str', b',"invalid":'), DecimalIntegerValueModelElement('invalid'), FixedDataModelElement('ipv4_str', b',"ipv4":'), DecimalIntegerValueModelElement('ipv4'), FixedDataModelElement('ipv6_str', b',"ipv6":'), DecimalIntegerValueModelElement('ipv6'), FixedDataModelElement('ethernet_str', b',"ethernet":'), DecimalIntegerValueModelElement('ethernet'), FixedDataModelElement('raw_str', b',"raw":'), DecimalIntegerValueModelElement('raw'), FixedDataModelElement('null_str', b',"null":'), DecimalIntegerValueModelElement('null'), FixedDataModelElement('sll_str', b',"sll":'), DecimalIntegerValueModelElement('sll'), FixedDataModelElement('tcp_str', b',"tcp":'), DecimalIntegerValueModelElement('tcp'), FixedDataModelElement('udp_str', b',"udp":'), DecimalIntegerValueModelElement('udp'), FixedDataModelElement('sctp_str', b',"sctp":'), DecimalIntegerValueModelElement('sctp'), FixedDataModelElement('icmpv4_str', b',"icmpv4":'), DecimalIntegerValueModelElement('icmpv4'), FixedDataModelElement('icmpv6_str', b',"icmpv6":'), DecimalIntegerValueModelElement('icmpv6'), FixedDataModelElement('ppp_str', b',"ppp":'), DecimalIntegerValueModelElement('ppp'), FixedDataModelElement('pppoe_str', b',"pppoe":'), DecimalIntegerValueModelElement('pppoe'), FixedDataModelElement('gre_str', b',"gre":'), DecimalIntegerValueModelElement('gre'), FixedDataModelElement('vlan_str', b',"vlan":'), DecimalIntegerValueModelElement('vlan'), FixedDataModelElement('vlan_qinq_str', b',"vlan_qinq":'), DecimalIntegerValueModelElement('vlan_qinq'), FixedDataModelElement('teredo_str', b',"teredo":'), DecimalIntegerValueModelElement('teredo'), FixedDataModelElement('ipv4_in_ipv6_str', b',"ipv4_in_ipv6":'), DecimalIntegerValueModelElement('ipv4_in_ipv6'), FixedDataModelElement('ipv6_in_ipv6_str', b',"ipv6_in_ipv6":'), DecimalIntegerValueModelElement('ipv6_in_ipv6'), FixedDataModelElement('mpls_str', b',"mpls":'), DecimalIntegerValueModelElement('mpls'), FixedDataModelElement('avg_pkt_size_str', b',"avg_pkt_size":'), DecimalIntegerValueModelElement('avg_pkt_size'), FixedDataModelElement('max_pkt_size_str', b',"max_pkt_size":'), DecimalIntegerValueModelElement('max_pkt_size'), FixedDataModelElement('erspan_str', b',"erspan":'), DecimalIntegerValueModelElement('erspan'), SequenceModelElement('ipraw', [ FixedDataModelElement( 'invalid_ip_version_str', b',"ipraw":{"invalid_ip_version":'), DecimalIntegerValueModelElement('invalid_ip_version'), ]), SequenceModelElement('ltnull', [ FixedDataModelElement('ipraw_pkt_too_small_str', b'},"ltnull":{"pkt_too_small":'), DecimalIntegerValueModelElement('ipraw_pkt_too_small'), FixedDataModelElement('unsupported_type', b',"unsupported_type":'), DecimalIntegerValueModelElement('unsupported_type'), ]), SequenceModelElement('dce', [ FixedDataModelElement('dce_pkt_too_small_str', b'},"dce":{"pkt_too_small":'), DecimalIntegerValueModelElement('dce_pkt_too_small'), FixedDataModelElement('brack_str', b'}') ]) ]), SequenceModelElement('flow', [ FixedDataModelElement('memcap_str', b'},"flow":{"memcap":'), DecimalIntegerValueModelElement('memcap'), FixedDataModelElement('spare_str', b',"spare":'), DecimalIntegerValueModelElement('spare'), FixedDataModelElement('emerg_mode_entered_str', b',"emerg_mode_entered":'), DecimalIntegerValueModelElement('emerg_mode_entered'), FixedDataModelElement('emerg_mode_over_str', b',"emerg_mode_over":'), DecimalIntegerValueModelElement('emerg_mode_over'), FixedDataModelElement('tcp_reuse_str', b',"tcp_reuse":'), DecimalIntegerValueModelElement('tcp_reuse'), FixedDataModelElement('memuse_str', b',"memuse":'), DecimalIntegerValueModelElement('memuse'), ]), SequenceModelElement('defrag', [ SequenceModelElement('ipv4', [ FixedDataModelElement( 'fragments_str', b'},"defrag":{"ipv4":{"fragments":'), DecimalIntegerValueModelElement('fragments'), FixedDataModelElement('reassembled_str', b',"reassembled":'), DecimalIntegerValueModelElement('reassembled_str'), FixedDataModelElement('timeouts_str', b',"timeouts":'), DecimalIntegerValueModelElement('timeouts'), ]), SequenceModelElement('ipv6', [ FixedDataModelElement('fragments_str', b'},"ipv6":{"fragments":'), DecimalIntegerValueModelElement('fragments'), FixedDataModelElement('reassembled_str', b',"reassembled":'), DecimalIntegerValueModelElement('reassembled_str'), FixedDataModelElement('timeouts_str', b',"timeouts":'), DecimalIntegerValueModelElement('timeouts'), ]), FixedDataModelElement('max_frag_hits_str', b'},"max_frag_hits":'), DecimalIntegerValueModelElement('max_frag_hits'), ]), SequenceModelElement('tcp', [ FixedDataModelElement('sessions_str', b'},"tcp":{"sessions":'), DecimalIntegerValueModelElement('sessions'), FixedDataModelElement('ssn_memcap_drop_str', b',"ssn_memcap_drop":'), DecimalIntegerValueModelElement('ssn_memcap_drop'), FixedDataModelElement('pseudo_str', b',"pseudo":'), DecimalIntegerValueModelElement('pseudo'), FixedDataModelElement('pseudo_failed_str', b',"pseudo_failed":'), DecimalIntegerValueModelElement('pseudo_failed'), FixedDataModelElement('invalid_checksum_str', b',"invalid_checksum":'), DecimalIntegerValueModelElement('invalid_checksum'), FixedDataModelElement('no_flow_str', b',"no_flow":'), DecimalIntegerValueModelElement('no_flow'), FixedDataModelElement('syn_str', b',"syn":'), DecimalIntegerValueModelElement('syn'), FixedDataModelElement('synack_str', b',"synack":'), DecimalIntegerValueModelElement('synack'), FixedDataModelElement('rst_str', b',"rst":'), DecimalIntegerValueModelElement('rst'), FixedDataModelElement('segment_memcap_drop_str', b',"segment_memcap_drop":'), DecimalIntegerValueModelElement('segment_memcap_drop'), FixedDataModelElement('stream_depth_reached_str', b',"stream_depth_reached":'), DecimalIntegerValueModelElement('stream_depth_reached'), FixedDataModelElement('reassembly_gap_str', b',"reassembly_gap":'), DecimalIntegerValueModelElement('reassembly_gap'), FixedDataModelElement('memuse_str', b',"memuse":'), DecimalIntegerValueModelElement('memuse'), FixedDataModelElement('reassembly_memuse_str', b',"reassembly_memuse":'), DecimalIntegerValueModelElement('reassembly_memuse'), ]), SequenceModelElement('detect', [ FixedDataModelElement('alert_str', b'},"detect":{"alert":'), DecimalIntegerValueModelElement('alert') ]), SequenceModelElement('app_layer', [ SequenceModelElement('flow', [ FixedDataModelElement( 'http_str', b'},"app_layer":{"flow":{"http":'), DecimalIntegerValueModelElement('http'), FixedDataModelElement('ftp_str', b',"ftp":'), DecimalIntegerValueModelElement('ftp'), FixedDataModelElement('smtp_str', b',"smtp":'), DecimalIntegerValueModelElement('smtp'), FixedDataModelElement('tls_str', b',"tls":'), DecimalIntegerValueModelElement('tls'), FixedDataModelElement('ssh_str', b',"ssh":'), DecimalIntegerValueModelElement('ssh'), FixedDataModelElement('imap_str', b',"imap":'), DecimalIntegerValueModelElement('imap'), FixedDataModelElement('msn_str', b',"msn":'), DecimalIntegerValueModelElement('msn'), FixedDataModelElement('smb_str', b',"smb":'), DecimalIntegerValueModelElement('smb'), FixedDataModelElement('dcerpc_tcp_str', b',"dcerpc_tcp":'), DecimalIntegerValueModelElement('dcerpc_tcp'), FixedDataModelElement('dns_tcp_str', b',"dns_tcp":'), DecimalIntegerValueModelElement('dns_tcp'), FixedDataModelElement('failed_tcp_str', b',"failed_tcp":'), DecimalIntegerValueModelElement('failed_tcp'), FixedDataModelElement('dcerpc_udp_str', b',"dcerpc_udp":'), DecimalIntegerValueModelElement('dcerpc_udp'), FixedDataModelElement('dns_udp_str', b',"dns_udp":'), DecimalIntegerValueModelElement('dns_udp'), FixedDataModelElement('failed_udp_str', b',"failed_udp":'), DecimalIntegerValueModelElement('failed_udp'), ]), SequenceModelElement('tx', [ FixedDataModelElement('http_str', b'},"tx":{"http":'), DecimalIntegerValueModelElement('http'), FixedDataModelElement('smtp_str', b',"smtp":'), DecimalIntegerValueModelElement('smtp'), FixedDataModelElement('tls_str', b',"tls":'), DecimalIntegerValueModelElement('tls'), FixedDataModelElement('dns_tcp_str', b',"dns_tcp":'), DecimalIntegerValueModelElement('dns_tcp'), FixedDataModelElement('dns_udp_str', b',"dns_udp":'), DecimalIntegerValueModelElement('dns_udp'), ]) ]), SequenceModelElement('flow_mgr', [ FixedDataModelElement('closed_pruned_str', b'}},"flow_mgr":{"closed_pruned":'), DecimalIntegerValueModelElement('closed_pruned'), FixedDataModelElement('new_pruned_str', b',"new_pruned":'), DecimalIntegerValueModelElement('new_pruned'), FixedDataModelElement('est_pruned_str', b',"est_pruned":'), DecimalIntegerValueModelElement('est_pruned'), FixedDataModelElement('bypassed_pruned_str', b',"bypassed_pruned":'), DecimalIntegerValueModelElement('bypassed_pruned'), FixedDataModelElement('flows_checked_str', b',"flows_checked":'), DecimalIntegerValueModelElement('flows_checked'), FixedDataModelElement('flows_notimeout_str', b',"flows_notimeout":'), DecimalIntegerValueModelElement('flows_notimeout'), FixedDataModelElement('flows_timeout_str', b',"flows_timeout":'), DecimalIntegerValueModelElement('flows_timeout'), FixedDataModelElement('flows_timeout_inuse_str', b',"flows_timeout_inuse":'), DecimalIntegerValueModelElement('flows_timeout_inuse'), FixedDataModelElement('flows_removed_str', b',"flows_removed":'), DecimalIntegerValueModelElement('flows_removed'), FixedDataModelElement('rows_checked_str', b',"rows_checked":'), DecimalIntegerValueModelElement('rows_checked'), FixedDataModelElement('rows_skipped_str', b',"rows_skipped":'), DecimalIntegerValueModelElement('rows_skipped'), FixedDataModelElement('rows_empty_str', b',"rows_empty":'), DecimalIntegerValueModelElement('rows_empty'), FixedDataModelElement('rows_busy_str', b',"rows_busy":'), DecimalIntegerValueModelElement('rows_busy'), FixedDataModelElement('rows_maxlen_str', b',"rows_maxlen":'), DecimalIntegerValueModelElement('rows_maxlen'), ]), SequenceModelElement('dns', [ FixedDataModelElement('memuse_str', b'},"dns":{"memuse":'), DecimalIntegerValueModelElement('memuse'), FixedDataModelElement('memcap_state_str', b',"memcap_state":'), DecimalIntegerValueModelElement('memcap_state'), FixedDataModelElement('memcap_global_str', b',"memcap_global":'), DecimalIntegerValueModelElement('memcap_global'), ]), SequenceModelElement('http', [ FixedDataModelElement('memuse_str', b'},"http":{"memuse":'), DecimalIntegerValueModelElement('memuse'), FixedDataModelElement('memcap_str', b',"memcap":'), DecimalIntegerValueModelElement('memcap'), ]), FixedDataModelElement('quote_str', b'}}}') ]), SequenceModelElement('tls', [ FixedDataModelElement('tls_str', b'tls",'), conn, SequenceModelElement('tls', [ FixedDataModelElement('subject_str', b',"tls":{"subject":"'), DelimitedDataModelElement('subject', b'"'), FixedDataModelElement('issuerdn_str', b'","issuerdn":"'), DelimitedDataModelElement('issuerdn', b'"'), FixedDataModelElement('fingerprint_str', b'","fingerprint":"'), DelimitedDataModelElement('fingerprint', b'"'), OptionalMatchModelElement( 'sni', SequenceModelElement('sni', [ FixedDataModelElement('sni_str', b'","sni":"'), DelimitedDataModelElement('sni', b'"'), ])), FixedDataModelElement('version_str', b'","version":"'), DelimitedDataModelElement('version', b'"'), FixedDataModelElement('notbefore_str', b'","notbefore":"'), DelimitedDataModelElement('notbefore', b'"'), FixedDataModelElement('notafter_str', b'","notafter":"'), DelimitedDataModelElement('notafter', b'"'), ]), FixedDataModelElement('brack_str', b'"}}') ]), SequenceModelElement('alert', [ FixedDataModelElement('alert_str', b'alert",'), conn, OptionalMatchModelElement( 'tx_id', SequenceModelElement('tx_id', [ FixedDataModelElement('tx_id', b',"tx_id":'), DecimalIntegerValueModelElement('tx_id'), ])), SequenceModelElement('alert', [ FixedDataModelElement('action_str', b',"alert":{"action":"'), DelimitedDataModelElement('action', b'"'), FixedDataModelElement('gid_str', b'","gid":'), DecimalIntegerValueModelElement('gid'), FixedDataModelElement('signature_id_str', b',"signature_id":'), DecimalIntegerValueModelElement('signature_id'), FixedDataModelElement('rev_str', b',"rev":'), DecimalIntegerValueModelElement('rev'), FixedDataModelElement('signature_str', b',"signature":"'), DelimitedDataModelElement('signature', b'"'), FixedDataModelElement('category_str', b'","category":"'), DelimitedDataModelElement('category', b'"'), FixedDataModelElement('severity_str', b'","severity":'), DecimalIntegerValueModelElement('severity'), FixedDataModelElement('brack_str', b'}') ]), http, FixedDataModelElement('brack_str', b'}') ]), ]) ]) return model
def get_model(): """Get the model.""" interface_name_model = VariableByteDataModelElement( "interface", b"0123456789abcdefghijklmnopqrstuvwxyz.") dtme = DateTimeModelElement("expire-date", b"%Y-%m-%dT%H:%M:%SZ") type_children = [ SequenceModelElement("exit", [ FixedDataModelElement("s0", b"ntpd exiting on signal "), DecimalIntegerValueModelElement("signal") ]), SequenceModelElement("listen-drop", [ FixedDataModelElement("s0", b"Listen and drop on "), DecimalIntegerValueModelElement("fd"), FixedDataModelElement("s1", b" "), interface_name_model, FixedDataModelElement("s2", b" "), FirstMatchModelElement("address", [ IpAddressDataModelElement("ipv4"), DelimitedDataModelElement("ipv6", b" "), FixedDataModelElement("ipv6_missing", b"[::]") ]), FirstMatchModelElement("udp", [ FixedDataModelElement("s3", b" UDP 123"), FixedDataModelElement("s3", b":123") ]) ]), SequenceModelElement("listen-normal", [ FixedDataModelElement("s0", b"Listen normally on "), DecimalIntegerValueModelElement("fd"), FixedDataModelElement("s1", b" "), interface_name_model, FixedDataModelElement("s2", b" "), IpAddressDataModelElement("ip"), FirstMatchModelElement("msg", [ FixedDataModelElement("port-new", b":123"), FixedDataModelElement("port-old", b" UDP 123") ]) ]), SequenceModelElement("listen-routing", [ FixedDataModelElement("s0", b"Listening on routing socket on fd #"), DecimalIntegerValueModelElement("fd"), FixedDataModelElement("s1", b" for interface updates") ]), SequenceModelElement("soliciting-pool", [ FixedDataModelElement("s0", b"Soliciting pool server "), IpAddressDataModelElement("pool-server-ip") ]), SequenceModelElement("starting", [ FixedDataModelElement("s0", b"ntpd "), DelimitedDataModelElement("version", b" "), FixedDataModelElement("s1", b" (1): Starting") ]), SequenceModelElement("no-root", [ FixedDataModelElement("s0", b"must be run as root, not uid "), DecimalIntegerValueModelElement("uid") ]), SequenceModelElement("leapsecond-file", [ FixedDataModelElement("s0", b"leapsecond file ('"), DelimitedDataModelElement("file", b"'"), FixedDataModelElement("s1", b"'): "), FirstMatchModelElement("first", [ FixedDataModelElement("msg", b"good hash signature"), SequenceModelElement("seq", [ FixedDataModelElement("s2", b"loaded, expire="), dtme, FixedDataModelElement("s3", b" last="), dtme, FixedDataModelElement("s4", b" ofs="), DecimalIntegerValueModelElement("ofs") ]) ]) ]), FixedDataModelElement( "unable-to-bind", b"unable to bind to wildcard address :: - another process may be running - EXITING" ), FixedDataModelElement("new-interfaces", b"new interface(s) found: waking up resolver"), FixedDataModelElement( "ntp-io", b"ntp_io: estimated max descriptors: 1024, initial socket boundary: 16" ), FixedDataModelElement("peers-refreshed", b"peers refreshed"), FixedDataModelElement("log-file", b"logging to file /var/log/ntplog"), FixedDataModelElement("command-line", b"Command line: ntpd"), SequenceModelElement("precision", [ FixedDataModelElement("s0", b"proto: precision = "), DecimalFloatValueModelElement("precision"), FixedDataModelElement("s1", b" usec ("), DecimalIntegerValueModelElement( "usec", value_sign_type=DecimalIntegerValueModelElement. SIGN_TYPE_OPTIONAL), FixedDataModelElement("s2", b")") ]) ] model = SequenceModelElement("ntpd", [ FixedDataModelElement("sname", b"ntpd["), DecimalIntegerValueModelElement("pid"), FixedDataModelElement("s0", b"]: "), FirstMatchModelElement("msg", type_children) ]) return model
def get_model(): """Return a model to parse Syslogs from the AIT-LDS.""" alphabet = b'!"#$%&\'()*+,-./0123456789:;<>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\\^_`abcdefghijklmnopqrstuvwxyz{|}~=[]' user_info = SequenceModelElement('user_info', [ FixedDataModelElement('user_str', b'user=<'), OptionalMatchModelElement('user', DelimitedDataModelElement('user', b'>')), FixedDataModelElement('method_str', b'>'), OptionalMatchModelElement( 'method', SequenceModelElement('method', [ FixedDataModelElement('method_str', b', method='), DelimitedDataModelElement('method', b','), ])), FixedDataModelElement('rip_str', b', rip='), IpAddressDataModelElement('rip'), FixedDataModelElement('lip_str', b', lip='), IpAddressDataModelElement('lip'), OptionalMatchModelElement( 'mpid', SequenceModelElement('mpid', [ FixedDataModelElement('mpid_str', b', mpid='), DecimalIntegerValueModelElement('mpid'), ])), OptionalMatchModelElement( 'secured', FixedDataModelElement('secured_str', b', secured')), FixedDataModelElement('session_str', b', session=<'), DelimitedDataModelElement('session', b'>'), FixedDataModelElement('bracket_str', b'>'), ]) model = SequenceModelElement( 'model', [ DateTimeModelElement('time', b'%b %d %H:%M:%S', start_year=2020), FixedDataModelElement('sp1', b' '), DelimitedDataModelElement('host', b' '), FirstMatchModelElement( 'service', [ SequenceModelElement('dovecot', [ FixedDataModelElement('dovecot_str', b' dovecot: '), FirstMatchModelElement('imap', [ SequenceModelElement('imap', [ FixedDataModelElement('imap_str', b'imap('), DelimitedDataModelElement('user', b')'), FixedDataModelElement('bracket_str', b'): '), FirstMatchModelElement('message', [ SequenceModelElement( 'logout', [ FixedDataModelElement( 'logout_str', b'Logged out in='), DecimalIntegerValueModelElement( 'in'), FixedDataModelElement( 'out_str', b' out='), DecimalIntegerValueModelElement( 'out') ]), SequenceModelElement( 'err_mail', [ FixedDataModelElement( 'mail_str', b'Error: Failed to autocreate mailbox INBOX: Internal error occurred. ' b'Refer to server log for more information. [' ), DelimitedDataModelElement( 'err_time', b']'), FixedDataModelElement( 'brack', b']') ]), SequenceModelElement( 'err_open', [ FixedDataModelElement( 'err_str', b'Error: '), DelimitedDataModelElement( 'function_name', b'('), FixedDataModelElement( 'brack_str1', b'('), DelimitedDataModelElement( 'arg', b')'), FixedDataModelElement( 'failed_str', b') failed: Permission denied (euid=' ), DecimalIntegerValueModelElement( 'euid'), FixedDataModelElement( 'brack_str2', b'('), DelimitedDataModelElement( 'euid_user', b')'), FixedDataModelElement( 'egid_str', b') egid='), DecimalIntegerValueModelElement( 'egid'), FixedDataModelElement( 'brack_str3', b'('), DelimitedDataModelElement( 'egid_user', b')'), FixedDataModelElement( 'perm_str', b') missing +w perm: '), DelimitedDataModelElement( 'mail_path', b','), FixedDataModelElement( 'group_str', b', we\'re not in group '), DecimalIntegerValueModelElement( 'group_id'), FixedDataModelElement( 'brack_str4', b'('), DelimitedDataModelElement( 'group_name', b')'), FixedDataModelElement( 'owned_str', b'), dir owned by '), DelimitedDataModelElement( 'owner', b' '), FixedDataModelElement( 'mode_str', b' mode='), DelimitedDataModelElement( 'mode', b')'), FixedDataModelElement( 'brack_str5', b')'), OptionalMatchModelElement( 'set', SequenceModelElement( 'set', [ FixedDataModelElement( 'set_str', b' (set'), DelimitedDataModelElement( 'param', b'='), FixedDataModelElement( 'equal_str', b'='), DelimitedDataModelElement( 'val', b')'), FixedDataModelElement( 'brack_str6', b')') ])) ]), SequenceModelElement( 'err_mail', [ FixedDataModelElement( 'mail_str', b'Failed to autocreate mailbox INBOX: Internal error occurred. ' b'Refer to server log for more information. [' ), DelimitedDataModelElement( 'err_time', b']'), FixedDataModelElement( 'brack', b']') ]), ]), ]), SequenceModelElement('imap_login', [ FixedDataModelElement('imap_login_str', b'imap-login: '******'login', [ SequenceModelElement( 'disconnected_str', [ FixedDataModelElement( 'disconnected_str', b'Disconnected '), FirstMatchModelElement( 'auth', [ SequenceModelElement( 'auth_failed', [ FixedDataModelElement( 'auth_failed_str', b'(auth failed, ' ), DecimalIntegerValueModelElement( 'attempts'), FixedDataModelElement( 'attempts_str', b' attempts in ' ), ]), FixedDataModelElement( 'no_auth_str', b'(no auth attempts in ' ), FixedDataModelElement( 'no_auth_str', b'(disconnected before auth was ready, waited ' ), ]), DecimalIntegerValueModelElement( 'duration'), FixedDataModelElement( 'secs_str', b' secs): '), user_info ]), SequenceModelElement( 'login', [ FixedDataModelElement( 'login_str', b'Login: '******'anvil', [ FixedDataModelElement( 'anvil_str', b'Error: anvil:'), AnyByteDataModelElement( 'anvil_msg') ]), SequenceModelElement( 'auth_responding', [ FixedDataModelElement( 'auth_responding_str', b'Warning: Auth process not responding, ' b'delayed sending initial response (greeting): ' ), user_info ]), ]), ]), SequenceModelElement('auth', [ FixedDataModelElement('auth_worker_str', b'auth: '), AnyByteDataModelElement('message') ]), SequenceModelElement('auth_worker', [ FixedDataModelElement('auth_worker_str', b'auth-worker('), DecimalIntegerValueModelElement('pid'), FixedDataModelElement('brack', b'):'), AnyByteDataModelElement('message') ]), SequenceModelElement('master', [ FixedDataModelElement('master_str', b'master: '), AnyByteDataModelElement('message') ]) ]), ]), SequenceModelElement('horde', [ FixedDataModelElement('horde_str', b' HORDE: '), FirstMatchModelElement('horde', [ SequenceModelElement('imp', [ FixedDataModelElement('succ_str', b'[imp] '), FirstMatchModelElement('imp', [ SequenceModelElement( 'login', [ FixedDataModelElement( 'succ_str', b'Login success for '), DelimitedDataModelElement( 'user', b' '), FixedDataModelElement( 'brack_str1', b' ('), IpAddressDataModelElement('ip'), FixedDataModelElement( 'to_str', b') to {'), DelimitedDataModelElement( 'imap_addr', b'}'), FixedDataModelElement( 'brack_str2', b'}'), ]), SequenceModelElement( 'message_sent', [ FixedDataModelElement( 'message_sent_str', b'Message sent to '), VariableByteDataModelElement( 'user', b'.0123456789@ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz' ), OptionalMatchModelElement( 'mail', SequenceModelElement( 'mail', [ FixedDataModelElement( 'brack_mail1', b' <'), DelimitedDataModelElement( 'mail', b'>'), FixedDataModelElement( 'brack_mail2', b'>') ])), RepeatedElementDataModelElement( 'more_recepients_rep', SequenceModelElement( 'more_recepients', [ FixedDataModelElement( 'comma_str', b', '), VariableByteDataModelElement( 'more_recepients_mail', b'.0123456789@ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz' ), OptionalMatchModelElement( 'more_recepients_mail', SequenceModelElement( 'more_recepients', [ FixedDataModelElement( 'brack_more_recepients1', b' <'), DelimitedDataModelElement( 'more_recepients', b'>'), FixedDataModelElement( 'brack_more_recepients2', b'>') ])), ])), FixedDataModelElement( 'from_str', b' from '), DelimitedDataModelElement( 'user', b' '), FixedDataModelElement( 'brack_str1', b' ('), IpAddressDataModelElement('ip'), FixedDataModelElement( 'brack_str2', b')'), ]), SequenceModelElement( 'login_failed', [ FixedDataModelElement( 'succ_str', b'FAILED LOGIN for '), DelimitedDataModelElement( 'user', b' '), FixedDataModelElement( 'brack_str1', b' ('), IpAddressDataModelElement('ip'), FixedDataModelElement( 'to_str', b') to {'), DelimitedDataModelElement( 'imap_addr', b'}'), FixedDataModelElement( 'brack_str2', b'}'), ]), SequenceModelElement( 'status', [ FixedDataModelElement( 'status_str', b'[status] Could not open mailbox "INBOX".' ), ]), SequenceModelElement( 'sync_token', [ FixedDataModelElement( 'sync_token_str', b'[getSyncToken] IMAP error reported by server.' ), ]), SequenceModelElement( 'auth_failed', [ FixedDataModelElement( 'auth_failed_str', b'[login] Authentication failed.' ), ]), ]), ]), SequenceModelElement('horde', [ FixedDataModelElement('succ_str', b'[horde] '), FirstMatchModelElement('horde', [ SequenceModelElement( 'success', [ FixedDataModelElement( 'success_str', b'Login success for '), DelimitedDataModelElement( 'user', b' '), FixedDataModelElement( 'brack_str1', b' to horde ('), IpAddressDataModelElement('ip'), FixedDataModelElement( 'brack_str2', b')'), ]), SequenceModelElement( 'success', [ FixedDataModelElement( 'success_str', b'User '), DelimitedDataModelElement( 'user', b' '), FixedDataModelElement( 'brack_str1', b' logged out of Horde ('), IpAddressDataModelElement('ip'), FixedDataModelElement( 'brack_str2', b')'), ]), SequenceModelElement( 'login_failed', [ FixedDataModelElement( 'failed_str', b'FAILED LOGIN for '), DelimitedDataModelElement( 'user', b' '), FixedDataModelElement( 'to_horde_str', b' to horde ('), IpAddressDataModelElement('ip'), FixedDataModelElement( 'brack_str', b')'), ]), ]) ]), SequenceModelElement('function', [ FixedWordlistDataModelElement( 'horde_function', [b'[nag]', b'[turba]', b'[horde]']), FixedDataModelElement('nag_str', b' PHP ERROR: '), FirstMatchModelElement('php_error', [ SequenceModelElement( 'declaration', [ FixedDataModelElement( 'declaration_str', b'Declaration of '), DelimitedDataModelElement( 'function_name1', b'('), FixedDataModelElement( 'brack_str1', b'('), OptionalMatchModelElement( 'arg1', DelimitedDataModelElement( 'arg1', b')')), FixedDataModelElement( 'failed_str', b') should be compatible with ' ), DelimitedDataModelElement( 'function_name2', b'('), FixedDataModelElement( 'brack_str2', b'('), OptionalMatchModelElement( 'arg2', DelimitedDataModelElement( 'arg2', b')')), FixedDataModelElement( 'brack_str3', b')'), ]), FixedDataModelElement( 'file_str', b'finfo_file(): Empty filename or path' ), FixedDataModelElement( 'header_str', b'Cannot modify header information - headers already sent' ) ]) ]), SequenceModelElement('guest', [ FixedDataModelElement( 'guest_str', b'Guest user is not authorized for Horde (Host: ' ), IpAddressDataModelElement('ip'), FixedDataModelElement('brack_str', b').') ]), SequenceModelElement('php_error', [ FixedDataModelElement('php_error_str', b'PHP ERROR: '), DelimitedDataModelElement('msg', b' ['), ]), SequenceModelElement('free_msg', [ DelimitedDataModelElement('msg', b' ['), ]) ]), FixedDataModelElement('to_str', b' [pid '), DecimalIntegerValueModelElement('pid'), FixedDataModelElement('line_str', b' on line '), DecimalIntegerValueModelElement('line'), FixedDataModelElement('of_str', b' of "'), DelimitedDataModelElement('path', b'"'), FixedDataModelElement('brack_str', b'"]') ]), SequenceModelElement( 'cron', [ FixedDataModelElement('cron_str', b' CRON['), DecimalIntegerValueModelElement('pid'), FixedDataModelElement('brack_str1', b']: '), FirstMatchModelElement( 'cron', [ SequenceModelElement( 'cmd', [ FixedDataModelElement( 'brack_str', b'('), DelimitedDataModelElement( 'user', b')'), FixedDataModelElement( 'cmd_str', b') CMD '), AnyByteDataModelElement('cmd_msg') ]), SequenceModelElement( 'session', [ # This only occurs in auth.log DelimitedDataModelElement( 'pam', b'('), FixedDataModelElement( 'brack_str', b'('), DelimitedDataModelElement( 'name', b')'), FixedDataModelElement( 'session_str', b'): session '), FixedWordlistDataModelElement( 'status', [b'opened', b'closed']), FixedDataModelElement( 'user_str', b' for user '), VariableByteDataModelElement( 'user', alphabet), OptionalMatchModelElement( 'uid', SequenceModelElement( 'uid', [ FixedDataModelElement( 'uid_str', b' by (uid='), DecimalIntegerValueModelElement( 'uid'), FixedDataModelElement( 'brack_str', b')') ])) ]) ]) ]), SequenceModelElement( 'auth', [ # This only occurs in auth.log FixedDataModelElement('auth_str', b' auth: '), DelimitedDataModelElement('pam', b'('), FixedDataModelElement('brack_str', b'('), DelimitedDataModelElement('name', b')'), FixedDataModelElement( 'session_str', b'): authentication failure; logname='), OptionalMatchModelElement( 'logname', DelimitedDataModelElement('logname', b' ')), FixedDataModelElement('uid_str', b' uid='), DecimalIntegerValueModelElement('uid'), FixedDataModelElement('euid_str', b' euid='), DecimalIntegerValueModelElement('euid'), FixedDataModelElement('tty_str', b' tty='), DelimitedDataModelElement('tty', b' '), FixedDataModelElement('ruser_str', b' ruser='******'ruser', b' '), FixedDataModelElement('rhost_str', b' rhost='), IpAddressDataModelElement('rhost'), OptionalMatchModelElement( 'user', SequenceModelElement('user', [ FixedDataModelElement( 'user_str', b' user='******'user', alphabet) ])) ]), SequenceModelElement('systemd', [ FixedDataModelElement('systemd_str', b' systemd['), DecimalIntegerValueModelElement('pid'), FixedDataModelElement('brack_str1', b']: '), FirstMatchModelElement('systemd', [ FixedDataModelElement( 'php_starting_str', b'Starting Clean php session files...'), FixedDataModelElement( 'php_started_str', b'Started Clean php session files.'), FixedDataModelElement( 'php_starting_str', b'Starting Cleanup of Temporary Directories...' ), FixedDataModelElement( 'php_started_str', b'Started Cleanup of Temporary Directories.'), FixedDataModelElement( 'php_started_str', b'Starting Daily apt upgrade and clean activities...' ), FixedDataModelElement( 'php_started_str', b'Started Daily apt upgrade and clean activities.' ), FixedDataModelElement( 'php_started_str', b'Starting Daily apt download activities...'), FixedDataModelElement( 'php_started_str', b'Started Daily apt download activities.'), FixedDataModelElement( 'php_started_str', b'Starting Security Auditing Service...'), FixedDataModelElement( 'php_started_str', b'Started Security Auditing Service.'), FixedDataModelElement( 'php_started_str', b'Stopping Security Auditing Service...'), FixedDataModelElement( 'php_started_str', b'Stopped Security Auditing Service.'), FixedDataModelElement( 'php_started_str', b'Reloading The Apache HTTP Server.'), FixedDataModelElement( 'php_started_str', b'Reloaded The Apache HTTP Server.'), FixedDataModelElement( 'php_started_str', b'Mounting Arbitrary Executable File Formats File System...' ), FixedDataModelElement( 'php_started_str', b'Mounted Arbitrary Executable File Formats File System.' ), SequenceModelElement('apt', [ FixedDataModelElement('apt_str', b'apt'), AnyByteDataModelElement('apt_msg') ]), SequenceModelElement('service', [ DelimitedDataModelElement('service', b':'), FixedDataModelElement('col_str', b': '), AnyByteDataModelElement('_msg') ]) ]), ]), SequenceModelElement('kernel', [ FixedDataModelElement('kernel_str', b' kernel'), OptionalMatchModelElement( 'id', SequenceModelElement('id', [ FixedDataModelElement('brack_str', b'['), DecimalIntegerValueModelElement('id'), FixedDataModelElement('brack_str2', b']') ])), FixedDataModelElement('col_str', b': '), AnyByteDataModelElement('kernel_msg') ]), SequenceModelElement('augenrules', [ FixedDataModelElement('augenrules_str', b' augenrules['), DecimalIntegerValueModelElement('id'), FixedDataModelElement('brack_str1', b']: '), AnyByteDataModelElement('augenrules_msg') ]), SequenceModelElement('auditd', [ FixedDataModelElement('auditd_str', b' auditd['), DecimalIntegerValueModelElement('id'), FixedDataModelElement('brack_str1', b']: '), AnyByteDataModelElement('auditd_msg') ]), SequenceModelElement('auditd2', [ FixedDataModelElement('auditd2_str', b' auditd: '), AnyByteDataModelElement('auditd_msg') ]), SequenceModelElement('audispd', [ FixedDataModelElement('audispd_str', b' audispd: '), AnyByteDataModelElement('audispd_msg') ]), SequenceModelElement('liblogging', [ FixedDataModelElement('liblogging_str', b' liblogging-stdlog: '), AnyByteDataModelElement('liblogging_msg') ]), SequenceModelElement('freshclam', [ FixedDataModelElement('freshclam_str', b' freshclam['), DecimalIntegerValueModelElement('id'), FixedDataModelElement('brack_str1', b']: '), AnyByteDataModelElement('freshclam_msg') ]), SequenceModelElement('dhclient', [ FixedDataModelElement('dhclient_str', b' dhclient['), DecimalIntegerValueModelElement('id'), FixedDataModelElement('brack_str1', b']: '), FirstMatchModelElement('dhclient', [ SequenceModelElement('dhcprequest', [ FixedDataModelElement('dhcprequest_str', b'DHCPREQUEST of '), IpAddressDataModelElement('src_ip'), FixedDataModelElement('on_str', b' on '), DelimitedDataModelElement( 'network_interface', b' '), FixedDataModelElement('to_str', b' to '), IpAddressDataModelElement('dst_ip'), FixedDataModelElement('port_str', b' port '), DecimalIntegerValueModelElement('port') ]), SequenceModelElement('dhcpack', [ FixedDataModelElement('dhcpack_str', b'DHCPACK of '), IpAddressDataModelElement('dst_ip'), FixedDataModelElement('on_str', b' from '), IpAddressDataModelElement('src_ip') ]), SequenceModelElement('bound', [ FixedDataModelElement('bound_str', b'bound to '), IpAddressDataModelElement('ip'), FixedDataModelElement('renewal_str', b' -- renewal in '), DecimalIntegerValueModelElement('seconds'), FixedDataModelElement('seconds_str', b' seconds.') ]), ]), ]), ]) ]) return model
def get_model(): """Return a model to parse Apache Error logs from the AIT-LDS.""" model = SequenceModelElement("model", [ FixedDataModelElement("sp1", b"["), FixedWordlistDataModelElement( "day", [b"Mon", b"Tue", b"Wed", b"Thu", b"Fri", b"Sat", b"Sun"]), FixedDataModelElement("sp2", b" "), DateTimeModelElement("time", b"%b %d %H:%M:%S.%f %Y"), FixedDataModelElement("error_str", b"] [:error] [pid "), DecimalIntegerValueModelElement("pid"), FixedDataModelElement("sp3", b"] [client "), IpAddressDataModelElement("client_ip"), FixedDataModelElement("colon", b":"), DecimalIntegerValueModelElement("client_port"), FixedDataModelElement("php", b"] PHP "), FirstMatchModelElement("fphp", [ SequenceModelElement("warning", [ FixedDataModelElement("warning_str", b"Warning: "), FirstMatchModelElement("warning", [ SequenceModelElement("declaration", [ FixedDataModelElement("declaration_str", b"Declaration of "), DelimitedDataModelElement("function", b")"), FixedDataModelElement("compatible_str", b") should be compatible with "), DelimitedDataModelElement("function2", b")"), FixedDataModelElement("compatible_str", b") in "), DelimitedDataModelElement("path", b" "), FixedDataModelElement("compatible_str", b" on line "), DecimalIntegerValueModelElement("line"), FixedDataModelElement("referer_str", b", referer: "), AnyByteDataModelElement("referer") ]), SequenceModelElement("system", [ FixedDataModelElement( "system_str", b"system(): Cannot execute a blank command in "), DelimitedDataModelElement("path", b" "), FixedDataModelElement("compatible_str", b" on line "), DecimalIntegerValueModelElement("line") ]) ]) ]), SequenceModelElement("notice", [ FixedDataModelElement("notice_str", b"Notice: Undefined index: "), DelimitedDataModelElement("command", b" "), FixedDataModelElement("sp", b" in "), DelimitedDataModelElement("path", b" "), FixedDataModelElement("compatible_str", b" on line "), DecimalIntegerValueModelElement("line") ]), SequenceModelElement("deprecated", [ FixedDataModelElement( "deprecated_str", b"Deprecated: Methods with the same name as their class " b"will not be constructors in a future version of PHP; "), DelimitedDataModelElement("class", b" "), FixedDataModelElement("constructor_str", b" has a deprecated constructor in "), DelimitedDataModelElement("path", b" "), FixedDataModelElement("compatible_str", b" on line "), DecimalIntegerValueModelElement("line"), FixedDataModelElement("referer_str", b", referer: "), AnyByteDataModelElement("referer"), ]) ]) ]) return model
def test10ipv4_in_rfc1918_match_rule(self): """This case unit the ValueDependentModuloTimeMatchRule.""" description = "Test10Rules" i_pv4_in_rfc1918_match_rule = IPv4InRFC1918MatchRule(self.match_ipv4) self.analysis_context.register_component(i_pv4_in_rfc1918_match_rule, description) ip_address_data_model_element = IpAddressDataModelElement('IPv4') # private addresses match_context = MatchContext(b'192.168.0.0') match_element = ip_address_data_model_element.get_match_element( 'match', match_context) log_atom = LogAtom(match_context.match_data, ParserMatch(match_element), time(), i_pv4_in_rfc1918_match_rule) self.assertTrue(i_pv4_in_rfc1918_match_rule.match(log_atom)) match_context = MatchContext(b'192.168.255.255') match_element = ip_address_data_model_element.get_match_element( 'match', match_context) log_atom = LogAtom(match_context.match_data, ParserMatch(match_element), time(), i_pv4_in_rfc1918_match_rule) self.assertTrue(i_pv4_in_rfc1918_match_rule.match(log_atom)) match_context = MatchContext(b'172.16.0.0') match_element = ip_address_data_model_element.get_match_element( 'match', match_context) log_atom = LogAtom(match_context.match_data, ParserMatch(match_element), time(), i_pv4_in_rfc1918_match_rule) self.assertTrue(i_pv4_in_rfc1918_match_rule.match(log_atom)) match_context = MatchContext(b'172.31.255.255') match_element = ip_address_data_model_element.get_match_element( 'match', match_context) log_atom = LogAtom(match_context.match_data, ParserMatch(match_element), time(), i_pv4_in_rfc1918_match_rule) self.assertTrue(i_pv4_in_rfc1918_match_rule.match(log_atom)) match_context = MatchContext(b'10.0.0.0') match_element = ip_address_data_model_element.get_match_element( 'match', match_context) log_atom = LogAtom(match_context.match_data, ParserMatch(match_element), time(), i_pv4_in_rfc1918_match_rule) self.assertTrue(i_pv4_in_rfc1918_match_rule.match(log_atom)) match_context = MatchContext(b'10.255.255.255') match_element = ip_address_data_model_element.get_match_element( 'match', match_context) log_atom = LogAtom(match_context.match_data, ParserMatch(match_element), time(), i_pv4_in_rfc1918_match_rule) self.assertTrue(i_pv4_in_rfc1918_match_rule.match(log_atom)) # public addresses match_context = MatchContext(b'192.167.255.255') match_element = ip_address_data_model_element.get_match_element( 'match', match_context) log_atom = LogAtom(match_context.match_data, ParserMatch(match_element), time(), i_pv4_in_rfc1918_match_rule) self.assertTrue(not i_pv4_in_rfc1918_match_rule.match(log_atom)) match_context = MatchContext(b'192.169.0.0') match_element = ip_address_data_model_element.get_match_element( 'match', match_context) log_atom = LogAtom(match_context.match_data, ParserMatch(match_element), time(), i_pv4_in_rfc1918_match_rule) self.assertTrue(not i_pv4_in_rfc1918_match_rule.match(log_atom)) match_context = MatchContext(b'172.15.255.255') match_element = ip_address_data_model_element.get_match_element( 'match', match_context) log_atom = LogAtom(not match_context.match_data, ParserMatch(match_element), time(), i_pv4_in_rfc1918_match_rule) self.assertTrue(not i_pv4_in_rfc1918_match_rule.match(log_atom)) match_context = MatchContext(b'172.32.0.0') match_element = ip_address_data_model_element.get_match_element( 'match', match_context) log_atom = LogAtom(match_context.match_data, ParserMatch(match_element), time(), i_pv4_in_rfc1918_match_rule) self.assertTrue(not i_pv4_in_rfc1918_match_rule.match(log_atom)) match_context = MatchContext(b'9.255.255.255') match_element = ip_address_data_model_element.get_match_element( 'match', match_context) log_atom = LogAtom(match_context.match_data, ParserMatch(match_element), time(), i_pv4_in_rfc1918_match_rule) self.assertTrue(not i_pv4_in_rfc1918_match_rule.match(log_atom)) match_context = MatchContext(b'11.0.0.0') match_element = ip_address_data_model_element.get_match_element( 'match', match_context) log_atom = LogAtom(match_context.match_data, ParserMatch(match_element), time(), i_pv4_in_rfc1918_match_rule) self.assertTrue(not i_pv4_in_rfc1918_match_rule.match(log_atom))
def get_model(): """Return a model to parse Syslogs from the AIT-LDS.""" alphabet = b"!'#$%&\"()*+,-./0123456789:;<>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\\^_`abcdefghijklmnopqrstuvwxyz{|}~=[]" user_info = SequenceModelElement("user_info", [ FixedDataModelElement("user_str", b"user=<"), OptionalMatchModelElement( "user", DelimitedDataModelElement("user", b">") ), FixedDataModelElement("method_str", b">"), OptionalMatchModelElement( "method", SequenceModelElement("method", [ FixedDataModelElement("method_str", b", method="), DelimitedDataModelElement("method", b","), ]) ), FixedDataModelElement("rip_str", b", rip="), IpAddressDataModelElement("rip"), FixedDataModelElement("lip_str", b", lip="), IpAddressDataModelElement("lip"), OptionalMatchModelElement( "mpid", SequenceModelElement("mpid", [ FixedDataModelElement("mpid_str", b", mpid="), DecimalIntegerValueModelElement("mpid"), ]) ), OptionalMatchModelElement( "secured", FixedDataModelElement("secured_str", b", secured") ), FixedDataModelElement("session_str", b", session=<"), DelimitedDataModelElement("session", b">"), FixedDataModelElement("bracket_str", b">"), ]) model = SequenceModelElement("model", [ DateTimeModelElement("time", b"%b %d %H:%M:%S", start_year=2020), FixedDataModelElement("sp1", b" "), DelimitedDataModelElement("host", b" "), FirstMatchModelElement("service", [ SequenceModelElement("dovecot", [ FixedDataModelElement("dovecot_str", b" dovecot: "), FirstMatchModelElement("imap", [ SequenceModelElement("imap", [ FixedDataModelElement("imap_str", b"imap("), DelimitedDataModelElement("user", b")"), FixedDataModelElement("bracket_str", b"): "), FirstMatchModelElement("message", [ SequenceModelElement("logout", [ FixedDataModelElement("logout_str", b"Logged out in="), DecimalIntegerValueModelElement("in"), FixedDataModelElement("out_str", b" out="), DecimalIntegerValueModelElement("out") ]), SequenceModelElement("err_mail", [ FixedDataModelElement("mail_str", b"Error: Failed to autocreate mailbox INBOX: Internal error occurred. " b"Refer to server log for more information. ["), DelimitedDataModelElement("err_time", b"]"), FixedDataModelElement("brack", b"]") ]), SequenceModelElement("err_open", [ FixedDataModelElement("err_str", b"Error: "), DelimitedDataModelElement("function_name", b"("), FixedDataModelElement("brack_str1", b"("), DelimitedDataModelElement("arg", b")"), FixedDataModelElement("failed_str", b") failed: Permission denied (euid="), DecimalIntegerValueModelElement("euid"), FixedDataModelElement("brack_str2", b"("), DelimitedDataModelElement("euid_user", b")"), FixedDataModelElement("egid_str", b") egid="), DecimalIntegerValueModelElement("egid"), FixedDataModelElement("brack_str3", b"("), DelimitedDataModelElement("egid_user", b")"), FixedDataModelElement("perm_str", b") missing +w perm: "), DelimitedDataModelElement("mail_path", b","), FixedDataModelElement("group_str", b", we're not in group "), DecimalIntegerValueModelElement("group_id"), FixedDataModelElement("brack_str4", b"("), DelimitedDataModelElement("group_name", b")"), FixedDataModelElement("owned_str", b"), dir owned by "), DelimitedDataModelElement("owner", b" "), FixedDataModelElement("mode_str", b" mode="), DelimitedDataModelElement("mode", b")"), FixedDataModelElement("brack_str5", b")"), OptionalMatchModelElement( "set", SequenceModelElement("set", [ FixedDataModelElement("set_str", b" (set"), DelimitedDataModelElement("param", b"="), FixedDataModelElement("equal_str", b"="), DelimitedDataModelElement("val", b")"), FixedDataModelElement("brack_str6", b")") ]) ) ]), SequenceModelElement("err_mail", [ FixedDataModelElement("mail_str", b"Failed to autocreate mailbox INBOX: Internal error occurred. " b"Refer to server log for more information. ["), DelimitedDataModelElement("err_time", b"]"), FixedDataModelElement("brack", b"]") ]), ]), ]), SequenceModelElement("imap_login", [ FixedDataModelElement("imap_login_str", b"imap-login: "******"login", [ SequenceModelElement("disconnected_str", [ FixedDataModelElement("disconnected_str", b"Disconnected "), FirstMatchModelElement("auth", [ SequenceModelElement("auth_failed", [ FixedDataModelElement("auth_failed_str", b"(auth failed, "), DecimalIntegerValueModelElement("attempts"), FixedDataModelElement("attempts_str", b" attempts in "), ]), FixedDataModelElement("no_auth_str", b"(no auth attempts in "), FixedDataModelElement("no_auth_str", b"(disconnected before auth was ready, waited "), ]), DecimalIntegerValueModelElement("duration"), FixedDataModelElement("secs_str", b" secs): "), user_info ]), SequenceModelElement("login", [ FixedDataModelElement("login_str", b"Login: "******"anvil", [ FixedDataModelElement("anvil_str", b"Error: anvil:"), AnyByteDataModelElement("anvil_msg") ]), SequenceModelElement("auth_responding", [ FixedDataModelElement("auth_responding_str", b"Warning: Auth process not responding, " b"delayed sending initial response (greeting): "), user_info ]), ]), ]), SequenceModelElement("auth", [ FixedDataModelElement("auth_worker_str", b"auth: "), AnyByteDataModelElement("message") ]), SequenceModelElement("auth_worker", [ FixedDataModelElement("auth_worker_str", b"auth-worker("), DecimalIntegerValueModelElement("pid"), FixedDataModelElement("brack", b"):"), AnyByteDataModelElement("message") ]), SequenceModelElement("master", [ FixedDataModelElement("master_str", b"master: "), AnyByteDataModelElement("message") ]) ]), ]), SequenceModelElement("horde", [ FixedDataModelElement("horde_str", b" HORDE: "), FirstMatchModelElement("horde", [ SequenceModelElement("imp", [ FixedDataModelElement("succ_str", b"[imp] "), FirstMatchModelElement("imp", [ SequenceModelElement("login", [ FixedDataModelElement("succ_str", b"Login success for "), DelimitedDataModelElement("user", b" "), FixedDataModelElement("brack_str1", b" ("), IpAddressDataModelElement("ip"), FixedDataModelElement("to_str", b") to {"), DelimitedDataModelElement("imap_addr", b"}"), FixedDataModelElement("brack_str2", b"}"), ]), SequenceModelElement("message_sent", [ FixedDataModelElement("message_sent_str", b"Message sent to "), VariableByteDataModelElement("user", b".0123456789@ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz"), OptionalMatchModelElement( "mail", SequenceModelElement("mail", [ FixedDataModelElement("brack_mail1", b" <"), DelimitedDataModelElement("mail", b">"), FixedDataModelElement("brack_mail2", b">") ]) ), RepeatedElementDataModelElement( "more_recepients_rep", SequenceModelElement("more_recepients", [ FixedDataModelElement("comma_str", b", "), VariableByteDataModelElement("more_recepients_mail", b".0123456789@ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz"), OptionalMatchModelElement( "more_recepients_mail", SequenceModelElement("more_recepients", [ FixedDataModelElement("brack_more_recepients1", b" <"), DelimitedDataModelElement("more_recepients", b">"), FixedDataModelElement("brack_more_recepients2", b">") ]) ), ]) ), FixedDataModelElement("from_str", b" from "), DelimitedDataModelElement("user", b" "), FixedDataModelElement("brack_str1", b" ("), IpAddressDataModelElement("ip"), FixedDataModelElement("brack_str2", b")"), ]), SequenceModelElement("login_failed", [ FixedDataModelElement("succ_str", b"FAILED LOGIN for "), DelimitedDataModelElement("user", b" "), FixedDataModelElement("brack_str1", b" ("), IpAddressDataModelElement("ip"), FixedDataModelElement("to_str", b") to {"), DelimitedDataModelElement("imap_addr", b"}"), FixedDataModelElement("brack_str2", b"}"), ]), SequenceModelElement("status", [ FixedDataModelElement("status_str", b'[status] Could not open mailbox "INBOX".'), ]), SequenceModelElement("sync_token", [ FixedDataModelElement("sync_token_str", b"[getSyncToken] IMAP error reported by server."), ]), SequenceModelElement("auth_failed", [ FixedDataModelElement("auth_failed_str", b"[login] Authentication failed."), ]), ]), ]), SequenceModelElement("horde", [ FixedDataModelElement("succ_str", b"[horde] "), FirstMatchModelElement("horde", [ SequenceModelElement("success", [ FixedDataModelElement("success_str", b"Login success for "), DelimitedDataModelElement("user", b" "), FixedDataModelElement("brack_str1", b" to horde ("), IpAddressDataModelElement("ip"), FixedDataModelElement("brack_str2", b")"), ]), SequenceModelElement("success", [ FixedDataModelElement("success_str", b"User "), DelimitedDataModelElement("user", b" "), FixedDataModelElement("brack_str1", b" logged out of Horde ("), IpAddressDataModelElement("ip"), FixedDataModelElement("brack_str2", b")"), ]), SequenceModelElement("login_failed", [ FixedDataModelElement("failed_str", b"FAILED LOGIN for "), DelimitedDataModelElement("user", b" "), FixedDataModelElement("to_horde_str", b" to horde ("), IpAddressDataModelElement("ip"), FixedDataModelElement("brack_str", b")"), ]), ]) ]), SequenceModelElement("function", [ FixedWordlistDataModelElement("horde_function", [b"[nag]", b"[turba]", b"[horde]"]), FixedDataModelElement("nag_str", b" PHP ERROR: "), FirstMatchModelElement("php_error", [ SequenceModelElement("declaration", [ FixedDataModelElement("declaration_str", b"Declaration of "), DelimitedDataModelElement("function_name1", b"("), FixedDataModelElement("brack_str1", b"("), OptionalMatchModelElement( "arg1", DelimitedDataModelElement("arg1", b")") ), FixedDataModelElement("failed_str", b") should be compatible with "), DelimitedDataModelElement("function_name2", b"("), FixedDataModelElement("brack_str2", b"("), OptionalMatchModelElement( "arg2", DelimitedDataModelElement("arg2", b")") ), FixedDataModelElement("brack_str3", b")"), ]), FixedDataModelElement("file_str", b"finfo_file(): Empty filename or path"), FixedDataModelElement("header_str", b"Cannot modify header information - headers already sent") ]) ]), SequenceModelElement("guest", [ FixedDataModelElement("guest_str", b"Guest user is not authorized for Horde (Host: "), IpAddressDataModelElement("ip"), FixedDataModelElement("brack_str", b").") ]), SequenceModelElement("php_error", [ FixedDataModelElement("php_error_str", b"PHP ERROR: "), DelimitedDataModelElement("msg", b" ["), ]), SequenceModelElement("free_msg", [ DelimitedDataModelElement("msg", b" ["), ]) ]), FixedDataModelElement("to_str", b" [pid "), DecimalIntegerValueModelElement("pid"), FixedDataModelElement("line_str", b" on line "), DecimalIntegerValueModelElement("line"), FixedDataModelElement("of_str", b' of "'), DelimitedDataModelElement("path", b'"'), FixedDataModelElement("brack_str", b'"]') ]), SequenceModelElement("cron", [ FixedDataModelElement("cron_str", b" CRON["), DecimalIntegerValueModelElement("pid"), FixedDataModelElement("brack_str1", b"]: "), FirstMatchModelElement("cron", [ SequenceModelElement("cmd", [ FixedDataModelElement("brack_str", b"("), DelimitedDataModelElement("user", b")"), FixedDataModelElement("cmd_str", b") CMD "), AnyByteDataModelElement("cmd_msg") ]), SequenceModelElement("session", [ # This only occurs in auth.log DelimitedDataModelElement("pam", b"("), FixedDataModelElement("brack_str", b"("), DelimitedDataModelElement("name", b")"), FixedDataModelElement("session_str", b"): session "), FixedWordlistDataModelElement("status", [b"opened", b"closed"]), FixedDataModelElement("user_str", b" for user "), VariableByteDataModelElement("user", alphabet), OptionalMatchModelElement( "uid", SequenceModelElement("uid", [ FixedDataModelElement("uid_str", b" by (uid="), DecimalIntegerValueModelElement("uid"), FixedDataModelElement("brack_str", b")") ]) ) ]) ]) ]), SequenceModelElement("auth", [ # This only occurs in auth.log FixedDataModelElement("auth_str", b" auth: "), DelimitedDataModelElement("pam", b"("), FixedDataModelElement("brack_str", b"("), DelimitedDataModelElement("name", b")"), FixedDataModelElement("session_str", b"): authentication failure; logname="), OptionalMatchModelElement( "logname", DelimitedDataModelElement("logname", b" ") ), FixedDataModelElement("uid_str", b" uid="), DecimalIntegerValueModelElement("uid"), FixedDataModelElement("euid_str", b" euid="), DecimalIntegerValueModelElement("euid"), FixedDataModelElement("tty_str", b" tty="), DelimitedDataModelElement("tty", b" "), FixedDataModelElement("ruser_str", b" ruser="******"ruser", b" "), FixedDataModelElement("rhost_str", b" rhost="), IpAddressDataModelElement("rhost"), OptionalMatchModelElement( "user", SequenceModelElement("user", [ FixedDataModelElement("user_str", b" user="******"user", alphabet) ]) ) ]), SequenceModelElement("systemd", [ FixedDataModelElement("systemd_str", b" systemd["), DecimalIntegerValueModelElement("pid"), FixedDataModelElement("brack_str1", b"]: "), FirstMatchModelElement("systemd", [ FixedDataModelElement("php_starting_str", b"Starting Clean php session files..."), FixedDataModelElement("php_started_str", b"Started Clean php session files."), FixedDataModelElement("php_starting_str", b"Starting Cleanup of Temporary Directories..."), FixedDataModelElement("php_started_str", b"Started Cleanup of Temporary Directories."), FixedDataModelElement("php_started_str", b"Starting Daily apt upgrade and clean activities..."), FixedDataModelElement("php_started_str", b"Started Daily apt upgrade and clean activities."), FixedDataModelElement("php_started_str", b"Starting Daily apt download activities..."), FixedDataModelElement("php_started_str", b"Started Daily apt download activities."), FixedDataModelElement("php_started_str", b"Starting Security Auditing Service..."), FixedDataModelElement("php_started_str", b"Started Security Auditing Service."), FixedDataModelElement("php_started_str", b"Stopping Security Auditing Service..."), FixedDataModelElement("php_started_str", b"Stopped Security Auditing Service."), FixedDataModelElement("php_started_str", b"Reloading The Apache HTTP Server."), FixedDataModelElement("php_started_str", b"Reloaded The Apache HTTP Server."), FixedDataModelElement("php_started_str", b"Mounting Arbitrary Executable File Formats File System..."), FixedDataModelElement("php_started_str", b"Mounted Arbitrary Executable File Formats File System."), SequenceModelElement("apt", [ FixedDataModelElement("apt_str", b"apt"), AnyByteDataModelElement("apt_msg") ]), SequenceModelElement("service", [ DelimitedDataModelElement("service", b":"), FixedDataModelElement("col_str", b": "), AnyByteDataModelElement("_msg") ]) ]), ]), SequenceModelElement("kernel", [ FixedDataModelElement("kernel_str", b" kernel"), OptionalMatchModelElement( "id", SequenceModelElement("id", [ FixedDataModelElement("brack_str", b"["), DecimalIntegerValueModelElement("id"), FixedDataModelElement("brack_str2", b"]") ]) ), FixedDataModelElement("col_str", b": "), AnyByteDataModelElement("kernel_msg") ]), SequenceModelElement("augenrules", [ FixedDataModelElement("augenrules_str", b" augenrules["), DecimalIntegerValueModelElement("id"), FixedDataModelElement("brack_str1", b"]: "), AnyByteDataModelElement("augenrules_msg") ]), SequenceModelElement("auditd", [ FixedDataModelElement("auditd_str", b" auditd["), DecimalIntegerValueModelElement("id"), FixedDataModelElement("brack_str1", b"]: "), AnyByteDataModelElement("auditd_msg") ]), SequenceModelElement("auditd2", [ FixedDataModelElement("auditd2_str", b" auditd: "), AnyByteDataModelElement("auditd_msg") ]), SequenceModelElement("audispd", [ FixedDataModelElement("audispd_str", b" audispd: "), AnyByteDataModelElement("audispd_msg") ]), SequenceModelElement("liblogging", [ FixedDataModelElement("liblogging_str", b" liblogging-stdlog: "), AnyByteDataModelElement("liblogging_msg") ]), SequenceModelElement("freshclam", [ FixedDataModelElement("freshclam_str", b" freshclam["), DecimalIntegerValueModelElement("id"), FixedDataModelElement("brack_str1", b"]: "), AnyByteDataModelElement("freshclam_msg") ]), SequenceModelElement("dhclient", [ FixedDataModelElement("dhclient_str", b" dhclient["), DecimalIntegerValueModelElement("id"), FixedDataModelElement("brack_str1", b"]: "), FirstMatchModelElement("dhclient", [ SequenceModelElement("dhcprequest", [ FixedDataModelElement("dhcprequest_str", b"DHCPREQUEST of "), IpAddressDataModelElement("src_ip"), FixedDataModelElement("on_str", b" on "), DelimitedDataModelElement("network_interface", b" "), FixedDataModelElement("to_str", b" to "), IpAddressDataModelElement("dst_ip"), FixedDataModelElement("port_str", b" port "), DecimalIntegerValueModelElement("port") ]), SequenceModelElement("dhcpack", [ FixedDataModelElement("dhcpack_str", b"DHCPACK of "), IpAddressDataModelElement("dst_ip"), FixedDataModelElement("on_str", b" from "), IpAddressDataModelElement("src_ip") ]), SequenceModelElement("bound", [ FixedDataModelElement("bound_str", b"bound to "), IpAddressDataModelElement("ip"), FixedDataModelElement("renewal_str", b" -- renewal in "), DecimalIntegerValueModelElement("seconds"), FixedDataModelElement("seconds_str", b" seconds.") ]), ]), ]), ]) ]) return model
def get_model(user_name_model=None): """Return a model to parse a sshd information message after any standard logging preamble, e.g. from syslog.""" if user_name_model is None: user_name_model = VariableByteDataModelElement( 'user', b'0123456789abcdefghijklmnopqrstuvwxyz.-') from_str = b' from ' port = b' port ' preauth = b' [preauth]' type_children = [ SequenceModelElement('accepted key', [ FixedDataModelElement('s0', b'Accepted publickey for '), user_name_model, FixedDataModelElement('s1', from_str), IpAddressDataModelElement('clientip'), FixedDataModelElement('s2', port), DecimalIntegerValueModelElement('port'), FixedDataModelElement('s3', b' ssh2: RSA '), VariableByteDataModelElement( 'fingerprint', b'0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ+/:' ) ]), SequenceModelElement('btmp-perm', [ FixedDataModelElement( 's0', b'Excess permission or bad ownership on file /var/log/btmp') ]), SequenceModelElement('close-sess', [ FixedDataModelElement('s0', b'Close session: user '), user_name_model, FixedDataModelElement('s1', from_str), IpAddressDataModelElement('clientip'), FixedDataModelElement('s2', port), DecimalIntegerValueModelElement('port'), FixedDataModelElement('s3', b' id '), DecimalIntegerValueModelElement('userid') ]), SequenceModelElement('closing', [ FixedDataModelElement('s0', b'Closing connection to '), IpAddressDataModelElement('clientip'), FixedDataModelElement('s1', port), DecimalIntegerValueModelElement('port') ]), SequenceModelElement('closed', [ FixedDataModelElement('s0', b'Connection closed by '), IpAddressDataModelElement('clientip') ]), SequenceModelElement('connect', [ FixedDataModelElement('s0', b'Connection from '), IpAddressDataModelElement('clientip'), FixedDataModelElement('s1', port), DecimalIntegerValueModelElement('port'), FixedDataModelElement('s2', b' on '), IpAddressDataModelElement('serverip'), FixedDataModelElement('s3', port), DecimalIntegerValueModelElement('sport') ]), SequenceModelElement('disconnectreq', [ FixedDataModelElement('s0', b'Received disconnect from '), IpAddressDataModelElement('clientip'), FixedDataModelElement('s1', port), DecimalIntegerValueModelElement('port'), FixedDataModelElement('s2', b':'), DecimalIntegerValueModelElement('session'), FixedDataModelElement('s3', b': '), FixedWordlistDataModelElement('reason', [b'disconnected by user']) ]), SequenceModelElement('disconnected', [ FixedDataModelElement('s0', b'Disconnected from '), IpAddressDataModelElement('clientip'), FixedDataModelElement('s1', port), DecimalIntegerValueModelElement('port') ]), SequenceModelElement('error-bind', [ FixedDataModelElement( 's0', b'error: bind: Cannot assign requested address') ]), SequenceModelElement('error-channel-setup', [ FixedDataModelElement( 's0', b'error: channel_setup_fwd_listener: cannot listen to port: '), DecimalIntegerValueModelElement('port') ]), SequenceModelElement('ident-missing', [ FixedDataModelElement( 's0', b'Did not receive identification string from '), IpAddressDataModelElement('clientip') ]), SequenceModelElement('invalid-user', [ FixedDataModelElement('s0', b'Invalid user '), DelimitedDataModelElement('user', from_str), FixedDataModelElement('s1', from_str), IpAddressDataModelElement('clientip') ]), SequenceModelElement('invalid-user-auth-req', [ FixedDataModelElement('s0', b'input_userauth_request: invalid user '), DelimitedDataModelElement('user', preauth), FixedDataModelElement('s1', preauth) ]), SequenceModelElement('postppk', [ FixedDataModelElement('s0', b'Postponed publickey for '), user_name_model, FixedDataModelElement('s1', from_str), IpAddressDataModelElement('clientip'), FixedDataModelElement('s2', port), DecimalIntegerValueModelElement('port'), FixedDataModelElement('s3', b' ssh2 [preauth]') ]), SequenceModelElement('readerr', [ FixedDataModelElement('s0', b'Read error from remote host '), IpAddressDataModelElement('clientip'), FixedDataModelElement('s1', b': Connection timed out') ]), SequenceModelElement('disconnect', [ FixedDataModelElement('s0', b'Received disconnect from '), IpAddressDataModelElement('clientip'), FixedDataModelElement('s1', b': 11: '), FirstMatchModelElement('reason', [ FixedDataModelElement('disconnected', b'disconnected by user'), SequenceModelElement('remotemsg', [ DelimitedDataModelElement('msg', preauth), FixedDataModelElement('s0', preauth) ]) ]) ]), SequenceModelElement('signal', [ FixedDataModelElement('s0', b'Received signal '), DecimalIntegerValueModelElement('signal'), FixedDataModelElement('s1', b'; terminating.') ]), SequenceModelElement('server', [ FixedDataModelElement('s0', b'Server listening on '), DelimitedDataModelElement('serverip', b' '), FixedDataModelElement('s1', port), DecimalIntegerValueModelElement('port'), FixedDataModelElement('s2', b'.') ]), SequenceModelElement('oom-adjust', [ FixedDataModelElement('s0', b'Set /proc/self/oom_score_adj '), OptionalMatchModelElement( 'from', FixedDataModelElement('default', b'from 0 ')), FixedDataModelElement('s1', b'to '), DecimalIntegerValueModelElement( 'newval', value_sign_type=DecimalIntegerValueModelElement. SIGN_TYPE_OPTIONAL) ]), SequenceModelElement('session-start', [ FixedDataModelElement('s0', b'Starting session: '), FirstMatchModelElement('sess-info', [ SequenceModelElement('shell', [ FixedDataModelElement('s0', b'shell on '), DelimitedDataModelElement('terminal', b' ') ]), SequenceModelElement( 'subsystem', [FixedDataModelElement('s0', b'subsystem \'sftp\'')]), SequenceModelElement('forced-command', [ FixedDataModelElement('s0', b'forced-command (key-option) \''), DelimitedDataModelElement('command', b'\' for '), FixedDataModelElement('s1', b'\'') ]) ]), FixedDataModelElement('s1', b' for '), user_name_model, FixedDataModelElement('s2', from_str), IpAddressDataModelElement('clientip'), FixedDataModelElement('s3', port), DecimalIntegerValueModelElement('port'), OptionalMatchModelElement( 'idinfo', SequenceModelElement('idinfo', [ FixedDataModelElement('s0', b' id '), DecimalIntegerValueModelElement('id') ])) ]), SequenceModelElement('transferred', [ FixedDataModelElement('s0', b'Transferred: sent '), DecimalIntegerValueModelElement('sent'), FixedDataModelElement('s1', b', received '), DecimalIntegerValueModelElement('received'), FixedDataModelElement('s1', b' bytes') ]), SequenceModelElement('pam', [ FixedDataModelElement('s0', b'pam_unix(sshd:session): session '), FixedWordlistDataModelElement('change', [b'opened', b'closed']), FixedDataModelElement('s1', b' for user '), user_name_model, OptionalMatchModelElement( 'openby', FixedDataModelElement('default', b' by (uid=0)')) ]), SequenceModelElement('child', [ FixedDataModelElement('s0', b'User child is on pid '), DecimalIntegerValueModelElement('pid') ]) ] model = SequenceModelElement('sshd', [ FixedDataModelElement('sname', b'sshd['), DecimalIntegerValueModelElement('pid'), FixedDataModelElement('s0', b']: '), FirstMatchModelElement('msg', type_children) ]) return model
def get_model(): """Return a model to parse Suricata Event logs from the AIT-LDS.""" conn = SequenceModelElement("conn", [ FixedDataModelElement("src_ip_str", b'"src_ip":"'), FirstMatchModelElement("ip", [ SequenceModelElement("ipv4", [ IpAddressDataModelElement("src_ip"), FixedDataModelElement("src_port_str", b'","src_port":'), DecimalIntegerValueModelElement("src_port"), FixedDataModelElement("dest_ip_str", b',"dest_ip":"'), IpAddressDataModelElement("dest_ip"), FixedDataModelElement("dest_port_str", b'","dest_port":'), DecimalIntegerValueModelElement("dest_port"), FixedDataModelElement("proto_str", b',"proto":"'), DelimitedDataModelElement("proto", b'"'), FixedDataModelElement("quote", b'"') ]), SequenceModelElement("ipv6", [ DelimitedDataModelElement("src_ip", b'"'), FixedDataModelElement("dest_ip_str", b'","dest_ip":"'), DelimitedDataModelElement("dest_ip", b'"'), FixedDataModelElement("proto_str", b'","proto":"'), DelimitedDataModelElement("proto", b'"'), FixedDataModelElement("icmp_type_str", b'","icmp_type":'), DecimalIntegerValueModelElement("icmp_type"), FixedDataModelElement("icmp_code_str", b',"icmp_code":'), DecimalIntegerValueModelElement("icmp_code"), ]), ]) ]) http = SequenceModelElement("http", [ FixedDataModelElement("hostname_str", b',"http":{"hostname":"'), DelimitedDataModelElement("hostname", b'"'), FixedDataModelElement("url_str", b'","url":"'), DelimitedDataModelElement("url", b'"', escape=b"\\"), FixedDataModelElement("http_user_agent_str", b'","http_user_agent":"'), DelimitedDataModelElement("http_user_agent", b'"'), OptionalMatchModelElement( "content_type", SequenceModelElement("content_type", [ FixedDataModelElement("http_content_type_str", b'","http_content_type":"'), DelimitedDataModelElement("http_content_type", b'"'), ])), OptionalMatchModelElement( "http_refer", SequenceModelElement("http_refer", [ FixedDataModelElement("http_refer_str", b'","http_refer":"'), DelimitedDataModelElement("http_refer", b'"'), ])), FixedDataModelElement("http_method_str", b'","http_method":"'), DelimitedDataModelElement("http_method", b'"'), FixedDataModelElement("protocol_str", b'","protocol":"'), DelimitedDataModelElement("protocol", b'"'), FixedDataModelElement("quote_str", b'"'), OptionalMatchModelElement( "status", SequenceModelElement("status", [ FixedDataModelElement("status_str", b',"status":'), DecimalIntegerValueModelElement("status"), ])), OptionalMatchModelElement( "redirect", SequenceModelElement("redirect", [ FixedDataModelElement("redirect_str", b',"redirect":"'), DelimitedDataModelElement("redirect", b'"'), FixedDataModelElement("quote_str", b'"') ])), FixedDataModelElement("length_str", b',"length":'), DecimalIntegerValueModelElement("length"), FixedDataModelElement("brack_str", b"}") ]) model = SequenceModelElement("model", [ FixedDataModelElement("time_str", b'{"timestamp":"'), DateTimeModelElement("time", b"%Y-%m-%dT%H:%M:%S.%f%z"), FixedDataModelElement("comma_str", b'",'), OptionalMatchModelElement( "flow_id", SequenceModelElement("flow_id", [ FixedDataModelElement("flow_id_str", b'"flow_id":'), DecimalIntegerValueModelElement("flow_id"), FixedDataModelElement("comma_str", b",") ])), OptionalMatchModelElement( "in_iface", SequenceModelElement("in_iface", [ FixedDataModelElement("in_iface_str", b'"in_iface":"'), DelimitedDataModelElement("in_iface", b'"'), FixedDataModelElement("comma_str", b'",') ])), FixedDataModelElement("event_type_str", b'"event_type":"'), FirstMatchModelElement("event_type", [ SequenceModelElement("dns", [ FixedDataModelElement("dns_str", b'dns",'), conn, SequenceModelElement("dns", [ FixedDataModelElement("type_str", b',"dns":{"type":"'), DelimitedDataModelElement("type", b'"'), FixedDataModelElement("id_str", b'","id":'), DecimalIntegerValueModelElement("id"), OptionalMatchModelElement( "rcode", SequenceModelElement("rcode", [ FixedDataModelElement("rcode_str", b',"rcode":"'), DelimitedDataModelElement("rcode", b'"'), FixedDataModelElement("quote_str", b'"') ])), FixedDataModelElement("rrname_str", b',"rrname":"'), DelimitedDataModelElement("rrname", b'"'), OptionalMatchModelElement( "rrtype", SequenceModelElement("rrtype", [ FixedDataModelElement("rrtype_str", b'","rrtype":"'), DelimitedDataModelElement("rrtype", b'"') ])), FixedDataModelElement("quote", b'"'), OptionalMatchModelElement( "tx_id", SequenceModelElement("tx_id", [ FixedDataModelElement("tx_id_str", b',"tx_id":'), DecimalIntegerValueModelElement("tx_id") ])), OptionalMatchModelElement( "ttl", SequenceModelElement("ttl", [ FixedDataModelElement("ttl_str", b',"ttl":'), DecimalIntegerValueModelElement("ttl") ])), OptionalMatchModelElement( "rdata", SequenceModelElement("rdata", [ FixedDataModelElement("rdata_str", b',"rdata":"'), DelimitedDataModelElement("rdata", b'"'), FixedDataModelElement("quote_str", b'"') ])), FixedDataModelElement("brack_str", b"}}") ]), ]), SequenceModelElement("flow", [ FixedDataModelElement("flow_str", b'flow",'), conn, OptionalMatchModelElement( "app_proto", SequenceModelElement("app_proto", [ FixedDataModelElement("app_proto_str", b',"app_proto":"'), DelimitedDataModelElement("app_proto", b'"'), FixedDataModelElement("quote_str", b'"') ])), OptionalMatchModelElement( "app_proto_tc", SequenceModelElement("app_proto_tc", [ FixedDataModelElement("app_proto_tc_str", b',"app_proto_tc":"'), DelimitedDataModelElement("app_proto_tc", b'"'), FixedDataModelElement("quote_str", b'"') ])), SequenceModelElement("flow", [ FixedDataModelElement("pkts_toserver_str", b',"flow":{"pkts_toserver":'), DecimalIntegerValueModelElement("pkts_toserver"), FixedDataModelElement("pkts_toclient_str", b',"pkts_toclient":'), DecimalIntegerValueModelElement("pkts_toclient"), FixedDataModelElement("bytes_toserver_str", b',"bytes_toserver":'), DecimalIntegerValueModelElement("bytes_toserver"), FixedDataModelElement("bytes_toclient_str", b',"bytes_toclient":'), DecimalIntegerValueModelElement("bytes_toclient"), FixedDataModelElement("start_str", b',"start":"'), DelimitedDataModelElement("start", b'"'), FixedDataModelElement("end_str", b'","end":"'), DelimitedDataModelElement("end", b'"'), FixedDataModelElement("age_str", b'","age":'), DecimalIntegerValueModelElement("age"), FixedDataModelElement("state_str", b',"state":"'), DelimitedDataModelElement("state", b'"'), FixedDataModelElement("reason_str", b'","reason":"'), DelimitedDataModelElement("reason", b'"'), FixedDataModelElement("alerted_str", b'","alerted":'), FixedWordlistDataModelElement("alerted", [b"true", b"false"]), FixedDataModelElement("brack_str1", b"}"), OptionalMatchModelElement( "tcp", SequenceModelElement("tcp", [ FixedDataModelElement("tcp_flags_str", b',"tcp":{"tcp_flags":"'), HexStringModelElement("tcp_flags"), FixedDataModelElement("tcp_flags_ts_str", b'","tcp_flags_ts":"'), HexStringModelElement("tcp_flags_ts"), FixedDataModelElement("tcp_flags_tc_str", b'","tcp_flags_tc":"'), HexStringModelElement("tcp_flags_tc"), OptionalMatchModelElement( "flags", SequenceModelElement("flags", [ FixedDataModelElement( "syn_str", b'","syn":'), FixedWordlistDataModelElement( "syn", [b"true", b"false"]), OptionalMatchModelElement( "fin", SequenceModelElement( "fin", [ FixedDataModelElement( "fin_str", b',"fin":'), FixedWordlistDataModelElement( "fin", [b"true", b"false"]), ])), OptionalMatchModelElement( "rst", SequenceModelElement( "rst", [ FixedDataModelElement( "rst_str", b',"rst":'), FixedWordlistDataModelElement( "rst", [b"true", b"false"]), ])), OptionalMatchModelElement( "psh", SequenceModelElement( "psh", [ FixedDataModelElement( "psh_str", b',"psh":'), FixedWordlistDataModelElement( "psh", [b"true", b"false"]), ])), FixedDataModelElement( "ack_str", b',"ack":'), FixedWordlistDataModelElement( "ack", [b"true", b"false"]), FixedDataModelElement( "tcp_state_str", b',"state":"'), DelimitedDataModelElement( "tcp_state", b'"'), ])), FixedDataModelElement("tcp_brack_str", b'"}'), ])), FixedDataModelElement("brack_str2", b"}") ]), ]), SequenceModelElement("http", [ FixedDataModelElement("http_str", b'http",'), conn, FixedDataModelElement("tx_id_str", b',"tx_id":'), DecimalIntegerValueModelElement("tx_id"), http, FixedDataModelElement("brack_str", b"}") ]), SequenceModelElement("fileinfo", [ FixedDataModelElement("fileinfo_str", b'fileinfo",'), conn, http, FixedDataModelElement("app_proto_str", b',"app_proto":"'), DelimitedDataModelElement("app_proto", b'"'), SequenceModelElement("fileinfo", [ FixedDataModelElement("fileinfo_str", b'","fileinfo":{'), OptionalMatchModelElement( "filename", SequenceModelElement("filename", [ FixedDataModelElement("filename_str", b'"filename":"'), DelimitedDataModelElement("filename", b'"'), FixedDataModelElement("quote_str", b'",') ])), FixedDataModelElement("state_str", b'"state":"'), DelimitedDataModelElement("state", b'"'), FixedDataModelElement("stored_str", b'","stored":'), FixedWordlistDataModelElement("stored", [b"true", b"false"]), FixedDataModelElement("size_str", b',"size":'), DecimalIntegerValueModelElement("size"), FixedDataModelElement("tx_id_str", b',"tx_id":'), DecimalIntegerValueModelElement("tx_id"), FixedDataModelElement("brack_str", b"}}") ]), ]), SequenceModelElement("stats", [ FixedDataModelElement("stats_str", b'stats",'), FixedDataModelElement("uptime_str", b'"stats":{"uptime":'), DecimalIntegerValueModelElement("uptime"), SequenceModelElement("capture", [ FixedDataModelElement("capture_str", b',"capture":{'), FixedDataModelElement("kernel_packets_str", b'"kernel_packets":'), DecimalIntegerValueModelElement("kernel_packets"), FixedDataModelElement("kernel_drops_str", b',"kernel_drops":'), DecimalIntegerValueModelElement("kernel_drops"), FixedDataModelElement("brack_str", b"}") ]), SequenceModelElement("decoder", [ FixedDataModelElement("pkts_str", b',"decoder":{"pkts":'), DecimalIntegerValueModelElement("pkts"), FixedDataModelElement("bytes_str", b',"bytes":'), DecimalIntegerValueModelElement("bytes"), FixedDataModelElement("invalid_str", b',"invalid":'), DecimalIntegerValueModelElement("invalid"), FixedDataModelElement("ipv4_str", b',"ipv4":'), DecimalIntegerValueModelElement("ipv4"), FixedDataModelElement("ipv6_str", b',"ipv6":'), DecimalIntegerValueModelElement("ipv6"), FixedDataModelElement("ethernet_str", b',"ethernet":'), DecimalIntegerValueModelElement("ethernet"), FixedDataModelElement("raw_str", b',"raw":'), DecimalIntegerValueModelElement("raw"), FixedDataModelElement("null_str", b',"null":'), DecimalIntegerValueModelElement("null"), FixedDataModelElement("sll_str", b',"sll":'), DecimalIntegerValueModelElement("sll"), FixedDataModelElement("tcp_str", b',"tcp":'), DecimalIntegerValueModelElement("tcp"), FixedDataModelElement("udp_str", b',"udp":'), DecimalIntegerValueModelElement("udp"), FixedDataModelElement("sctp_str", b',"sctp":'), DecimalIntegerValueModelElement("sctp"), FixedDataModelElement("icmpv4_str", b',"icmpv4":'), DecimalIntegerValueModelElement("icmpv4"), FixedDataModelElement("icmpv6_str", b',"icmpv6":'), DecimalIntegerValueModelElement("icmpv6"), FixedDataModelElement("ppp_str", b',"ppp":'), DecimalIntegerValueModelElement("ppp"), FixedDataModelElement("pppoe_str", b',"pppoe":'), DecimalIntegerValueModelElement("pppoe"), FixedDataModelElement("gre_str", b',"gre":'), DecimalIntegerValueModelElement("gre"), FixedDataModelElement("vlan_str", b',"vlan":'), DecimalIntegerValueModelElement("vlan"), FixedDataModelElement("vlan_qinq_str", b',"vlan_qinq":'), DecimalIntegerValueModelElement("vlan_qinq"), FixedDataModelElement("teredo_str", b',"teredo":'), DecimalIntegerValueModelElement("teredo"), FixedDataModelElement("ipv4_in_ipv6_str", b',"ipv4_in_ipv6":'), DecimalIntegerValueModelElement("ipv4_in_ipv6"), FixedDataModelElement("ipv6_in_ipv6_str", b',"ipv6_in_ipv6":'), DecimalIntegerValueModelElement("ipv6_in_ipv6"), FixedDataModelElement("mpls_str", b',"mpls":'), DecimalIntegerValueModelElement("mpls"), FixedDataModelElement("avg_pkt_size_str", b',"avg_pkt_size":'), DecimalIntegerValueModelElement("avg_pkt_size"), FixedDataModelElement("max_pkt_size_str", b',"max_pkt_size":'), DecimalIntegerValueModelElement("max_pkt_size"), FixedDataModelElement("erspan_str", b',"erspan":'), DecimalIntegerValueModelElement("erspan"), SequenceModelElement("ipraw", [ FixedDataModelElement( "invalid_ip_version_str", b',"ipraw":{"invalid_ip_version":'), DecimalIntegerValueModelElement("invalid_ip_version"), ]), SequenceModelElement("ltnull", [ FixedDataModelElement("ipraw_pkt_too_small_str", b'},"ltnull":{"pkt_too_small":'), DecimalIntegerValueModelElement("ipraw_pkt_too_small"), FixedDataModelElement("unsupported_type", b',"unsupported_type":'), DecimalIntegerValueModelElement("unsupported_type"), ]), SequenceModelElement("dce", [ FixedDataModelElement("dce_pkt_too_small_str", b'},"dce":{"pkt_too_small":'), DecimalIntegerValueModelElement("dce_pkt_too_small"), FixedDataModelElement("brack_str", b"}") ]) ]), SequenceModelElement("flow", [ FixedDataModelElement("memcap_str", b'},"flow":{"memcap":'), DecimalIntegerValueModelElement("memcap"), FixedDataModelElement("spare_str", b',"spare":'), DecimalIntegerValueModelElement("spare"), FixedDataModelElement("emerg_mode_entered_str", b',"emerg_mode_entered":'), DecimalIntegerValueModelElement("emerg_mode_entered"), FixedDataModelElement("emerg_mode_over_str", b',"emerg_mode_over":'), DecimalIntegerValueModelElement("emerg_mode_over"), FixedDataModelElement("tcp_reuse_str", b',"tcp_reuse":'), DecimalIntegerValueModelElement("tcp_reuse"), FixedDataModelElement("memuse_str", b',"memuse":'), DecimalIntegerValueModelElement("memuse"), ]), SequenceModelElement("defrag", [ SequenceModelElement("ipv4", [ FixedDataModelElement( "fragments_str", b'},"defrag":{"ipv4":{"fragments":'), DecimalIntegerValueModelElement("fragments"), FixedDataModelElement("reassembled_str", b',"reassembled":'), DecimalIntegerValueModelElement("reassembled_str"), FixedDataModelElement("timeouts_str", b',"timeouts":'), DecimalIntegerValueModelElement("timeouts"), ]), SequenceModelElement("ipv6", [ FixedDataModelElement("fragments_str", b'},"ipv6":{"fragments":'), DecimalIntegerValueModelElement("fragments"), FixedDataModelElement("reassembled_str", b',"reassembled":'), DecimalIntegerValueModelElement("reassembled_str"), FixedDataModelElement("timeouts_str", b',"timeouts":'), DecimalIntegerValueModelElement("timeouts"), ]), FixedDataModelElement("max_frag_hits_str", b'},"max_frag_hits":'), DecimalIntegerValueModelElement("max_frag_hits"), ]), SequenceModelElement("tcp", [ FixedDataModelElement("sessions_str", b'},"tcp":{"sessions":'), DecimalIntegerValueModelElement("sessions"), FixedDataModelElement("ssn_memcap_drop_str", b',"ssn_memcap_drop":'), DecimalIntegerValueModelElement("ssn_memcap_drop"), FixedDataModelElement("pseudo_str", b',"pseudo":'), DecimalIntegerValueModelElement("pseudo"), FixedDataModelElement("pseudo_failed_str", b',"pseudo_failed":'), DecimalIntegerValueModelElement("pseudo_failed"), FixedDataModelElement("invalid_checksum_str", b',"invalid_checksum":'), DecimalIntegerValueModelElement("invalid_checksum"), FixedDataModelElement("no_flow_str", b',"no_flow":'), DecimalIntegerValueModelElement("no_flow"), FixedDataModelElement("syn_str", b',"syn":'), DecimalIntegerValueModelElement("syn"), FixedDataModelElement("synack_str", b',"synack":'), DecimalIntegerValueModelElement("synack"), FixedDataModelElement("rst_str", b',"rst":'), DecimalIntegerValueModelElement("rst"), FixedDataModelElement("segment_memcap_drop_str", b',"segment_memcap_drop":'), DecimalIntegerValueModelElement("segment_memcap_drop"), FixedDataModelElement("stream_depth_reached_str", b',"stream_depth_reached":'), DecimalIntegerValueModelElement("stream_depth_reached"), FixedDataModelElement("reassembly_gap_str", b',"reassembly_gap":'), DecimalIntegerValueModelElement("reassembly_gap"), FixedDataModelElement("memuse_str", b',"memuse":'), DecimalIntegerValueModelElement("memuse"), FixedDataModelElement("reassembly_memuse_str", b',"reassembly_memuse":'), DecimalIntegerValueModelElement("reassembly_memuse"), ]), SequenceModelElement("detect", [ FixedDataModelElement("alert_str", b'},"detect":{"alert":'), DecimalIntegerValueModelElement("alert") ]), SequenceModelElement("app_layer", [ SequenceModelElement("flow", [ FixedDataModelElement( "http_str", b'},"app_layer":{"flow":{"http":'), DecimalIntegerValueModelElement("http"), FixedDataModelElement("ftp_str", b',"ftp":'), DecimalIntegerValueModelElement("ftp"), FixedDataModelElement("smtp_str", b',"smtp":'), DecimalIntegerValueModelElement("smtp"), FixedDataModelElement("tls_str", b',"tls":'), DecimalIntegerValueModelElement("tls"), FixedDataModelElement("ssh_str", b',"ssh":'), DecimalIntegerValueModelElement("ssh"), FixedDataModelElement("imap_str", b',"imap":'), DecimalIntegerValueModelElement("imap"), FixedDataModelElement("msn_str", b',"msn":'), DecimalIntegerValueModelElement("msn"), FixedDataModelElement("smb_str", b',"smb":'), DecimalIntegerValueModelElement("smb"), FixedDataModelElement("dcerpc_tcp_str", b',"dcerpc_tcp":'), DecimalIntegerValueModelElement("dcerpc_tcp"), FixedDataModelElement("dns_tcp_str", b',"dns_tcp":'), DecimalIntegerValueModelElement("dns_tcp"), FixedDataModelElement("failed_tcp_str", b',"failed_tcp":'), DecimalIntegerValueModelElement("failed_tcp"), FixedDataModelElement("dcerpc_udp_str", b',"dcerpc_udp":'), DecimalIntegerValueModelElement("dcerpc_udp"), FixedDataModelElement("dns_udp_str", b',"dns_udp":'), DecimalIntegerValueModelElement("dns_udp"), FixedDataModelElement("failed_udp_str", b',"failed_udp":'), DecimalIntegerValueModelElement("failed_udp"), ]), SequenceModelElement("tx", [ FixedDataModelElement("http_str", b'},"tx":{"http":'), DecimalIntegerValueModelElement("http"), FixedDataModelElement("smtp_str", b',"smtp":'), DecimalIntegerValueModelElement("smtp"), FixedDataModelElement("tls_str", b',"tls":'), DecimalIntegerValueModelElement("tls"), FixedDataModelElement("dns_tcp_str", b',"dns_tcp":'), DecimalIntegerValueModelElement("dns_tcp"), FixedDataModelElement("dns_udp_str", b',"dns_udp":'), DecimalIntegerValueModelElement("dns_udp"), ]) ]), SequenceModelElement("flow_mgr", [ FixedDataModelElement("closed_pruned_str", b'}},"flow_mgr":{"closed_pruned":'), DecimalIntegerValueModelElement("closed_pruned"), FixedDataModelElement("new_pruned_str", b',"new_pruned":'), DecimalIntegerValueModelElement("new_pruned"), FixedDataModelElement("est_pruned_str", b',"est_pruned":'), DecimalIntegerValueModelElement("est_pruned"), FixedDataModelElement("bypassed_pruned_str", b',"bypassed_pruned":'), DecimalIntegerValueModelElement("bypassed_pruned"), FixedDataModelElement("flows_checked_str", b',"flows_checked":'), DecimalIntegerValueModelElement("flows_checked"), FixedDataModelElement("flows_notimeout_str", b',"flows_notimeout":'), DecimalIntegerValueModelElement("flows_notimeout"), FixedDataModelElement("flows_timeout_str", b',"flows_timeout":'), DecimalIntegerValueModelElement("flows_timeout"), FixedDataModelElement("flows_timeout_inuse_str", b',"flows_timeout_inuse":'), DecimalIntegerValueModelElement("flows_timeout_inuse"), FixedDataModelElement("flows_removed_str", b',"flows_removed":'), DecimalIntegerValueModelElement("flows_removed"), FixedDataModelElement("rows_checked_str", b',"rows_checked":'), DecimalIntegerValueModelElement("rows_checked"), FixedDataModelElement("rows_skipped_str", b',"rows_skipped":'), DecimalIntegerValueModelElement("rows_skipped"), FixedDataModelElement("rows_empty_str", b',"rows_empty":'), DecimalIntegerValueModelElement("rows_empty"), FixedDataModelElement("rows_busy_str", b',"rows_busy":'), DecimalIntegerValueModelElement("rows_busy"), FixedDataModelElement("rows_maxlen_str", b',"rows_maxlen":'), DecimalIntegerValueModelElement("rows_maxlen"), ]), SequenceModelElement("dns", [ FixedDataModelElement("memuse_str", b'},"dns":{"memuse":'), DecimalIntegerValueModelElement("memuse"), FixedDataModelElement("memcap_state_str", b',"memcap_state":'), DecimalIntegerValueModelElement("memcap_state"), FixedDataModelElement("memcap_global_str", b',"memcap_global":'), DecimalIntegerValueModelElement("memcap_global"), ]), SequenceModelElement("http", [ FixedDataModelElement("memuse_str", b'},"http":{"memuse":'), DecimalIntegerValueModelElement("memuse"), FixedDataModelElement("memcap_str", b',"memcap":'), DecimalIntegerValueModelElement("memcap"), ]), FixedDataModelElement("quote_str", b"}}}") ]), SequenceModelElement("tls", [ FixedDataModelElement("tls_str", b'tls",'), conn, SequenceModelElement("tls", [ FixedDataModelElement("subject_str", b',"tls":{"subject":"'), DelimitedDataModelElement("subject", b'"'), FixedDataModelElement("issuerdn_str", b'","issuerdn":"'), DelimitedDataModelElement("issuerdn", b'"'), FixedDataModelElement("fingerprint_str", b'","fingerprint":"'), DelimitedDataModelElement("fingerprint", b'"'), OptionalMatchModelElement( "sni", SequenceModelElement("sni", [ FixedDataModelElement("sni_str", b'","sni":"'), DelimitedDataModelElement("sni", b'"'), ])), FixedDataModelElement("version_str", b'","version":"'), DelimitedDataModelElement("version", b'"'), FixedDataModelElement("notbefore_str", b'","notbefore":"'), DelimitedDataModelElement("notbefore", b'"'), FixedDataModelElement("notafter_str", b'","notafter":"'), DelimitedDataModelElement("notafter", b'"'), ]), FixedDataModelElement("brack_str", b'"}}') ]), SequenceModelElement("alert", [ FixedDataModelElement("alert_str", b'alert",'), conn, OptionalMatchModelElement( "tx_id", SequenceModelElement("tx_id", [ FixedDataModelElement("tx_id", b',"tx_id":'), DecimalIntegerValueModelElement("tx_id"), ])), SequenceModelElement("alert", [ FixedDataModelElement("action_str", b',"alert":{"action":"'), DelimitedDataModelElement("action", b'"'), FixedDataModelElement("gid_str", b'","gid":'), DecimalIntegerValueModelElement("gid"), FixedDataModelElement("signature_id_str", b',"signature_id":'), DecimalIntegerValueModelElement("signature_id"), FixedDataModelElement("rev_str", b',"rev":'), DecimalIntegerValueModelElement("rev"), FixedDataModelElement("signature_str", b',"signature":"'), DelimitedDataModelElement("signature", b'"'), FixedDataModelElement("category_str", b'","category":"'), DelimitedDataModelElement("category", b'"'), FixedDataModelElement("severity_str", b'","severity":'), DecimalIntegerValueModelElement("severity"), FixedDataModelElement("brack_str", b"}") ]), http, FixedDataModelElement("brack_str", b"}") ]), ]) ]) return model
def test5get_match_element_valid_ipv6_match(self): """ This test case checks the functionality by parsing a real IP-addresses. The numerical representation of the ip address was calculated with the help of https://www.ipaddressguide.com/ipv6-to-decimal. """ ip_addr_dme = IpAddressDataModelElement(self.id_, True) data = b"2001:4860:4860::8888 followed by some text" value = b"2001:4860:4860::8888" number = 42541956123769884636017138956568135816 match_context = DummyMatchContext(data) match_element = ip_addr_dme.get_match_element(self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_, self.path, value, number, None) # full form of IPv6 data = b"fe80:0000:0000:0000:0204:61ff:fe9d:f156." value = b"fe80:0000:0000:0000:0204:61ff:fe9d:f156" number = 338288524927261089654164245681446711638 match_context = DummyMatchContext(data) match_element = ip_addr_dme.get_match_element(self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_, self.path, value, number, None) # drop leading zeroes data = b"fe80:0:0:0:204:61ff:fe9d:f156." value = b"fe80:0:0:0:204:61ff:fe9d:f156" match_context = DummyMatchContext(data) match_element = ip_addr_dme.get_match_element(self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_, self.path, value, number, None) # collapse multiple zeroes to :: in the IPv6 address data = b"fe80::204:61ff:fe9d:f156 followed by some text" value = b"fe80::204:61ff:fe9d:f156" match_context = DummyMatchContext(data) match_element = ip_addr_dme.get_match_element(self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_, self.path, value, number, None) # localhost data = b"::1 followed by some text" value = b"::1" number = 1 match_context = DummyMatchContext(data) match_element = ip_addr_dme.get_match_element(self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_, self.path, value, number, None) # link-local prefix data = b"fe80:: followed by some text" value = b"fe80::" number = 338288524927261089654018896841347694592 match_context = DummyMatchContext(data) match_element = ip_addr_dme.get_match_element(self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_, self.path, value, number, None) # global unicast prefix data = b"2001:: followed by some text" value = b"2001::" number = 42540488161975842760550356425300246528 match_context = DummyMatchContext(data) match_element = ip_addr_dme.get_match_element(self.path, match_context) self.compare_match_results(data, match_element, match_context, self.id_, self.path, value, number, None)
def get_model(): """Return a model for su session information messages after any standard logging preamble, e.g. from syslog.""" type_children = [ SequenceModelElement("build-stack", [ FixedDataModelElement("s0", b"building new pluginstance stack: \""), DelimitedDataModelElement("stack", b"\""), FixedDataModelElement("s1", b"\"") ]), SequenceModelElement( "nfct-event", [ FixedDataModelElement("s0", b"[DESTROY] ORIG: SRC="), IpAddressDataModelElement("osrcip"), FixedDataModelElement("s1", b" DST="), IpAddressDataModelElement("odstip"), FixedDataModelElement("s2", b" PROTO="), FixedWordlistDataModelElement("proto", [b"TCP", b"UDP"]), FixedDataModelElement("s3", b" SPT="), DecimalIntegerValueModelElement("ospt"), FixedDataModelElement("s4", b" DPT="), DecimalIntegerValueModelElement("odpt"), FixedDataModelElement("s5", b" PKTS="), DecimalIntegerValueModelElement("opkts"), FixedDataModelElement("s6", b" BYTES="), DecimalIntegerValueModelElement("obytes"), FixedDataModelElement("s7", b" , REPLY: SRC="), IpAddressDataModelElement("rsrcip"), FixedDataModelElement("s8", b" DST="), IpAddressDataModelElement("rdstip"), FixedDataModelElement("s9", b" PROTO="), FixedWordlistDataModelElement("rproto", [b"TCP", b"UDP"]), FixedDataModelElement("s10", b" SPT="), DecimalIntegerValueModelElement("rspt"), FixedDataModelElement("s11", b" DPT="), DecimalIntegerValueModelElement("rdpt"), FixedDataModelElement("s12", b" PKTS="), DecimalIntegerValueModelElement("rpkts"), FixedDataModelElement("s13", b" BYTES="), DecimalIntegerValueModelElement("rbytes"), # No additional whitespace from Ubuntu Trusty 14.04 on. OptionalMatchModelElement("tail", FixedDataModelElement("s0", b" ")) ]), FixedDataModelElement("nfct-plugin", b"NFCT plugin working in event mode"), FixedDataModelElement("reopen", b"reopening capture file"), FixedDataModelElement("signal", b"signal received, calling pluginstances"), FixedDataModelElement("uidchange", b"Changing UID / GID"), SequenceModelElement("seq", [ FixedDataModelElement("s0", b"id=\""), DecimalIntegerValueModelElement("id"), FixedDataModelElement("s1", b"\" severity=\""), DelimitedDataModelElement("severity", b"\""), FixedDataModelElement("s2", b"\" sys=\""), DelimitedDataModelElement("sys", b"\""), FixedDataModelElement("s3", b"\" sub=\""), DelimitedDataModelElement("sub", b"\""), FixedDataModelElement("s4", b"\" name=\""), DelimitedDataModelElement("name", b"\""), FixedDataModelElement("s5", b"\" action=\""), DelimitedDataModelElement("action", b"\""), FixedDataModelElement("s6", b"\" fwrule=\""), DelimitedDataModelElement("fwrule", b"\""), FixedDataModelElement("s7", b"\" initf=\""), DelimitedDataModelElement("initf", b"\""), FixedDataModelElement("s8", b"\" srcmac=\""), DelimitedDataModelElement("srcmac", b"\""), FixedDataModelElement("s9", b"\" dstmac=\""), DelimitedDataModelElement("dstmac", b"\""), FixedDataModelElement("s10", b"\" srcip=\""), DelimitedDataModelElement("srcip", b"\""), FixedDataModelElement("s11", b"\" dstip=\""), DelimitedDataModelElement("dstip", b"\""), FixedDataModelElement("s12", b"\" proto=\""), DelimitedDataModelElement("proto", b"\""), FixedDataModelElement("s13", b"\" length=\""), DelimitedDataModelElement("length", b"\""), FixedDataModelElement("s14", b"\" tos=\""), DelimitedDataModelElement("tos", b"\""), FixedDataModelElement("s15", b"\" prec=\""), DelimitedDataModelElement("prec", b"\""), FixedDataModelElement("s16", b"\" ttl=\""), DelimitedDataModelElement("ttl", b"\""), FixedDataModelElement("s17", b"\" srcport=\""), DelimitedDataModelElement("srcport", b"\""), FixedDataModelElement("s18", b"\" dstport=\""), DelimitedDataModelElement("dstport", b"\""), FixedDataModelElement("s19", b"\" tcpflags=\""), DelimitedDataModelElement("tcpflags", b"\""), FixedDataModelElement("s20", b"\"") ]) ] # Netflow entry model = SequenceModelElement("ulogd", [ FixedDataModelElement("sname", b"ulogd["), DecimalIntegerValueModelElement("pid"), FixedDataModelElement("s0", b"]: "), FirstMatchModelElement("msg", type_children) ]) return model
def get_model(): """Return a model to parse Exim logs from the AIT-LDS.""" alphabet = b"!'#$%&\"()*+,-./0123456789:;<>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\\^_`abcdefghijklmnopqrstuvwxyz{|}~=[]" model = SequenceModelElement("model", [ DateTimeModelElement("time", b"%Y-%m-%d %H:%M:%S"), FixedDataModelElement("sp", b" "), FirstMatchModelElement("fm", [ SequenceModelElement("start", [ FixedDataModelElement("start", b"Start queue run: pid="), DecimalIntegerValueModelElement("pid"), ]), SequenceModelElement("end", [ FixedDataModelElement("end", b"End queue run: pid="), DecimalIntegerValueModelElement("pid"), ]), SequenceModelElement("no_host_found", [ FixedDataModelElement("no_host_found_str", b"no host name found for IP address "), IpAddressDataModelElement("ip"), ]), SequenceModelElement("vrfy_failed", [ FixedDataModelElement("vrfy_failed_str", b"VRFY failed for "), DelimitedDataModelElement("mail", b" "), FixedDataModelElement("h_str", b" H="), DelimitedDataModelElement("h", b" "), FixedDataModelElement("sp1", b" ["), IpAddressDataModelElement("ip"), FixedDataModelElement("sp2", b"]") ]), SequenceModelElement("mail", [ DelimitedDataModelElement("id", b" "), FirstMatchModelElement("dir", [ SequenceModelElement("dir_in", [ FixedDataModelElement("in", b" <= "), FirstMatchModelElement("fm", [ SequenceModelElement("seq1", [ FixedDataModelElement("brack", b"<> "), FirstMatchModelElement("fm", [ SequenceModelElement("r", [ FixedDataModelElement("r_str", b"R="), DelimitedDataModelElement("r", b" "), FixedDataModelElement("u_str", b" U="), DelimitedDataModelElement("u", b" "), ]), SequenceModelElement("h", [ FixedDataModelElement("h_str", b"H="), DelimitedDataModelElement("h", b" "), FixedDataModelElement("sp1", b" ["), IpAddressDataModelElement("ip"), FixedDataModelElement("sp1", b"]"), ]) ]), FixedDataModelElement("sp2", b" P="), DelimitedDataModelElement("p", b" "), FixedDataModelElement("sp2", b" S="), DecimalIntegerValueModelElement("s"), ]), SequenceModelElement("seq2", [ DelimitedDataModelElement("mail", b" "), FixedDataModelElement("user_str", b" U="), DelimitedDataModelElement("user", b" "), FixedDataModelElement("p_str", b" P="), DelimitedDataModelElement("p", b" "), FixedDataModelElement("s_str", b" S="), DecimalIntegerValueModelElement("s"), OptionalMatchModelElement( "id", SequenceModelElement("id", [ FixedDataModelElement("id_str", b" id="), AnyByteDataModelElement("id") ]) ) ]) ]) ]), SequenceModelElement("dir_out", [ FixedDataModelElement("in", b" => "), DelimitedDataModelElement("name", b" "), FixedDataModelElement("sp1", b" "), OptionalMatchModelElement( "mail_opt", SequenceModelElement("mail", [ FixedDataModelElement("brack1", b"("), DelimitedDataModelElement("brack_mail", b")"), FixedDataModelElement("brack2", b") "), ])), FixedDataModelElement("sp2", b"<"), DelimitedDataModelElement("mail", b">"), FixedDataModelElement("r_str", b"> R="), DelimitedDataModelElement("r", b" "), FixedDataModelElement("t_str", b" T="), VariableByteDataModelElement("t", alphabet), ]), SequenceModelElement("aster", [ FixedDataModelElement("aster", b" ** "), DelimitedDataModelElement("command", b" "), FixedDataModelElement("headers_str", b' Too many "Received" headers - suspected mail loop')]), FixedDataModelElement("completed", b" Completed"), FixedDataModelElement("frozen", b" Message is frozen"), FixedDataModelElement("frozen", b" Frozen (delivery error message)") ]) ])])]) return model
def build_analysis_pipeline(analysis_context): """ Define the function to create pipeline for parsing the log data. It has also to define an AtomizerFactory to instruct aminer how to process incoming data streams to create log atoms from them. """ # Build the parsing model: service_children_disk_report = [ FixedDataModelElement( 'Space', b' Current Disk Data is: Filesystem Type Size Used Avail Use%' ), DelimitedDataModelElement('Data', b'%'), AnyByteDataModelElement('Rest') ] service_children_login_details = [ FixedDataModelElement('User', b'User '), DelimitedDataModelElement('Username', b' '), FixedWordlistDataModelElement('Status', [b' logged in', b' logged out']), OptionalMatchModelElement( 'PastTime', SequenceModelElement('Time', [ FixedDataModelElement('Blank', b' '), DecimalIntegerValueModelElement('Minutes'), FixedDataModelElement('Ago', b' minutes ago.') ])) ] service_children_cron_job = [ DateTimeModelElement('DTM', b'%Y-%m-%d %H:%M:%S'), FixedDataModelElement('UNameSpace1', b' '), DelimitedDataModelElement('UName', b' '), FixedDataModelElement('UNameSpace2', b' '), DelimitedDataModelElement('User', b' '), FixedDataModelElement('Cron', b' cron['), DecimalIntegerValueModelElement('JobNumber'), FixedDataModelElement('Details', b']: Job `cron.daily` started.') ] service_children_random_time = [ FixedDataModelElement('Space', b'Random: '), DecimalIntegerValueModelElement('Random') ] service_children_sensors = [ SequenceModelElement('CPUTemp', [ FixedDataModelElement('FixedTemp', b'CPU Temp: '), DecimalIntegerValueModelElement('Temp'), FixedDataModelElement('Degrees', b'\xc2\xb0C') ]), FixedDataModelElement('Space1', b', '), SequenceModelElement('CPUWorkload', [ FixedDataModelElement('FixedWorkload', b'CPUWorkload: '), DecimalIntegerValueModelElement('Workload'), FixedDataModelElement('Percent', b'%') ]), FixedDataModelElement('Space2', b', '), DateTimeModelElement('DTM', b'%Y-%m-%d %H:%M:%S') ] service_children_user_ip_address = [ FixedDataModelElement('User', b'User '), DelimitedDataModelElement('Username', b' '), FixedDataModelElement('Action', b' changed IP address to '), IpAddressDataModelElement('IP') ] service_children_cron_job_announcement = [ DateTimeModelElement('DTM', b'%Y-%m-%d %H:%M:%S'), FixedDataModelElement('Space', b' '), DelimitedDataModelElement('UName', b' '), FixedDataModelElement('Cron', b' cron['), DecimalIntegerValueModelElement('JobNumber'), FixedDataModelElement('Run', b']: Will run job `'), FixedWordlistDataModelElement( 'CronType', [b'cron.daily', b'cron.hourly', b'cron.monthly', b'cron.weekly']), FixedDataModelElement('StartTime', b'\' in 5 min.') ] service_children_cron_job_execution = [ DateTimeModelElement('DTM', b'%Y-%m-%d %H:%M:%S'), FixedDataModelElement('Space1', b' '), DelimitedDataModelElement('UName', b' '), FixedDataModelElement('Cron', b' cron['), DecimalIntegerValueModelElement('JobNumber'), FixedDataModelElement('Job', b']: Job `'), FixedWordlistDataModelElement( 'CronType', [b'cron.daily', b'cron.hourly', b'cron.monthly', b'cron.weekly']), FixedDataModelElement('Started', b'\' started') ] parsing_model = FirstMatchModelElement('model', [ SequenceModelElement('CronAnnouncement', service_children_cron_job_announcement), SequenceModelElement('CronExecution', service_children_cron_job_execution), SequenceModelElement('DailyCron', service_children_cron_job), SequenceModelElement('DiskReport', service_children_disk_report), SequenceModelElement('LoginDetails', service_children_login_details), DecimalIntegerValueModelElement('Random'), SequenceModelElement('RandomTime', service_children_random_time), SequenceModelElement('Sensors', service_children_sensors), SequenceModelElement('IPAddresses', service_children_user_ip_address) ]) # Some generic imports. from aminer.analysis import AtomFilters # Create all global handler lists here and append the real handlers later on. # Use this filter to distribute all atoms to the analysis handlers. atom_filters = AtomFilters.SubhandlerFilter(None) analysis_context.register_component(atom_filters, component_name="AtomFilter") from aminer.analysis.TimestampCorrectionFilters import SimpleMonotonicTimestampAdjust simple_monotonic_timestamp_adjust = SimpleMonotonicTimestampAdjust( [atom_filters]) analysis_context.register_component( simple_monotonic_timestamp_adjust, component_name="SimpleMonotonicTimestampAdjust") from aminer.events.StreamPrinterEventHandler import StreamPrinterEventHandler stream_printer_event_handler = StreamPrinterEventHandler(analysis_context) from aminer.events.Utils import VolatileLogarithmicBackoffEventHistory volatile_logarithmic_backoff_event_history = VolatileLogarithmicBackoffEventHistory( 100) anomaly_event_handlers = [ stream_printer_event_handler, volatile_logarithmic_backoff_event_history ] analysis_context.register_component( volatile_logarithmic_backoff_event_history, component_name="VolatileLogarithmicBackoffEventHistory") # Now define the AtomizerFactory using the model. A simple line based one is usually sufficient. from aminer.input.SimpleByteStreamLineAtomizerFactory import SimpleByteStreamLineAtomizerFactory analysis_context.atomizer_factory = SimpleByteStreamLineAtomizerFactory( parsing_model, [simple_monotonic_timestamp_adjust], anomaly_event_handlers) # Just report all unparsed atoms to the event handlers. from aminer.input.SimpleUnparsedAtomHandler import SimpleUnparsedAtomHandler simple_unparsed_atom_handler = SimpleUnparsedAtomHandler( anomaly_event_handlers) atom_filters.add_handler(simple_unparsed_atom_handler, stop_when_handled_flag=True) analysis_context.register_component(simple_unparsed_atom_handler, component_name="UnparsedHandler") from aminer.analysis.TimestampsUnsortedDetector import TimestampsUnsortedDetector timestamps_unsorted_detector = TimestampsUnsortedDetector( analysis_context.aminer_config, anomaly_event_handlers) atom_filters.add_handler(timestamps_unsorted_detector) analysis_context.register_component( timestamps_unsorted_detector, component_name="TimestampsUnsortedDetector") from aminer.analysis import Rules from aminer.analysis.AllowlistViolationDetector import AllowlistViolationDetector allowlist_rules = [ Rules.OrMatchRule([ Rules.AndMatchRule([ Rules.PathExistsMatchRule( '/model/LoginDetails/PastTime/Time/Minutes'), Rules.NegationMatchRule( Rules.ValueMatchRule('/model/LoginDetails/Username', b'root')) ]), Rules.AndMatchRule([ Rules.NegationMatchRule( Rules.PathExistsMatchRule( '/model/LoginDetails/PastTime/Time/Minutes')), Rules.PathExistsMatchRule('/model/LoginDetails') ]), Rules.NegationMatchRule( Rules.PathExistsMatchRule('/model/LoginDetails')) ]) ] # This rule list should trigger, when the line does not look like: User root (logged in, logged out) # or User 'username' (logged in, logged out) x minutes ago. allowlist_violation_detector = AllowlistViolationDetector( analysis_context.aminer_config, allowlist_rules, anomaly_event_handlers) analysis_context.register_component(allowlist_violation_detector, component_name="Allowlist") atom_filters.add_handler(allowlist_violation_detector) from aminer.analysis.ParserCount import ParserCount parser_count = ParserCount(analysis_context.aminer_config, None, anomaly_event_handlers, 10) analysis_context.register_component(parser_count, component_name="ParserCount") atom_filters.add_handler(parser_count) from aminer.analysis.EventCorrelationDetector import EventCorrelationDetector ecd = EventCorrelationDetector(analysis_context.aminer_config, anomaly_event_handlers, check_rules_flag=True, hypothesis_max_delta_time=1.0, auto_include_flag=True) analysis_context.register_component( ecd, component_name="EventCorrelationDetector") atom_filters.add_handler(ecd) from aminer.analysis.NewMatchPathDetector import NewMatchPathDetector new_match_path_detector = NewMatchPathDetector( analysis_context.aminer_config, anomaly_event_handlers, auto_include_flag=True) analysis_context.register_component(new_match_path_detector, component_name="NewMatchPath") atom_filters.add_handler(new_match_path_detector) def tuple_transformation_function(match_value_list): """Only allow output of the EnhancedNewMatchPathValueComboDetector after every 10000th element.""" extra_data = enhanced_new_match_path_value_combo_detector.known_values_dict.get( tuple(match_value_list)) if extra_data is not None: mod = 10000 if (extra_data[2] + 1) % mod == 0: enhanced_new_match_path_value_combo_detector.auto_include_flag = False else: enhanced_new_match_path_value_combo_detector.auto_include_flag = True return match_value_list from aminer.analysis.EnhancedNewMatchPathValueComboDetector import EnhancedNewMatchPathValueComboDetector enhanced_new_match_path_value_combo_detector = EnhancedNewMatchPathValueComboDetector( analysis_context.aminer_config, ['/model/DailyCron/UName', '/model/DailyCron/JobNumber'], anomaly_event_handlers, auto_include_flag=False, tuple_transformation_function=tuple_transformation_function) analysis_context.register_component( enhanced_new_match_path_value_combo_detector, component_name="EnhancedNewValueCombo") atom_filters.add_handler(enhanced_new_match_path_value_combo_detector) from aminer.analysis.HistogramAnalysis import HistogramAnalysis, LinearNumericBinDefinition, ModuloTimeBinDefinition, \ PathDependentHistogramAnalysis modulo_time_bin_definition = ModuloTimeBinDefinition( 86400, 3600, 0, 1, 24, True) linear_numeric_bin_definition = LinearNumericBinDefinition(50, 5, 20, True) histogram_analysis = HistogramAnalysis( analysis_context.aminer_config, [('/model/RandomTime/Random', modulo_time_bin_definition), ('/model/Random', linear_numeric_bin_definition)], 10, anomaly_event_handlers) analysis_context.register_component(histogram_analysis, component_name="HistogramAnalysis") atom_filters.add_handler(histogram_analysis) path_dependent_histogram_analysis = PathDependentHistogramAnalysis( analysis_context.aminer_config, '/model/RandomTime', modulo_time_bin_definition, 10, anomaly_event_handlers) analysis_context.register_component( path_dependent_histogram_analysis, component_name="PathDependentHistogramAnalysis") atom_filters.add_handler(path_dependent_histogram_analysis) from aminer.analysis.MatchValueAverageChangeDetector import MatchValueAverageChangeDetector match_value_average_change_detector = MatchValueAverageChangeDetector( analysis_context.aminer_config, anomaly_event_handlers, None, ['/model/Random'], 100, 10) analysis_context.register_component( match_value_average_change_detector, component_name="MatchValueAverageChange") atom_filters.add_handler(match_value_average_change_detector) import sys from aminer.analysis.MatchValueStreamWriter import MatchValueStreamWriter match_value_stream_writer = MatchValueStreamWriter(sys.stdout, [ '/model/Sensors/CPUTemp', '/model/Sensors/CPUWorkload', '/model/Sensors/DTM' ], b';', b'') analysis_context.register_component( match_value_stream_writer, component_name="MatchValueStreamWriter") atom_filters.add_handler(match_value_stream_writer) from aminer.analysis.NewMatchPathValueComboDetector import NewMatchPathValueComboDetector new_match_path_value_combo_detector = NewMatchPathValueComboDetector( analysis_context.aminer_config, ['/model/IPAddresses/Username', '/model/IPAddresses/IP'], anomaly_event_handlers, auto_include_flag=False) analysis_context.register_component( new_match_path_value_combo_detector, component_name="NewMatchPathValueCombo") atom_filters.add_handler(new_match_path_value_combo_detector) from aminer.analysis.NewMatchIdValueComboDetector import NewMatchIdValueComboDetector new_match_id_value_combo_detector = NewMatchIdValueComboDetector( analysis_context.aminer_config, ['/model/type/path/name', '/model/type/syscall/syscall'], anomaly_event_handlers, id_path_list=['/model/type/path/id', '/model/type/syscall/id'], min_allowed_time_diff=5, auto_include_flag=True, allow_missing_values_flag=True, output_log_line=True) analysis_context.register_component( new_match_id_value_combo_detector, component_name="NewMatchIdValueComboDetector") atom_filters.add_handler(new_match_id_value_combo_detector) from aminer.analysis.NewMatchPathValueDetector import NewMatchPathValueDetector new_match_path_value_detector = NewMatchPathValueDetector( analysis_context.aminer_config, ['/model/DailyCron/Job Number', '/model/IPAddresses/Username'], anomaly_event_handlers, auto_include_flag=False) analysis_context.register_component(new_match_path_value_detector, component_name="NewMatchPathValue") atom_filters.add_handler(new_match_path_value_detector) from aminer.analysis.MissingMatchPathValueDetector import MissingMatchPathValueDetector missing_match_path_value_detector = MissingMatchPathValueDetector( analysis_context.aminer_config, ['/model/DiskReport/Space'], anomaly_event_handlers, auto_include_flag=False, default_interval=2, realert_interval=5) analysis_context.register_component(missing_match_path_value_detector, component_name="MissingMatch") atom_filters.add_handler(missing_match_path_value_detector) from aminer.analysis.TimeCorrelationDetector import TimeCorrelationDetector time_correlation_detector = TimeCorrelationDetector( analysis_context.aminer_config, anomaly_event_handlers, 2, min_rule_attributes=1, max_rule_attributes=5, record_count_before_event=70000, output_log_line=True) analysis_context.register_component( time_correlation_detector, component_name="TimeCorrelationDetector") atom_filters.add_handler(time_correlation_detector) from aminer.analysis.TimeCorrelationViolationDetector import TimeCorrelationViolationDetector, CorrelationRule, EventClassSelector cron_job_announcement = CorrelationRule( 'CronJobAnnouncement', 5, 6, max_artefacts_a_for_single_b=1, artefact_match_parameters=[('/model/CronAnnouncement/JobNumber', '/model/CronExecution/JobNumber')]) a_class_selector = EventClassSelector('Announcement', [cron_job_announcement], None) b_class_selector = EventClassSelector('Execution', None, [cron_job_announcement]) rules = [ Rules.PathExistsMatchRule('/model/CronAnnouncement/Run', a_class_selector), Rules.PathExistsMatchRule('/model/CronExecution/Job', b_class_selector) ] time_correlation_violation_detector = TimeCorrelationViolationDetector( analysis_context.aminer_config, rules, anomaly_event_handlers) analysis_context.register_component( time_correlation_violation_detector, component_name="TimeCorrelationViolationDetector") atom_filters.add_handler(time_correlation_violation_detector) from aminer.events.DefaultMailNotificationEventHandler import DefaultMailNotificationEventHandler if DefaultMailNotificationEventHandler.CONFIG_KEY_MAIL_TARGET_ADDRESS in analysis_context.aminer_config.config_properties: mail_notification_handler = DefaultMailNotificationEventHandler( analysis_context) analysis_context.register_component(mail_notification_handler, component_name="MailHandler") anomaly_event_handlers.append(mail_notification_handler)
def get_model(): """Return a model to parse a audispd message logged via syslog after any standard logging preamble, e.g. from syslog.""" class ExecArgumentDataModelElement(): """This is a helper class for parsing the (encoded) exec argument strings found within audit logs.""" def __init__(self, element_id): self.element_id = element_id @staticmethod def get_child_elements(): """Get the children of this element (none).""" return None def get_match_element(self, path, match_context): """ Find the maximum number of bytes belonging to an exec argument. @return a match when at least two bytes were found including the delimiters. """ data = match_context.match_data match_len = 0 match_value = b'' if data[0] == ord(b'"'): match_len = data.find(b'"', 1) if match_len == -1: return None match_value = data[1:match_len] match_len += 1 elif data.startswith(b'(null)'): match_len = 6 match_value = None else: # Must be upper case hex encoded: next_value = -1 for d_byte in data: if 0x30 <= d_byte <= 0x39: d_byte -= 0x30 elif 0x41 <= d_byte <= 0x46: d_byte -= 0x37 else: break if next_value == -1: next_value = (d_byte << 4) else: match_value += bytearray(((next_value | d_byte), )) next_value = -1 match_len += 1 if next_value != -1: return None match_data = data[:match_len] match_context.update(match_data) return MatchElement("%s/%s" % (path, self.element_id), match_data, match_value, None) pam_status_word_list = FixedWordlistDataModelElement( 'status', [b'failed', b'success']) pid = b' pid=' uid = b' uid=' auid = b' auid=' ses = b' ses=' exe = b' exe="' hostname = b'" hostname=' addr = b' addr=' terminal = b' terminal=' success = b' res=success\'' res = b' res=' exe1 = b'" exe="' type_branches = { 'ADD_USER': SequenceModelElement('adduser', [ FixedDataModelElement('s0', pid), DecimalIntegerValueModelElement('pid'), FixedDataModelElement('s1', uid), DecimalIntegerValueModelElement('uid'), FixedDataModelElement('s2', auid), DecimalIntegerValueModelElement('auid'), FixedDataModelElement('s3', ses), DecimalIntegerValueModelElement('ses'), FixedDataModelElement('s4', b' msg=\'op=adding user id='), DecimalIntegerValueModelElement('newuserid'), FixedDataModelElement('s5', exe), DelimitedDataModelElement('exec', b'"'), FixedDataModelElement('s6', hostname), DelimitedDataModelElement('clientname', b' '), FixedDataModelElement('s7', addr), DelimitedDataModelElement('clientip', b' '), FixedDataModelElement('s8', terminal), WhiteSpaceLimitedDataModelElement('terminal'), FixedDataModelElement('s9', success) ]), 'BPRM_FCAPS': SequenceModelElement('bprmfcaps', [ FixedDataModelElement('s0', b' fver=0 fp='), HexStringModelElement('fp'), FixedDataModelElement('s1', b' fi='), HexStringModelElement('fi'), FixedDataModelElement('s2', b' fe='), HexStringModelElement('fe'), FixedDataModelElement('s3', b' old_pp='), HexStringModelElement('pp-old'), FixedDataModelElement('s4', b' old_pi='), HexStringModelElement('pi-old'), FixedDataModelElement('s5', b' old_pe='), HexStringModelElement('pe-old'), FixedDataModelElement('s6', b' new_pp='), HexStringModelElement('pp-new'), FixedDataModelElement('s7', b' new_pi='), HexStringModelElement('pi-new'), FixedDataModelElement('s8', b' new_pe='), HexStringModelElement('pe-new') ]), 'CONFIG_CHANGE': SequenceModelElement('conf-change', [ FixedDataModelElement('s0', auid), DecimalIntegerValueModelElement('auid'), FixedDataModelElement('s1', ses), DecimalIntegerValueModelElement('ses'), FixedDataModelElement('s2', b' op="add rule" key=(null) list='), DecimalIntegerValueModelElement('list'), FixedDataModelElement('s3', res), DecimalIntegerValueModelElement('result') ]), 'CRED_ACQ': SequenceModelElement('credacq', [ FixedDataModelElement('s0', pid), DecimalIntegerValueModelElement('pid'), FixedDataModelElement('s1', uid), DecimalIntegerValueModelElement('uid'), FixedDataModelElement('s2', auid), DecimalIntegerValueModelElement('auid'), FixedDataModelElement('s3', ses), DecimalIntegerValueModelElement('ses'), FixedDataModelElement('s4', b' msg=\'op=PAM:setcred acct="'), DelimitedDataModelElement('username', b'"'), FixedDataModelElement('s5', exe1), DelimitedDataModelElement('exec', b'"'), FixedDataModelElement('s6', hostname), DelimitedDataModelElement('clientname', b' '), FixedDataModelElement('s7', addr), DelimitedDataModelElement('clientip', b' '), FixedDataModelElement('s8', terminal), WhiteSpaceLimitedDataModelElement('terminal'), FixedDataModelElement('s9', success) ]), 'CRED_DISP': SequenceModelElement('creddisp', [ FixedDataModelElement('s0', pid), DecimalIntegerValueModelElement('pid'), FixedDataModelElement('s1', uid), DecimalIntegerValueModelElement('uid'), FixedDataModelElement('s2', auid), DecimalIntegerValueModelElement('auid'), FixedDataModelElement('s3', ses), DecimalIntegerValueModelElement('ses'), FixedDataModelElement('s4', b' msg=\'op=PAM:setcred acct="'), DelimitedDataModelElement('username', b'"'), FixedDataModelElement('s5', exe1), DelimitedDataModelElement('exec', b'"'), FixedDataModelElement('s6', hostname), DelimitedDataModelElement('clientname', b' '), FixedDataModelElement('s7', addr), DelimitedDataModelElement('clientip', b' '), FixedDataModelElement('s8', terminal), WhiteSpaceLimitedDataModelElement('terminal'), FixedDataModelElement('s9', success) ]), 'CRED_REFR': SequenceModelElement('creddisp', [ FixedDataModelElement('s0', pid), DecimalIntegerValueModelElement('pid'), FixedDataModelElement('s1', uid), DecimalIntegerValueModelElement('uid'), FixedDataModelElement('s2', auid), DecimalIntegerValueModelElement('auid'), FixedDataModelElement('s3', ses), DecimalIntegerValueModelElement('ses'), FixedDataModelElement( 's4', b' msg=\'op=PAM:setcred acct="root" exe="/usr/sbin/sshd" hostname=' ), IpAddressDataModelElement('clientname'), FixedDataModelElement('s5', addr), IpAddressDataModelElement('clientip'), FixedDataModelElement('s6', b' terminal=ssh res=success\'') ]), 'CWD': SequenceModelElement('cwd', [ FixedDataModelElement('s0', b' cwd='), ExecArgumentDataModelElement('cwd') ]), 'EOE': OptionalMatchModelElement('eoe', FixedDataModelElement('s0', b'')) } # We need a type branch here also, but there is no additional # data in EOE records after Ubuntu Trusty any more. exec_arg_model = SequenceModelElement('execarg', [ FixedDataModelElement('s0', b' a'), DecimalIntegerValueModelElement('argn'), FixedDataModelElement('s1', b'='), ExecArgumentDataModelElement('argval') ]) type_branches['EXECVE'] = SequenceModelElement('execve', [ FixedDataModelElement('s0', b' argc='), DecimalIntegerValueModelElement('argc'), RepeatedElementDataModelElement('arg', exec_arg_model) ]) type_branches['FD_PAIR'] = SequenceModelElement('fdpair', [ FixedDataModelElement('s0', b' fd0='), DecimalIntegerValueModelElement('fd0'), FixedDataModelElement('s1', b' fd1='), DecimalIntegerValueModelElement('fd1') ]) # This message differs on Ubuntu 32/64 bit variants. type_branches['LOGIN'] = SequenceModelElement('login', [ FixedDataModelElement('s0', pid), DecimalIntegerValueModelElement('pid'), FixedDataModelElement('s1', uid), DecimalIntegerValueModelElement('uid'), FixedWordlistDataModelElement('s2', [b' old auid=', b' old-auid=']), DecimalIntegerValueModelElement('auid-old'), FixedWordlistDataModelElement('s3', [b' new auid=', auid]), DecimalIntegerValueModelElement('auid-new'), FixedWordlistDataModelElement('s4', [b' old ses=', b' old-ses=']), DecimalIntegerValueModelElement('ses-old'), FixedWordlistDataModelElement('s5', [b' new ses=', ses]), DecimalIntegerValueModelElement('ses-new'), FixedDataModelElement('s6', res), DecimalIntegerValueModelElement('result') ]) inode_info_model_element = SequenceModelElement( 'inodeinfo', [ FixedDataModelElement('s0', b' inode='), DecimalIntegerValueModelElement('inode'), FixedDataModelElement('s1', b' dev='), # A special major/minor device element could be better here. VariableByteDataModelElement('dev', b'0123456789abcdef:'), FixedDataModelElement('s2', b' mode='), # is octal DecimalIntegerValueModelElement('mode'), FixedDataModelElement('s3', b' ouid='), DecimalIntegerValueModelElement('ouid'), FixedDataModelElement('s4', b' ogid='), DecimalIntegerValueModelElement('ogid'), FixedDataModelElement('s5', b' rdev='), # A special major/minor device element could be better here (see above). VariableByteDataModelElement('rdev', b'0123456789abcdef:'), FixedDataModelElement('s6', b' nametype=') ]) type_branches['NETFILTER_CFG'] = SequenceModelElement( 'conf-change', [ FixedDataModelElement('s0', b' table='), FixedWordlistDataModelElement('table', [b'filter', b'mangle', b'nat']), FixedDataModelElement('s1', b' family='), DecimalIntegerValueModelElement('family'), FixedDataModelElement('s2', b' entries='), DecimalIntegerValueModelElement('entries') ]) type_branches['OBJ_PID'] = SequenceModelElement('objpid', [ FixedDataModelElement('s0', b' opid='), DecimalIntegerValueModelElement('opid'), FixedDataModelElement('s1', b' oauid='), DecimalIntegerValueModelElement( 'oauid', value_sign_type=DecimalIntegerValueModelElement.SIGN_TYPE_OPTIONAL ), FixedDataModelElement('s2', b' ouid='), DecimalIntegerValueModelElement('ouid'), FixedDataModelElement('s3', b' oses='), DecimalIntegerValueModelElement( 'oses', value_sign_type=DecimalIntegerValueModelElement.SIGN_TYPE_OPTIONAL ), FixedDataModelElement('s4', b' ocomm='), ExecArgumentDataModelElement('ocomm') ]) type_branches['PATH'] = SequenceModelElement('path', [ FixedDataModelElement('s0', b' item='), DecimalIntegerValueModelElement('item'), FixedDataModelElement('s1', b' name='), ExecArgumentDataModelElement('name'), FirstMatchModelElement('fsinfo', [ inode_info_model_element, FixedDataModelElement('noinfo', b' nametype=') ]), FixedWordlistDataModelElement( 'nametype', [b'CREATE', b'DELETE', b'NORMAL', b'PARENT', b'UNKNOWN']) ]) type_branches['PROCTITLE'] = SequenceModelElement('proctitle', [ FixedDataModelElement('s1', b' proctitle='), ExecArgumentDataModelElement('proctitle') ]) type_branches['SERVICE_START'] = SequenceModelElement( 'service', [ FixedDataModelElement('s0', pid), DecimalIntegerValueModelElement('pid'), FixedDataModelElement('s1', uid), DecimalIntegerValueModelElement('uid'), FixedDataModelElement('s2', auid), DecimalIntegerValueModelElement('auid'), FixedDataModelElement('s3', ses), DecimalIntegerValueModelElement('ses'), FixedDataModelElement('s4', b' msg=\'unit='), DelimitedDataModelElement('unit', b' '), FixedDataModelElement('s5', b' comm="systemd" exe="'), DelimitedDataModelElement('exec', b'"'), FixedDataModelElement('s6', hostname), DelimitedDataModelElement('clientname', b' '), FixedDataModelElement('s7', addr), DelimitedDataModelElement('clientip', b' '), FixedDataModelElement('s8', terminal), WhiteSpaceLimitedDataModelElement('terminal'), FixedDataModelElement('s9', res), pam_status_word_list, FixedDataModelElement('s10', b'\'') ]) type_branches['SERVICE_STOP'] = type_branches['SERVICE_START'] type_branches['SOCKADDR'] = SequenceModelElement('sockaddr', [ FixedDataModelElement('s0', b' saddr='), HexStringModelElement('sockaddr', upper_case=True) ]) type_branches['SYSCALL'] = SequenceModelElement('syscall', [ FixedDataModelElement('s0', b' arch='), HexStringModelElement('arch'), FixedDataModelElement('s1', b' syscall='), DecimalIntegerValueModelElement('syscall'), OptionalMatchModelElement( 'personality', SequenceModelElement('pseq', [ FixedDataModelElement('s0', b' per='), DecimalIntegerValueModelElement('personality') ])), OptionalMatchModelElement( 'result', SequenceModelElement('rseq', [ FixedDataModelElement('s2', b' success='), FixedWordlistDataModelElement('succes', [b'no', b'yes']), FixedDataModelElement('s3', b' exit='), DecimalIntegerValueModelElement( 'exit', value_sign_type=DecimalIntegerValueModelElement. SIGN_TYPE_OPTIONAL) ])), FixedDataModelElement('s4', b' a0='), HexStringModelElement('arg0'), FixedDataModelElement('s5', b' a1='), HexStringModelElement('arg1'), FixedDataModelElement('s6', b' a2='), HexStringModelElement('arg2'), FixedDataModelElement('s7', b' a3='), HexStringModelElement('arg3'), FixedDataModelElement('s8', b' items='), DecimalIntegerValueModelElement('items'), FixedDataModelElement('s9', b' ppid='), DecimalIntegerValueModelElement('ppid'), FixedDataModelElement('s10', pid), DecimalIntegerValueModelElement('pid'), FixedDataModelElement('s11', auid), DecimalIntegerValueModelElement('auid'), FixedDataModelElement('s12', uid), DecimalIntegerValueModelElement('uid'), FixedDataModelElement('s13', b' gid='), DecimalIntegerValueModelElement('gid'), FixedDataModelElement('s14', b' euid='), DecimalIntegerValueModelElement('euid'), FixedDataModelElement('s15', b' suid='), DecimalIntegerValueModelElement('suid'), FixedDataModelElement('s16', b' fsuid='), DecimalIntegerValueModelElement('fsuid'), FixedDataModelElement('s17', b' egid='), DecimalIntegerValueModelElement('egid'), FixedDataModelElement('s18', b' sgid='), DecimalIntegerValueModelElement('sgid'), FixedDataModelElement('s19', b' fsgid='), DecimalIntegerValueModelElement('fsgid'), FixedDataModelElement('s20', b' tty='), DelimitedDataModelElement('tty', b' '), FixedDataModelElement('s21', ses), DecimalIntegerValueModelElement('sesid'), FixedDataModelElement('s22', b' comm='), ExecArgumentDataModelElement('command'), FixedDataModelElement('s23', exe), DelimitedDataModelElement('executable', b'"'), FixedDataModelElement('s24', b'" key='), AnyByteDataModelElement('key') ]) # The UNKNOWN type is used then audispd does not know the type of the event, usually because the kernel is more recent than # audispd, thus emiting yet unknown event types. # * type=1327: procitle: see https://www.redhat.com/archives/linux-audit/2014-February/msg00047.html type_branches['UNKNOWN[1327]'] = SequenceModelElement( 'unknown-proctitle', [ FixedDataModelElement('s0', b' proctitle='), ExecArgumentDataModelElement('proctitle') ]) type_branches['USER_ACCT'] = SequenceModelElement('useracct', [ FixedDataModelElement('s0', pid), DecimalIntegerValueModelElement('pid'), FixedDataModelElement('s1', uid), DecimalIntegerValueModelElement('uid'), FixedDataModelElement('s2', auid), DecimalIntegerValueModelElement('auid'), FixedDataModelElement('s3', ses), DecimalIntegerValueModelElement('ses'), FixedDataModelElement('s4', b' msg=\'op=PAM:accounting acct="'), DelimitedDataModelElement('username', b'"'), FixedDataModelElement('s5', exe1), DelimitedDataModelElement('exec', b'"'), FixedDataModelElement('s6', hostname), DelimitedDataModelElement('clientname', b' '), FixedDataModelElement('s7', addr), DelimitedDataModelElement('clientip', b' '), FixedDataModelElement('s8', terminal), WhiteSpaceLimitedDataModelElement('terminal'), FixedDataModelElement('s9', success) ]) type_branches['USER_AUTH'] = SequenceModelElement('userauth', [ FixedDataModelElement('s0', pid), DecimalIntegerValueModelElement('pid'), FixedDataModelElement('s1', uid), DecimalIntegerValueModelElement('uid'), FixedDataModelElement('s2', auid), DecimalIntegerValueModelElement('auid'), FixedDataModelElement('s3', ses), DecimalIntegerValueModelElement('ses'), FixedDataModelElement('s4', b' msg=\'op=PAM:authentication acct="'), DelimitedDataModelElement('username', b'"'), FixedDataModelElement('s5', exe1), DelimitedDataModelElement('exec', b'"'), FixedDataModelElement('s6', hostname), DelimitedDataModelElement('clientname', b' '), FixedDataModelElement('s7', addr), DelimitedDataModelElement('clientip', b' '), FixedDataModelElement('s8', terminal), WhiteSpaceLimitedDataModelElement('terminal'), FixedDataModelElement('s9', success) ]) type_branches['USER_START'] = SequenceModelElement('userstart', [ FixedDataModelElement('s0', pid), DecimalIntegerValueModelElement('pid'), FixedDataModelElement('s1', uid), DecimalIntegerValueModelElement('uid'), FixedDataModelElement('s2', auid), DecimalIntegerValueModelElement('auid'), FixedDataModelElement('s3', ses), DecimalIntegerValueModelElement('ses'), FixedDataModelElement('s4', b' msg=\'op=PAM:session_open acct="'), DelimitedDataModelElement('username', b'"'), FixedDataModelElement('s5', exe1), DelimitedDataModelElement('exec', b'"'), FixedDataModelElement('s6', hostname), DelimitedDataModelElement('clientname', b' '), FixedDataModelElement('s7', addr), DelimitedDataModelElement('clientip', b' '), FixedDataModelElement('s8', terminal), WhiteSpaceLimitedDataModelElement('terminal'), FixedDataModelElement('s9', success) ]) type_branches['USER_END'] = SequenceModelElement('userend', [ FixedDataModelElement('s0', pid), DecimalIntegerValueModelElement('pid'), FixedDataModelElement('s1', uid), DecimalIntegerValueModelElement('uid'), FixedDataModelElement('s2', auid), DecimalIntegerValueModelElement('auid'), FixedDataModelElement('s3', ses), DecimalIntegerValueModelElement('ses'), FixedDataModelElement('s4', b' msg=\'op=PAM:session_close acct="'), DelimitedDataModelElement('username', b'"'), FixedDataModelElement('s5', exe1), DelimitedDataModelElement('exec', b'"'), FixedDataModelElement('s6', hostname), DelimitedDataModelElement('clientname', b' '), FixedDataModelElement('s7', addr), DelimitedDataModelElement('clientip', b' '), FixedDataModelElement('s8', terminal), WhiteSpaceLimitedDataModelElement('terminal'), FixedDataModelElement('s9', success) ]) type_branches['USER_ERR'] = SequenceModelElement('usererr', [ FixedDataModelElement('s0', pid), DecimalIntegerValueModelElement('pid'), FixedDataModelElement('s1', uid), DecimalIntegerValueModelElement('uid'), FixedDataModelElement('s2', auid), DecimalIntegerValueModelElement('auid'), FixedDataModelElement('s3', ses), DecimalIntegerValueModelElement('ses'), FixedDataModelElement('s4', b' msg=\'op=PAM:bad_ident acct="?" exe="'), DelimitedDataModelElement('exec', b'"'), FixedDataModelElement('s5', hostname), DelimitedDataModelElement('clientname', b' '), FixedDataModelElement('s6', addr), DelimitedDataModelElement('clientip', b' '), FixedDataModelElement('s7', terminal), WhiteSpaceLimitedDataModelElement('terminal'), FixedDataModelElement('s8', b' res=failed\'') ]) type_branches['USER_LOGIN'] = SequenceModelElement('userlogin', [ FixedDataModelElement('s0', pid), DecimalIntegerValueModelElement('pid'), FixedDataModelElement('s1', uid), DecimalIntegerValueModelElement('uid'), FixedDataModelElement('s2', auid), DecimalIntegerValueModelElement('auid'), FixedDataModelElement('s3', ses), DecimalIntegerValueModelElement('ses'), FixedDataModelElement('s4', b' msg=\'op=login '), FirstMatchModelElement('msgtype', [ FixedDataModelElement('loginok', b'id=0'), SequenceModelElement('loginfail', [ FixedDataModelElement('s0', b'acct='), ExecArgumentDataModelElement('account') ]) ]), FixedDataModelElement('s5', exe), DelimitedDataModelElement('exec', b'"'), FixedDataModelElement('s6', hostname), DelimitedDataModelElement('clientname', b' '), FixedDataModelElement('s7', addr), DelimitedDataModelElement('clientip', b' '), FixedDataModelElement('s8', terminal), WhiteSpaceLimitedDataModelElement('terminal'), FixedDataModelElement('s9', res), pam_status_word_list, FixedDataModelElement('s10', b'\'') ]) model = SequenceModelElement('audispd', [ FixedDataModelElement('sname', b'audispd: '), FirstMatchModelElement('msg', [ ElementValueBranchModelElement( 'record', SequenceModelElement('preamble', [ FixedDataModelElement('s0', b'type='), WhiteSpaceLimitedDataModelElement('type'), FixedDataModelElement('s1', b' msg=audit('), DecimalIntegerValueModelElement('time'), FixedDataModelElement('s0', b'.'), DecimalIntegerValueModelElement('ms'), FixedDataModelElement('s1', b':'), DecimalIntegerValueModelElement('seq'), FixedDataModelElement('s2', b'):') ]), 'type', type_branches, default_branch=None), FixedDataModelElement('queue-full', b'queue is full - dropping event') ]) ]) return model
def get_model(): """Return a model to parse Exim logs from the AIT-LDS.""" alphabet = b"!'#$%&\"()*+,-./0123456789:;<>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\\^_`abcdefghijklmnopqrstuvwxyz{|}~=[]" size_str = b" SIZE=" host_str1 = b" host " host_str = b":" + host_str1 status_code421 = b": 421" status_code450 = b": 450 " status_code451 = b": 451 " status_code452 = b": 452 <" status_code550 = b": 550" status_code553 = b": 553 " status_code554 = b": 554 " dtme = DateTimeModelElement("time", b"%Y-%m-%d %H:%M:%S") msg_id = DelimitedDataModelElement("id", b" ") ip = IpAddressDataModelElement("ip") host_ip = IpAddressDataModelElement("host_ip") host = DelimitedDataModelElement("host", b" ") size = DecimalIntegerValueModelElement("size") port = DecimalIntegerValueModelElement("port") h_str = b" H=" h_str1 = b"H=" r_str = b" R=" t_str = b" T=" f_str = b" F=<" a_str = b" A=" u_str = b" U=" p_str = b" P=" s_str = b" S=" x_str = b" X=" c_str = b" C=\"" id_str = b" id=" a = DelimitedDataModelElement("a", b" ") r = DelimitedDataModelElement("r", b" ") t = DelimitedDataModelElement("t", b" ") u = DelimitedDataModelElement("u", b" ") p = DelimitedDataModelElement("p", b" ") h = DelimitedDataModelElement("h", b" ") x = DelimitedDataModelElement("x", b" ") c = DelimitedDataModelElement("c", b'"') s = DecimalIntegerValueModelElement("s") mail_from = DelimitedDataModelElement("mail_from", b" ") smtp_error_from_remote = b"SMTP error from remote mail server after MAIL FROM:<" model = FirstMatchModelElement("model", [ SequenceModelElement("date_seq", [ dtme, FixedDataModelElement("sp", b" "), FirstMatchModelElement("fm", [ SequenceModelElement("start", [ FixedDataModelElement("start", b"Start queue run: pid="), DecimalIntegerValueModelElement("pid"), ]), SequenceModelElement("end", [ FixedDataModelElement("end", b"End queue run: pid="), DecimalIntegerValueModelElement("pid"), ]), SequenceModelElement("no_host_found", [ FixedDataModelElement("no_host_found_str", b"no host name found for IP address "), ip, ]), SequenceModelElement("vrfy_failed", [ FixedDataModelElement("vrfy_failed_str", b"VRFY failed for "), DelimitedDataModelElement("mail", b" "), FixedDataModelElement("h_str", h_str), h, FixedDataModelElement("sp1", b" ["), ip, FixedDataModelElement("sp2", b"]") ]), SequenceModelElement("deferred", [ msg_id, FixedDataModelElement("smtp_error", b" SMTP error from remote mail server after MAIL FROM:<"), DelimitedDataModelElement("from_mail", b">"), FixedDataModelElement("s0", b">" + size_str), size, FixedDataModelElement("s1", host_str), host, FixedDataModelElement("s2", b" ["), host_ip, FixedDataModelElement("status_code", b"]" + status_code421 + b" "), # status code has always to be 421 in this error. DelimitedDataModelElement("version", b" "), FixedDataModelElement("s3", b" ["), DelimitedDataModelElement("domain", b"]"), FirstMatchModelElement("status", [ SequenceModelElement("temporary", [ FixedDataModelElement("s4", b"] Message from ("), IpAddressDataModelElement("from_ip"), FixedDataModelElement("s5", b") temporarily deferred - "), DelimitedDataModelElement("reason_code", b" "), FixedDataModelElement("s6", b" Please refer to "), VariableByteDataModelElement("refer_addr", alphabet) ]), SequenceModelElement("permanent", [ FixedDataModelElement("s4", b"] All messages from "), IpAddressDataModelElement("from_ip"), FixedDataModelElement("s5", b" will be permanently deferred; Retrying will NOT succeed. See "), VariableByteDataModelElement("refer_addr", alphabet) ]) ]), ]), SequenceModelElement("temporary_deferred_new", [ msg_id, FixedDataModelElement("s0", h_str), host, FixedDataModelElement("s1", b" ["), host_ip, FixedDataModelElement("s2", b"]:"), FixedDataModelElement("smtp_error", b" SMTP error from remote mail server after pipelined MAIL FROM:<"), DelimitedDataModelElement("from_mail", b">"), FixedDataModelElement("s3", b">" + size_str), size, FixedDataModelElement("status_code", status_code421 + b" "), # status code has to be 421 in this error message. DelimitedDataModelElement("version", b" "), FixedDataModelElement("s4", b" ["), DelimitedDataModelElement("domain", b"]"), FixedDataModelElement("s5", b"] Messages from "), IpAddressDataModelElement("from_ip"), FixedDataModelElement("s6", b" temporarily deferred due to unexpected volume or user complaints - "), DelimitedDataModelElement("reason_code", b" "), FixedDataModelElement("s7", b" see "), VariableByteDataModelElement("refer_addr", alphabet) ]), SequenceModelElement("rate_limited", [ msg_id, FixedDataModelElement("smtp_error", b" SMTP error from remote mail server after end of data" + host_str), host, FixedDataModelElement("s0", b" ["), host_ip, FixedDataModelElement("status_code", b"]" + status_code421 + b"-"), # status code has to be 421 in this error message. DelimitedDataModelElement("version", b" "), FixedDataModelElement("s1", b" ["), IpAddressDataModelElement("ip"), FixedDataModelElement("s2", b" "), DecimalIntegerValueModelElement("number"), FixedDataModelElement("msg", b"] Our system has detected an unusual rate of\\n421-"), DelimitedDataModelElement("version", b" "), FixedDataModelElement("msg", b" unsolicited mail originating from your IP address. To protect our\\n421-"), DelimitedDataModelElement("version", b" "), FixedDataModelElement("msg", b" users from spam, mail sent from your IP address has been temporarily\\n421-"), DelimitedDataModelElement("version", b" "), FixedDataModelElement("msg", b" rate limited. Please visit\\n421-"), DelimitedDataModelElement("version", b" ", consume_delimiter=True), DelimitedDataModelElement("website", b" "), FixedDataModelElement("msg", b" to review our Bulk\\n421 "), DelimitedDataModelElement("version", b" "), FixedDataModelElement("msg", b" Email Senders Guidelines. "), msg_id, FixedDataModelElement("gsmtp", b" - gsmtp") ]), SequenceModelElement("service_unavailable", [ msg_id, FixedDataModelElement("msg", b" SMTP error from remote mail server after RCPT TO:<"), DelimitedDataModelElement("mail_to", b">"), FixedDataModelElement("s0", b">" + host_str), host, FixedDataModelElement("s1", b" ["), host_ip, FixedDataModelElement("status_code", b"]" + status_code450), DelimitedDataModelElement("version", b" "), FixedDataModelElement("msg", b" Service unavailable") ]), SequenceModelElement("host_unable_to_send", [ msg_id, FixedDataModelElement("s0", b" == "), DelimitedDataModelElement("from_mail", b" "), FixedDataModelElement("s1", r_str), r, FixedDataModelElement("s2", t_str), t, FixedDataModelElement("msg", b" defer (-44): SMTP error from remote mail server after RCPT TO:<"), DelimitedDataModelElement("to_mail", b">"), FixedDataModelElement("s3", b">" + host_str), host, FixedDataModelElement("s4", b" ["), host_ip, FixedDataModelElement("status_code", b"]" + status_code451), FixedDataModelElement("msg", b"Temporary local problem - please try later") ]), SequenceModelElement("uncomplete_sender_verify", [ FixedDataModelElement("s0", h_str1), h, FixedDataModelElement("s1", b" ("), DelimitedDataModelElement("domain", b")"), FixedDataModelElement("s2", b") ["), IpAddressDataModelElement("ipv6", ipv6=True), FixedDataModelElement("s3", b"]:"), port, FirstMatchModelElement("reason", [ SequenceModelElement("permission_denied", [ FixedDataModelElement("msg", b" sender verify defer for <"), DelimitedDataModelElement("from_mail", b">"), FixedDataModelElement("msg", b">: require_files: error for "), DelimitedDataModelElement("required_file", b":"), FixedDataModelElement("msg", b": Permission denied") ]), SequenceModelElement("rejected_rcpt", [ FixedDataModelElement("s0", f_str), DelimitedDataModelElement("from", b">"), FixedDataModelElement("s1", b">" + a_str), DelimitedDataModelElement("a", b" "), FixedDataModelElement("msg", b" temporarily rejected RCPT <"), DelimitedDataModelElement("rcpt", b">"), FixedDataModelElement("msg", b">: Could not complete sender verify") ]) ]) ]), SequenceModelElement("domain_size_limit_exceeded", [ msg_id, FixedDataModelElement("s0", b" =="), DelimitedDataModelElement("mail_to", b" "), FixedDataModelElement("s1", r_str), r, FixedDataModelElement("s2", t_str), t, FixedDataModelElement("msg", b" defer (-44): SMTP error from remote mail server after RCPT TO:<"), DelimitedDataModelElement("mail_to", b">"), FixedDataModelElement("s3", b">" + host_str), host, FixedDataModelElement("s4", b" ["), host_ip, FixedDataModelElement("status_code", b"]" + status_code452), DelimitedDataModelElement("mail_to", b">"), FixedDataModelElement("msg", b"> Domain size limit exceeded") ]), SequenceModelElement("verification_error", [ msg_id, FixedDataModelElement("s0", b" ** "), DelimitedDataModelElement("mail_to", b" "), FixedDataModelElement("s1", r_str), r, FixedDataModelElement("s2", t_str), DelimitedDataModelElement("t", b":"), FirstMatchModelElement("fm", [ SequenceModelElement("verification_failed", [ FixedDataModelElement("msg", b": SMTP error from remote mail server after RCPT TO:<"), DelimitedDataModelElement("mail_to", b">"), FixedDataModelElement("s3", b">" + host_str), host, FixedDataModelElement("s4", b" ["), host_ip, FixedDataModelElement("status_code", b"]" + status_code550), FixedDataModelElement("msg", b"-Verification for <"), DelimitedDataModelElement("mail_from", b">"), FixedDataModelElement("msg", b">\\n550-The mail server could not deliver mail to "), DelimitedDataModelElement("mail_to", b" "), FixedDataModelElement("msg", b" The account or domain may not exist, they may be blacklisted, or missing the" b" proper dns entries.\\n550 Sender verify failed") ]), SequenceModelElement("unable_to_verify", [ FixedDataModelElement("msg", b": SMTP error from remote mail server after MAIL FROM:<"), DelimitedDataModelElement("mail_from", b">"), FixedDataModelElement("s3", b">" + size_str), size, FixedDataModelElement("s4", host_str), host, FixedDataModelElement("s5", b" ["), host_ip, FixedDataModelElement("status_code", b"]" + status_code553 + b"<"), DelimitedDataModelElement("mail_to", b">"), FixedDataModelElement("msg", b"> unable to verify address\\nVerify that SMPT authentication has been enabled.") ]) ]) ]), SequenceModelElement("mail_delivery_failure", [ msg_id, FixedDataModelElement("s0", b" <= <>" + r_str), r, FixedDataModelElement("s1", u_str), u, FixedDataModelElement("s2", p_str), p, FixedDataModelElement("s3", s_str), s, FixedDataModelElement("s4", t_str), FixedDataModelElement("t", b"\"Mail delivery failed: returning message to sender\""), FixedDataModelElement("s5", b" for "), VariableByteDataModelElement("mail_from", alphabet) ]), SequenceModelElement("mail_flagged_as_spam1", [ msg_id, FixedDataModelElement("s0", h_str), h, FixedDataModelElement("s1", b" ["), host_ip, FixedDataModelElement("s2", b"]:"), port, FixedDataModelElement("msg", b" Warning: \"SpamAssassin as marka22 detected message as spam ("), DelimitedDataModelElement("version", b")"), FixedDataModelElement("s3", b")\"") ]), SequenceModelElement("mail_flagged_as_spam2", [ msg_id, FixedDataModelElement("s0", b" <="), host_ip, FixedDataModelElement("s1", h_str), DelimitedDataModelElement("h", b"["), FixedDataModelElement("s2", b"["), host_ip, FixedDataModelElement("s3", b"]:"), port, FixedDataModelElement("s4", p_str), p, FixedDataModelElement("s5", s_str), s, FixedDataModelElement("s6", id_str), msg_id, FixedDataModelElement("s7", t_str), AnyByteDataModelElement("msg") ]), SequenceModelElement("mail_flagged_as_spam3", [ msg_id, FixedDataModelElement("s0", b" => "), DelimitedDataModelElement("user", b" "), DelimitedDataModelElement("s1", b"<", consume_delimiter=True), mail_from, FixedDataModelElement("s2", b" [>" + r_str), r, FixedDataModelElement("s3", t_str), AnyByteDataModelElement("t") ]), SequenceModelElement("mail_flagged_as_spam4", [ msg_id, FixedDataModelElement("msg", b" Completed"), OptionalMatchModelElement("opt", SequenceModelElement("seq", [ FixedDataModelElement("s0", b" "), dtme, FixedDataModelElement("s1", b" "), msg_id, FixedDataModelElement("s2", h_str), h, FixedDataModelElement("s3", b" ["), host_ip, FixedDataModelElement("s4", b"]:"), port, FixedDataModelElement("msg", b" Warning: \"SpamAssassin as marka22 detected message as spam ("), DelimitedDataModelElement("version", b")"), FixedDataModelElement("s5", b")\"") ])) ]), SequenceModelElement("mail_flagged_as_spam5", [ msg_id, FixedDataModelElement("s0", b" <= "), mail_from, FixedDataModelElement("s1", h_str), h, FixedDataModelElement("s2", b" ["), host_ip, FixedDataModelElement("s3", b"]:"), port, FixedDataModelElement("s4", p_str), p, FixedDataModelElement("s5", s_str), s, FixedDataModelElement("s6", id_str), msg_id, FixedDataModelElement("s7", t_str + b'"'), DelimitedDataModelElement("t", b"\""), FixedDataModelElement("s8", b'" for '), mail_from, FixedDataModelElement("s9", b" "), dtme, FixedDataModelElement("s10", b" "), msg_id, FixedDataModelElement("s11", b" => "), DelimitedDataModelElement("user", b" "), FixedDataModelElement("s12", b" <"), mail_from, FixedDataModelElement("s13", b" [>" + r_str), r, FixedDataModelElement("s14", t_str), AnyByteDataModelElement("t") ]), SequenceModelElement("mail_spam_allowed1", [ msg_id, FixedDataModelElement("s0", h_str), DelimitedDataModelElement("h", b"["), FixedDataModelElement("s1", b"["), host_ip, FixedDataModelElement("s2", b"]:"), port, FirstMatchModelElement("fm", [ FixedDataModelElement("msg", b" Warning: Message has been scanned: no virus or other harmful content was found"), SequenceModelElement("seq", [ FixedDataModelElement( "msg", b" Warning: \"SpamAssassin as cpaneleximscanner detected OUTGOING smtp message as NOT spam ("), DecimalFloatValueModelElement("spam_value", value_sign_type=DecimalFloatValueModelElement.SIGN_TYPE_OPTIONAL), FixedDataModelElement("s3", b")\"") ]) ]) ]), SequenceModelElement("mail_spam_allowed2", [ msg_id, FixedDataModelElement("s0", b" <= "), mail_from, FixedDataModelElement("s1", h_str), h, FixedDataModelElement("s2", b" ["), host_ip, FixedDataModelElement("s3", b"]:"), port, FixedDataModelElement("s4", p_str), p, FixedDataModelElement("s5", x_str), x, FixedDataModelElement("s6", a_str), a, FixedDataModelElement("s7", s_str), s, FixedDataModelElement("s8", t_str), t, FixedDataModelElement("msg", b" plates\" for "), AnyByteDataModelElement("mail_to") ]), SequenceModelElement("mail_spam_allowed3", [ msg_id, FixedDataModelElement("msg", b" SMTP connection outbound "), DecimalIntegerValueModelElement("timestamp"), FixedDataModelElement("s0", b" "), msg_id, FixedDataModelElement("s1", b" "), DelimitedDataModelElement("domain", b" "), FixedDataModelElement("s2", b" "), AnyByteDataModelElement("mail_to") ]), SequenceModelElement("mail_spam_allowed4", [ msg_id, FixedDataModelElement("s0", b" => "), mail_from, FixedDataModelElement("s1", r_str), r, FixedDataModelElement("s2", t_str), t, FixedDataModelElement("s3", h_str), h, FixedDataModelElement("s4", b" ["), host_ip, FixedDataModelElement("s5", b"]" + x_str), x, FixedDataModelElement("s6", c_str), c, FixedDataModelElement("s7", b"\" "), dtme, FixedDataModelElement("s8", b" "), msg_id, FixedDataModelElement("s9", b" Completed"), ]), SequenceModelElement("mail_flagged_as_spam1", [ msg_id, FixedDataModelElement("s0", h_str), h, FixedDataModelElement("s1", b" ["), host_ip, FixedDataModelElement("s2", b"]:"), port, FixedDataModelElement("msg", b" Warning: \"SpamAssassin as sfgthib detected message as spam ("), DelimitedDataModelElement("version", b")"), FixedDataModelElement("s3", b")\" "), dtme, FixedDataModelElement("s4", b" "), msg_id, FixedDataModelElement("s5", h_str), h, FixedDataModelElement("s6", b" ["), host_ip, FixedDataModelElement("s7", b"]:"), port, FixedDataModelElement("msg", b" Warning: Message has been scanned: no virus or other harmful content was found") ]), SequenceModelElement("mail_flagged_as_spam2", [ msg_id, FixedDataModelElement("s0", b" <= "), mail_from, FixedDataModelElement("s1", h_str), h, FixedDataModelElement("s2", b" ["), host_ip, FixedDataModelElement("s3", b"]:"), port, FixedDataModelElement("s4", p_str), p, FixedDataModelElement("s5", x_str), x, FixedDataModelElement("s6", s_str), s, FixedDataModelElement("s7", id_str), msg_id, FixedDataModelElement("s8", t_str), t, FixedDataModelElement("s9", b" for "), AnyByteDataModelElement("mail_to") ]), SequenceModelElement("mail", [ msg_id, FirstMatchModelElement("dir", [ SequenceModelElement("dir_in", [ FixedDataModelElement("in", b" <= "), FirstMatchModelElement("fm", [ SequenceModelElement("seq1", [ FixedDataModelElement("brack", b"<>"), FirstMatchModelElement("fm", [ SequenceModelElement("r", [ FixedDataModelElement("r_str", r_str), r, FixedDataModelElement("u_str", u_str), u, ]), SequenceModelElement("h", [ FixedDataModelElement("h_str", h_str), h, FixedDataModelElement("sp1", b" ["), ip, FixedDataModelElement("sp1", b"]"), ]) ]), FixedDataModelElement("sp2", p_str), p, FixedDataModelElement("sp2", p_str), s, ]), SequenceModelElement("seq2", [ DelimitedDataModelElement("mail", b" "), FixedDataModelElement("user_str", u_str), DelimitedDataModelElement("user", b" "), FixedDataModelElement("p_str", p_str), p, FixedDataModelElement("s_str", s_str), s, OptionalMatchModelElement( "id", SequenceModelElement("id", [ FixedDataModelElement("id_str", id_str), AnyByteDataModelElement("id") ]) ) ]) ]) ]), SequenceModelElement("dir_out", [ FixedDataModelElement("in", b" => "), DelimitedDataModelElement("name", b" "), FixedDataModelElement("sp1", b" "), OptionalMatchModelElement( "mail_opt", SequenceModelElement("mail", [ FixedDataModelElement("brack1", b"("), DelimitedDataModelElement("brack_mail", b")"), FixedDataModelElement("brack2", b") "), ])), FixedDataModelElement("sp2", b"<"), DelimitedDataModelElement("mail", b">"), FixedDataModelElement("r_str", b">" + r_str), r, FixedDataModelElement("t_str", t_str), VariableByteDataModelElement("t", alphabet), ]), SequenceModelElement("aster", [ FixedDataModelElement("aster", b" ** "), DelimitedDataModelElement("command", b" "), FixedDataModelElement("headers_str", b' Too many "Received" headers - suspected mail loop')]), FixedDataModelElement("completed", b" Completed"), FixedDataModelElement("frozen", b" Message is frozen"), FixedDataModelElement("frozen", b" Frozen (delivery error message)") ]) ]), ]) ]), SequenceModelElement("no_date_seq", [ FixedDataModelElement("s0", b"TO:<"), DelimitedDataModelElement("to_mail", b">"), FixedDataModelElement("s1", b">" + host_str), host, FixedDataModelElement("s2", b" ["), host_ip, FixedDataModelElement("status_code", b"]" + status_code450), # status code has to be 450 in this error message. DelimitedDataModelElement("version", b" "), FixedDataModelElement("msg", b" Client host rejected: cannot find your hostname, ["), host_ip, FixedDataModelElement("s3", b"] "), dtme, FixedDataModelElement("s4", b" "), msg_id, FixedDataModelElement("s5", b" ** "), DelimitedDataModelElement("to_mail", b">"), FixedDataModelElement("msg", b">: retry timeout exceeded") ]), SequenceModelElement("invalid_dns_record", [ FixedDataModelElement("msg", b"SMTP error from remote mail server after RCPT TO:" + host_str), DelimitedDataModelElement("host", b"["), FixedDataModelElement("s0", b"["), host_ip, FixedDataModelElement("status_code", b"]" + status_code550), FixedDataModelElement("msg", b"-Sender has no A, AAAA, or MX DNS records. "), DelimitedDataModelElement("host", b"\\"), FixedDataModelElement("s1", b"\\n550 l "), DelimitedDataModelElement("host", b"\\"), FixedDataModelElement("msg", b"\\nVerify the zone file in "), DelimitedDataModelElement("file", b" "), FixedDataModelElement("msg", b" for the correct information. If it appear correct, you can run named-checkzone " b"domain.com domain.com.db to verify if named is able to load the zone.") ]), SequenceModelElement("mail_rejected", [ FixedDataModelElement("msg", b"Diagnostic-Code: X-Postfix;" + host_str1), host, FixedDataModelElement("s0", b" ["), host_ip, FixedDataModelElement("status_code", b"] said" + status_code550 + b" "), DelimitedDataModelElement("version", b" "), FixedDataModelElement("msg", b" Message rejected due to content restrictions (in reply to end of DATA command)\\nWhen you see " b"an error such as 550 "), VariableByteDataModelElement("version", alphabet) ]), SequenceModelElement("mail_authentication_error", [ FixedDataModelElement("msg", b"Final-Recipient: rfc822;"), DelimitedDataModelElement("mail_from", b"\\"), FixedDataModelElement("msg", b"\\nAction: failed\\nStatus: "), DelimitedDataModelElement("status", b"\\"), FixedDataModelElement("msg", b"\\nDiagnostic-Code: smtp;550-Please turn on SMTP Authentication in your mail client.\\n550-"), host, FixedDataModelElement("s0", b" ["), host_ip, FixedDataModelElement("s1", b"]:"), port, FixedDataModelElement("msg", b" is not permitted to relay 550 through this server without authentication.") ]), SequenceModelElement("bad_helo_record", [ DelimitedDataModelElement("cipher_suite", b" "), FixedDataModelElement("msg", b" " + smtp_error_from_remote), DelimitedDataModelElement("mail_from", b">"), FixedDataModelElement("s0", b">" + size_str), size, FixedDataModelElement("s1", host_str), host, FixedDataModelElement("s2", b" ["), host_ip, OptionalMatchModelElement("optional", SequenceModelElement("seq", [ FixedDataModelElement("to", b".."), DecimalIntegerValueModelElement("upper_ip") ])), FixedDataModelElement("status_code", b"]" + status_code550), FixedDataModelElement("msg", b" \"REJECTED - Bad HELO - Host impersonating ["), DelimitedDataModelElement("original_host", b"]"), FixedDataModelElement("s3", b"]\"") ]), SequenceModelElement("domain_not_exists", [ FixedDataModelElement("msg", smtp_error_from_remote), DelimitedDataModelElement("mail_from", b">"), FixedDataModelElement("s0", b">" + host_str), host, FixedDataModelElement("s1", b" ["), host_ip, FixedDataModelElement("status_code", b"]" + status_code553), FixedDataModelElement("msg", b"sorry, your domain does not exists.") ]), SequenceModelElement("rejected_due_to_spam_content", [ DateTimeModelElement("time", b"[%H:%M:%S"), FixedDataModelElement("hosts", b" hosts"), DecimalIntegerValueModelElement("hosts_number"), FixedDataModelElement("s0", b" "), RepeatedElementDataModelElement("rep", FirstMatchModelElement("fm", [ SequenceModelElement("seq", [ dtme, FixedDataModelElement("s1", b" "), msg_id, FixedDataModelElement("s2", b" <= <>" + r_str), r, FixedDataModelElement("s3", u_str), u, FixedDataModelElement("s4", p_str), p, FixedDataModelElement("s5", s_str), s, FixedDataModelElement("s6", t_str + b'"'), DelimitedDataModelElement("t", b'"'), FixedDataModelElement("s7", b'" for '), mail_from, FixedDataModelElement("s8", b" "), dtme, FixedDataModelElement("s9", b" cwd="), DelimitedDataModelElement("cwd", b" "), FixedDataModelElement("s10", b" "), DecimalIntegerValueModelElement("args_num"), FixedDataModelElement("s11", b" args: "), RepeatedElementDataModelElement("rep", FirstMatchModelElement("fm", [ SequenceModelElement("seq", [ dtme, FixedDataModelElement("s12", b" "), msg_id, FixedDataModelElement("s13", b" ** "), mail_from, FixedDataModelElement("s14", r_str), r, FixedDataModelElement("s15", t_str), DelimitedDataModelElement("t", b":"), FixedDataModelElement("msg", b": SMTP error from remote mail server after end of data" + host_str), DelimitedDataModelElement("domain", b" "), FixedDataModelElement("s16", b" ["), host_ip, FixedDataModelElement("status_code", b"]" + status_code554), FixedDataModelElement("msg", b"rejected due to spam content") ]), # this is problematic as the number of arguments is variable! SequenceModelElement("arg_seq", [ DelimitedDataModelElement("arg", b" "), FixedDataModelElement("s17", b" ") ]) ])) ]), # this is problematic as the number of hosts is variable! SequenceModelElement("host_seq", [ host, FixedDataModelElement("s8", b" ") ]) ])) ]), ]) return model