def test3event_data_not_log_atom(self):
"""In this test case the EventHandler receives no logAtom from the test class and the output should not contain the log time."""
volatile_logarithmic_backoff_event_history = VolatileLogarithmicBackoffEventHistory(10)
match_context = MatchContext(self.pid)
fixed_dme = FixedDataModelElement('s1', self.pid)
match_element = fixed_dme.get_match_element("match", match_context)
t = time()
log_atom = LogAtom(fixed_dme.fixed_data, ParserMatch(match_element), t, self)
message = self.new_val % (self.match_s1, self.match_s2, repr(match_element.match_object))
volatile_logarithmic_backoff_event_history.receive_event(self.test % self.__class__.__name__, message, [
log_atom.raw_data, log_atom.raw_data], None, log_atom.get_parser_match(), self)
self.assertEqual(
volatile_logarithmic_backoff_event_history.get_history(), [
(0, self.test % self.__class__.__name__, message, [
log_atom.raw_data, log_atom.raw_data], None, log_atom.get_parser_match(), self)])
def test1add_multiple_objects(self):
"""In this test case multiple events are received by the VolatileLogarithmicBackoffEventHistory."""
volatile_logarithmic_backoff_event_history = VolatileLogarithmicBackoffEventHistory(10)
match_context = MatchContext(self.pid)
fixed_dme = FixedDataModelElement('s1', self.pid)
match_element = fixed_dme.get_match_element("match", match_context)
t = time()
log_atom = LogAtom(fixed_dme.fixed_data, ParserMatch(match_element), t, self)
message = self.new_val % (self.match_s1, self.match_s2, repr(match_element.match_object))
volatile_logarithmic_backoff_event_history.receive_event(self.test % self.__class__.__name__, message, [
log_atom.raw_data, log_atom.raw_data], None, log_atom, self)
self.assertEqual(volatile_logarithmic_backoff_event_history.get_history(), [
(0, self.test % self.__class__.__name__, message, [log_atom.raw_data, log_atom.raw_data], None, log_atom, self)])
volatile_logarithmic_backoff_event_history.receive_event(self.test % self.__class__.__name__, message, [
log_atom.raw_data, log_atom.raw_data], None, log_atom, self)
self.assertEqual(volatile_logarithmic_backoff_event_history.get_history(), [
(0, self.test % self.__class__.__name__, message, [log_atom.raw_data, log_atom.raw_data], None, log_atom, self),
(1, self.test % self.__class__.__name__, message, [log_atom.raw_data, log_atom.raw_data], None, log_atom, self)])
def build_analysis_pipeline(analysis_context):
"""
Define the function to create pipeline for parsing the log data.
It has also to define an AtomizerFactory to instruct aminer how to process incoming data streams to create log atoms from them.
"""
# Build the parsing model:
service_children_disk_report = [
FixedDataModelElement(
'Space',
b' Current Disk Data is: Filesystem Type Size Used Avail Use%'
),
DelimitedDataModelElement('Data', b'%'),
AnyByteDataModelElement('Rest')
]
service_children_login_details = [
FixedDataModelElement('User', b'User '),
DelimitedDataModelElement('Username', b' '),
FixedWordlistDataModelElement('Status',
[b' logged in', b' logged out']),
OptionalMatchModelElement(
'PastTime',
SequenceModelElement('Time', [
FixedDataModelElement('Blank', b' '),
DecimalIntegerValueModelElement('Minutes'),
FixedDataModelElement('Ago', b' minutes ago.')
]))
]
service_children_cron_job = [
DateTimeModelElement('DTM', b'%Y-%m-%d %H:%M:%S'),
FixedDataModelElement('UNameSpace1', b' '),
DelimitedDataModelElement('UName', b' '),
FixedDataModelElement('UNameSpace2', b' '),
DelimitedDataModelElement('User', b' '),
FixedDataModelElement('Cron', b' cron['),
DecimalIntegerValueModelElement('JobNumber'),
FixedDataModelElement('Details', b']: Job `cron.daily` started.')
]
service_children_random_time = [
FixedDataModelElement('Space', b'Random: '),
DecimalIntegerValueModelElement('Random')
]
service_children_sensors = [
SequenceModelElement('CPUTemp', [
FixedDataModelElement('FixedTemp', b'CPU Temp: '),
DecimalIntegerValueModelElement('Temp'),
FixedDataModelElement('Degrees', b'\xc2\xb0C')
]),
FixedDataModelElement('Space1', b', '),
SequenceModelElement('CPUWorkload', [
FixedDataModelElement('FixedWorkload', b'CPUWorkload: '),
DecimalIntegerValueModelElement('Workload'),
FixedDataModelElement('Percent', b'%')
]),
FixedDataModelElement('Space2', b', '),
DateTimeModelElement('DTM', b'%Y-%m-%d %H:%M:%S')
]
service_children_user_ip_address = [
FixedDataModelElement('User', b'User '),
DelimitedDataModelElement('Username', b' '),
FixedDataModelElement('Action', b' changed IP address to '),
IpAddressDataModelElement('IP')
]
service_children_cron_job_announcement = [
DateTimeModelElement('DTM', b'%Y-%m-%d %H:%M:%S'),
FixedDataModelElement('Space', b' '),
DelimitedDataModelElement('UName', b' '),
FixedDataModelElement('Cron', b' cron['),
DecimalIntegerValueModelElement('JobNumber'),
FixedDataModelElement('Run', b']: Will run job `'),
FixedWordlistDataModelElement(
'CronType',
[b'cron.daily', b'cron.hourly', b'cron.monthly', b'cron.weekly']),
FixedDataModelElement('StartTime', b'\' in 5 min.')
]
service_children_cron_job_execution = [
DateTimeModelElement('DTM', b'%Y-%m-%d %H:%M:%S'),
FixedDataModelElement('Space1', b' '),
DelimitedDataModelElement('UName', b' '),
FixedDataModelElement('Cron', b' cron['),
DecimalIntegerValueModelElement('JobNumber'),
FixedDataModelElement('Job', b']: Job `'),
FixedWordlistDataModelElement(
'CronType',
[b'cron.daily', b'cron.hourly', b'cron.monthly', b'cron.weekly']),
FixedDataModelElement('Started', b'\' started')
]
parsing_model = FirstMatchModelElement('model', [
SequenceModelElement('CronAnnouncement',
service_children_cron_job_announcement),
SequenceModelElement('CronExecution',
service_children_cron_job_execution),
SequenceModelElement('DailyCron', service_children_cron_job),
SequenceModelElement('DiskReport', service_children_disk_report),
SequenceModelElement('LoginDetails', service_children_login_details),
DecimalIntegerValueModelElement('Random'),
SequenceModelElement('RandomTime', service_children_random_time),
SequenceModelElement('Sensors', service_children_sensors),
SequenceModelElement('IPAddresses', service_children_user_ip_address)
])
# Some generic imports.
from aminer.analysis import AtomFilters
# Create all global handler lists here and append the real handlers later on.
# Use this filter to distribute all atoms to the analysis handlers.
atom_filters = AtomFilters.SubhandlerFilter(None)
analysis_context.register_component(atom_filters,
component_name="AtomFilter")
from aminer.analysis.TimestampCorrectionFilters import SimpleMonotonicTimestampAdjust
simple_monotonic_timestamp_adjust = SimpleMonotonicTimestampAdjust(
[atom_filters])
analysis_context.register_component(
simple_monotonic_timestamp_adjust,
component_name="SimpleMonotonicTimestampAdjust")
from aminer.events.StreamPrinterEventHandler import StreamPrinterEventHandler
stream_printer_event_handler = StreamPrinterEventHandler(
analysis_context) # skipcq: BAN-B108
from aminer.events.Utils import VolatileLogarithmicBackoffEventHistory
volatile_logarithmic_backoff_event_history = VolatileLogarithmicBackoffEventHistory(
100)
anomaly_event_handlers = [
stream_printer_event_handler,
volatile_logarithmic_backoff_event_history
]
analysis_context.register_component(
volatile_logarithmic_backoff_event_history,
component_name="VolatileLogarithmicBackoffEventHistory")
# Now define the AtomizerFactory using the model. A simple line based one is usually sufficient.
from aminer.input.SimpleByteStreamLineAtomizerFactory import SimpleByteStreamLineAtomizerFactory
analysis_context.atomizer_factory = SimpleByteStreamLineAtomizerFactory(
parsing_model, [simple_monotonic_timestamp_adjust],
anomaly_event_handlers)
# Just report all unparsed atoms to the event handlers.
from aminer.analysis.UnparsedAtomHandlers import SimpleUnparsedAtomHandler
simple_unparsed_atom_handler = SimpleUnparsedAtomHandler(
anomaly_event_handlers)
atom_filters.add_handler(simple_unparsed_atom_handler,
stop_when_handled_flag=True)
analysis_context.register_component(simple_unparsed_atom_handler,
component_name="UnparsedHandler")
from aminer.analysis.TimestampsUnsortedDetector import TimestampsUnsortedDetector
timestamps_unsorted_detector = TimestampsUnsortedDetector(
analysis_context.aminer_config, anomaly_event_handlers)
atom_filters.add_handler(timestamps_unsorted_detector)
analysis_context.register_component(
timestamps_unsorted_detector,
component_name="TimestampsUnsortedDetector")
from aminer.analysis import Rules
from aminer.analysis.AllowlistViolationDetector import AllowlistViolationDetector
allowlist_rules = [
Rules.OrMatchRule([
Rules.AndMatchRule([
Rules.PathExistsMatchRule(
'/model/LoginDetails/PastTime/Time/Minutes'),
Rules.NegationMatchRule(
Rules.ValueMatchRule('/model/LoginDetails/Username',
b'root'))
]),
Rules.AndMatchRule([
Rules.NegationMatchRule(
Rules.PathExistsMatchRule(
'/model/LoginDetails/PastTime/Time/Minutes')),
Rules.PathExistsMatchRule('/model/LoginDetails')
]),
Rules.NegationMatchRule(
Rules.PathExistsMatchRule('/model/LoginDetails'))
])
]
# This rule list should trigger, when the line does not look like: User root (logged in, logged out)
# or User 'username' (logged in, logged out) x minutes ago.
allowlist_violation_detector = AllowlistViolationDetector(
analysis_context.aminer_config, allowlist_rules,
anomaly_event_handlers)
analysis_context.register_component(allowlist_violation_detector,
component_name="Allowlist")
atom_filters.add_handler(allowlist_violation_detector)
from aminer.analysis.ParserCount import ParserCount
parser_count = ParserCount(analysis_context.aminer_config, None,
anomaly_event_handlers, 10)
analysis_context.register_component(parser_count,
component_name="ParserCount")
atom_filters.add_handler(parser_count)
from aminer.analysis.EventCorrelationDetector import EventCorrelationDetector
ecd = EventCorrelationDetector(analysis_context.aminer_config,
anomaly_event_handlers,
check_rules_flag=True,
hypothesis_max_delta_time=1.0,
auto_include_flag=True)
analysis_context.register_component(
ecd, component_name="EventCorrelationDetector")
atom_filters.add_handler(ecd)
from aminer.analysis.NewMatchPathDetector import NewMatchPathDetector
new_match_path_detector = NewMatchPathDetector(
analysis_context.aminer_config,
anomaly_event_handlers,
auto_include_flag=True)
analysis_context.register_component(new_match_path_detector,
component_name="NewMatchPath")
atom_filters.add_handler(new_match_path_detector)
def tuple_transformation_function(match_value_list):
"""Only allow output of the EnhancedNewMatchPathValueComboDetector after every 10000th element."""
extra_data = enhanced_new_match_path_value_combo_detector.known_values_dict.get(
tuple(match_value_list))
if extra_data is not None:
mod = 10000
if (extra_data[2] + 1) % mod == 0:
enhanced_new_match_path_value_combo_detector.auto_include_flag = False
else:
enhanced_new_match_path_value_combo_detector.auto_include_flag = True
return match_value_list
from aminer.analysis.EnhancedNewMatchPathValueComboDetector import EnhancedNewMatchPathValueComboDetector
enhanced_new_match_path_value_combo_detector = EnhancedNewMatchPathValueComboDetector(
analysis_context.aminer_config,
['/model/DailyCron/UName', '/model/DailyCron/JobNumber'],
anomaly_event_handlers,
auto_include_flag=False,
tuple_transformation_function=tuple_transformation_function)
analysis_context.register_component(
enhanced_new_match_path_value_combo_detector,
component_name="EnhancedNewValueCombo")
atom_filters.add_handler(enhanced_new_match_path_value_combo_detector)
from aminer.analysis.HistogramAnalysis import HistogramAnalysis, LinearNumericBinDefinition, ModuloTimeBinDefinition, \
PathDependentHistogramAnalysis
modulo_time_bin_definition = ModuloTimeBinDefinition(
86400, 3600, 0, 1, 24, True)
linear_numeric_bin_definition = LinearNumericBinDefinition(50, 5, 20, True)
histogram_analysis = HistogramAnalysis(
analysis_context.aminer_config,
[('/model/RandomTime/Random', modulo_time_bin_definition),
('/model/Random', linear_numeric_bin_definition)], 10,
anomaly_event_handlers)
analysis_context.register_component(histogram_analysis,
component_name="HistogramAnalysis")
atom_filters.add_handler(histogram_analysis)
path_dependent_histogram_analysis = PathDependentHistogramAnalysis(
analysis_context.aminer_config, '/model/RandomTime',
modulo_time_bin_definition, 10, anomaly_event_handlers)
analysis_context.register_component(
path_dependent_histogram_analysis,
component_name="PathDependentHistogramAnalysis")
atom_filters.add_handler(path_dependent_histogram_analysis)
from aminer.analysis.MatchValueAverageChangeDetector import MatchValueAverageChangeDetector
match_value_average_change_detector = MatchValueAverageChangeDetector(
analysis_context.aminer_config, anomaly_event_handlers, None,
['/model/Random'], 100, 10)
analysis_context.register_component(
match_value_average_change_detector,
component_name="MatchValueAverageChange")
atom_filters.add_handler(match_value_average_change_detector)
import sys
from aminer.analysis.MatchValueStreamWriter import MatchValueStreamWriter
match_value_stream_writer = MatchValueStreamWriter(sys.stdout, [
'/model/Sensors/CPUTemp', '/model/Sensors/CPUWorkload',
'/model/Sensors/DTM'
], b';', b'')
analysis_context.register_component(
match_value_stream_writer, component_name="MatchValueStreamWriter")
atom_filters.add_handler(match_value_stream_writer)
from aminer.analysis.NewMatchPathValueComboDetector import NewMatchPathValueComboDetector
new_match_path_value_combo_detector = NewMatchPathValueComboDetector(
analysis_context.aminer_config,
['/model/IPAddresses/Username', '/model/IPAddresses/IP'],
anomaly_event_handlers,
auto_include_flag=False)
analysis_context.register_component(
new_match_path_value_combo_detector,
component_name="NewMatchPathValueCombo")
atom_filters.add_handler(new_match_path_value_combo_detector)
from aminer.analysis.NewMatchIdValueComboDetector import NewMatchIdValueComboDetector
new_match_id_value_combo_detector = NewMatchIdValueComboDetector(
analysis_context.aminer_config,
['/model/type/path/name', '/model/type/syscall/syscall'],
anomaly_event_handlers,
id_path_list=['/model/type/path/id', '/model/type/syscall/id'],
min_allowed_time_diff=5,
auto_include_flag=True,
allow_missing_values_flag=True,
output_log_line=True)
analysis_context.register_component(
new_match_id_value_combo_detector,
component_name="NewMatchIdValueComboDetector")
atom_filters.add_handler(new_match_id_value_combo_detector)
from aminer.analysis.NewMatchPathValueDetector import NewMatchPathValueDetector
new_match_path_value_detector = NewMatchPathValueDetector(
analysis_context.aminer_config,
['/model/DailyCron/Job Number', '/model/IPAddresses/Username'],
anomaly_event_handlers,
auto_include_flag=False)
analysis_context.register_component(new_match_path_value_detector,
component_name="NewMatchPathValue")
atom_filters.add_handler(new_match_path_value_detector)
from aminer.analysis.MissingMatchPathValueDetector import MissingMatchPathValueDetector
missing_match_path_value_detector = MissingMatchPathValueDetector(
analysis_context.aminer_config, ['/model/DiskReport/Space'],
anomaly_event_handlers,
auto_include_flag=False,
default_interval=2,
realert_interval=5)
analysis_context.register_component(missing_match_path_value_detector,
component_name="MissingMatch")
atom_filters.add_handler(missing_match_path_value_detector)
from aminer.analysis.TimeCorrelationDetector import TimeCorrelationDetector
time_correlation_detector = TimeCorrelationDetector(
analysis_context.aminer_config,
anomaly_event_handlers,
2,
min_rule_attributes=1,
max_rule_attributes=5,
record_count_before_event=70000,
output_log_line=True)
analysis_context.register_component(
time_correlation_detector, component_name="TimeCorrelationDetector")
atom_filters.add_handler(time_correlation_detector)
from aminer.analysis.TimeCorrelationViolationDetector import TimeCorrelationViolationDetector, CorrelationRule, EventClassSelector
cron_job_announcement = CorrelationRule(
'CronJobAnnouncement',
5,
6,
max_artefacts_a_for_single_b=1,
artefact_match_parameters=[('/model/CronAnnouncement/JobNumber',
'/model/CronExecution/JobNumber')])
a_class_selector = EventClassSelector('Announcement',
[cron_job_announcement], None)
b_class_selector = EventClassSelector('Execution', None,
[cron_job_announcement])
rules = [
Rules.PathExistsMatchRule('/model/CronAnnouncement/Run',
a_class_selector),
Rules.PathExistsMatchRule('/model/CronExecution/Job', b_class_selector)
]
time_correlation_violation_detector = TimeCorrelationViolationDetector(
analysis_context.aminer_config, rules, anomaly_event_handlers)
analysis_context.register_component(
time_correlation_violation_detector,
component_name="TimeCorrelationViolationDetector")
atom_filters.add_handler(time_correlation_violation_detector)
from aminer.events.DefaultMailNotificationEventHandler import DefaultMailNotificationEventHandler
if DefaultMailNotificationEventHandler.CONFIG_KEY_MAIL_TARGET_ADDRESS in analysis_context.aminer_config.config_properties:
mail_notification_handler = DefaultMailNotificationEventHandler(
analysis_context)
analysis_context.register_component(mail_notification_handler,
component_name="MailHandler")
anomaly_event_handlers.append(mail_notification_handler)
def test4max_items_overflow(self):
"""In this test case more events than the VolatileLogarithmicBackoffEventHistory can handle are received."""
deviation = 0.05
size = 100000
msg = "%s=%f is not between %f and %f"
match_context = MatchContext(self.pid)
fixed_dme = FixedDataModelElement('s1', self.pid)
match_element = fixed_dme.get_match_element("match", match_context)
t = time()
log_atom = LogAtom(fixed_dme.fixed_data, ParserMatch(match_element), t, self)
message = self.new_val % (self.match_s1, self.match_s2, repr(match_element.match_object))
first = 0
second = 0
third = 0
fourth = 0
for _ in range(size):
volatile_logarithmic_backoff_event_history = VolatileLogarithmicBackoffEventHistory(2)
volatile_logarithmic_backoff_event_history.receive_event(self.test % self.__class__.__name__, message, [
log_atom.raw_data, log_atom.raw_data], None, log_atom.get_parser_match(), self)
volatile_logarithmic_backoff_event_history.receive_event(self.test % self.__class__.__name__, message, [
log_atom.raw_data, log_atom.raw_data], None, log_atom.get_parser_match(), self)
volatile_logarithmic_backoff_event_history.receive_event(self.test % self.__class__.__name__, message, [
log_atom.raw_data, log_atom.raw_data], None, log_atom.get_parser_match(), self)
volatile_logarithmic_backoff_event_history.receive_event(self.test % self.__class__.__name__, message, [
log_atom.raw_data, log_atom.raw_data], None, log_atom.get_parser_match(), self)
volatile_logarithmic_backoff_event_history.receive_event(self.test % self.__class__.__name__, message, [
log_atom.raw_data, log_atom.raw_data], None, log_atom.get_parser_match(), self)
history = volatile_logarithmic_backoff_event_history.get_history()
if history == [(0, self.test % self.__class__.__name__, message, [log_atom.raw_data, log_atom.raw_data], None,
log_atom.get_parser_match(), self), (4, self.test % self.__class__.__name__, message,
[log_atom.raw_data, log_atom.raw_data], None, log_atom.get_parser_match(), self)]:
first += 1
elif history == [(1, self.test % self.__class__.__name__, message, [log_atom.raw_data, log_atom.raw_data], None,
log_atom.get_parser_match(), self), (4, self.test % self.__class__.__name__, message,
[log_atom.raw_data, log_atom.raw_data], None, log_atom.get_parser_match(), self)]:
second += 1
elif history == [(2, self.test % self.__class__.__name__, message, [log_atom.raw_data, log_atom.raw_data], None,
log_atom.get_parser_match(), self), (4, self.test % self.__class__.__name__, message,
[log_atom.raw_data, log_atom.raw_data], None, log_atom.get_parser_match(), self)]:
third += 1
elif history == [(3, self.test % self.__class__.__name__, message, [log_atom.raw_data, log_atom.raw_data], None,
log_atom.get_parser_match(), self), (4, self.test % self.__class__.__name__, message,
[log_atom.raw_data, log_atom.raw_data], None, log_atom.get_parser_match(), self)]:
fourth += 1
val = 0.5 * 0.5 * 0.5
minimum = size * val * (1 - deviation)
maximum = size * val * (1 + deviation)
self.assertTrue(minimum <= first <= maximum, msg % ("first", first, minimum, maximum))
val = 0.5 * 0.5 * 0.5
minimum = size * val * (1 - deviation)
maximum = size * val * (1 + deviation)
self.assertTrue(minimum <= second <= maximum, msg % ("second", second, minimum, maximum))
val = 2 * 0.5 * 0.5 * 0.5
minimum = size * val * (1 - deviation)
maximum = size * val * (1 + deviation)
self.assertTrue(minimum <= third <= maximum, msg % ("third", third, minimum, maximum))
val = 0.5
minimum = size * val * (1 - deviation)
maximum = size * val * (1 + deviation)
self.assertTrue(minimum <= fourth <= maximum, msg % ("fourth", fourth, minimum, maximum))
def test2add_no_objects(self):
"""In this test case no events are received by the VolatileLogarithmicBackoffEventHistory."""
volatile_logarithmic_backoff_event_history = VolatileLogarithmicBackoffEventHistory(10)
self.assertEqual(volatile_logarithmic_backoff_event_history.get_history(), [])