def do_persist(self):
"""Immediately write persistence data to storage."""
for following_module in self.following_modules:
following_module.do_persist()
tmp_list = [[]]
for key in self.found_keys:
tmp_list[0].append(list(key))
tmp_list.append(self.variable_key_list)
tmp_list.append(self.values)
tmp_list.append(self.longest_path)
tmp_list.append(self.check_variables)
tmp_list.append(self.num_eventlines)
tmp_list.append(self.etd_time_trigger)
tmp_list.append(self.num_eventlines_tsa_ref)
tmp_list.append(self.id_path_list_tuples)
PersistenceUtil.store_json(self.persistence_file_name, tmp_list)
for following_module in self.following_modules:
following_module.do_persist()
self.next_persist_time = time.time(
) + self.aminer_config.config_properties.get(
KEY_PERSISTENCE_PERIOD, DEFAULT_PERSISTENCE_PERIOD)
logging.getLogger(DEBUG_LOG_NAME).debug('%s persisted data.',
self.__class__.__name__)
def __init__(self, aminer_config, property_path, bin_definition, report_interval, report_event_handlers, reset_after_report_flag=True,
persistence_id='Default', output_log_line=True):
"""
Initialize the analysis component.
@param report_interval delay in seconds between creation of two reports. The parameter is applied to the parsed record data
time, not the system time. Hence reports can be delayed when no data is received.
"""
self.last_report_time = None
self.next_report_time = 0.0
self.property_path = property_path
self.bin_definition = bin_definition
self.histogram_data = {}
self.report_interval = report_interval
self.report_event_handlers = report_event_handlers
self.reset_after_report_flag = reset_after_report_flag
self.persistence_id = persistence_id
self.next_persist_time = None
self.output_log_line = output_log_line
self.aminer_config = aminer_config
self.persistence_file_name = build_persistence_file_name(aminer_config, self.__class__.__name__, persistence_id)
PersistenceUtil.add_persistable_component(self)
persistence_data = PersistenceUtil.load_json(self.persistence_file_name)
if persistence_data is not None:
msg = 'No data reading, def merge yet'
logging.getLogger(DEBUG_LOG_NAME).error(msg)
raise Exception(msg)
def do_persist(self):
"""Immediately write persistence data to storage."""
persistence_data = []
for dict_record in self.known_values_dict.items():
persistence_data.append(dict_record)
PersistenceUtil.store_json(self.persistence_file_name, persistence_data)
logging.getLogger(DEBUG_LOG_NAME).debug('%s persisted data.', self.__class__.__name__)
def do_persist(self):
"""Immediately write persistence data to storage."""
PersistenceUtil.store_json(self.persistence_file_name,
self.expected_values_dict)
self.next_persist_time = None
logging.getLogger(AMinerConfig.DEBUG_LOG_NAME).debug(
'%s persisted data.', self.__class__.__name__)
def do_persist(self):
"""Immediately write persistence data to storage."""
PersistenceUtil.store_json(self.persistence_file_name,
list(self.known_values_set))
self.next_persist_time = None
logging.getLogger(DEBUG_LOG_NAME).debug('%s persisted data.',
self.__class__.__name__)
def __init__(self,
aminer_config,
anomaly_event_handlers,
persistence_id='Default',
target_path_list=None,
allow_missing_values_flag=True,
ignore_list=None,
output_log_line=True,
auto_include_flag=False,
time_window_length=86400,
max_time_diff=360,
num_reduce_time_list=10):
"""
Initialize the detector. This will also trigger reading or creation of persistence storage location.
@param aminer_config configuration from analysis_context.
@param anomaly_event_handlers for handling events, e.g., print events to stdout.
@param persistence_id name of persistency document.
@param target_path_list parser paths of values to be analyzed. Multiple paths mean that values are analyzed by their combined
occurrences. When no paths are specified, the events given by the full path list are analyzed.
@param allow_missing_values_flag when set to True, the detector will also use matches, where one of the pathes from target_path_list
does not refer to an existing parsed data object.
@param ignore_list list of paths that are not considered for correlation, i.e., events that contain one of these paths are
omitted. The default value is [] as None is not iterable.
@param output_log_line specifies whether the full parsed log atom should be provided in the output.
@param auto_include_flag specifies whether new frequency measurements override ground truth frequencies.
@param time_window_length length of the time window for which the appearances of log lines are identified with each other.
Value of 86400 specfies a day and 604800 a week.
@param max_time_diff maximal time difference in seconds for new times. If the difference of the new time to all previous times is
greater than max_time_diff the new time is considered an anomaly.
@param num_reduce_time_list number of new time entries appended to the time list, before the list is being reduced.
"""
self.next_persist_time = None
self.anomaly_event_handlers = anomaly_event_handlers
self.auto_include_flag = auto_include_flag
self.allow_missing_values_flag = allow_missing_values_flag
self.aminer_config = aminer_config
self.output_log_line = output_log_line
self.ignore_list = ignore_list
if self.ignore_list is None:
self.ignore_list = []
self.target_path_list = target_path_list
if self.target_path_list is None:
self.target_path_list = []
self.time_window_length = time_window_length
self.max_time_diff = max_time_diff
self.num_reduce_time_list = num_reduce_time_list
# Keys: Tuple of values of the paths of target_path_list, Entries: List of all appeared times to the tuple.
self.appeared_time_list = {}
# Keys: Tuple of values of the paths of target_path_list, Entries: Counter of appended times to the time list since last reduction.
self.counter_reduce_time_intervals = {}
# Loads the persistence
self.persistence_id = persistence_id
self.persistence_file_name = AminerConfig.build_persistence_file_name(
aminer_config, self.__class__.__name__, persistence_id)
PersistenceUtil.add_persistable_component(self)
# Imports the persistence
self.load_persistence_data()
def __init__(self,
aminer_config,
target_path_list,
anomaly_event_handlers,
persistence_id='Default',
allow_missing_values_flag=False,
auto_include_flag=False,
output_log_line=True):
"""
Initialize the detector. This will also trigger reading or creation of persistence storage location.
@param target_path_list the list of values to extract from each match to create the value combination to be checked.
@param allow_missing_values_flag when set to True, the detector will also use matches, where one of the pathes from target_path_list
does not refer to an existing parsed data object.
@param auto_include_flag when set to True, this detector will report a new value only the first time before including it
in the known values set automatically.
"""
self.target_path_list = target_path_list
self.anomaly_event_handlers = anomaly_event_handlers
self.allow_missing_values_flag = allow_missing_values_flag
self.auto_include_flag = auto_include_flag
self.output_log_line = output_log_line
self.aminer_config = aminer_config
self.persistence_id = persistence_id
self.log_success = 0
self.log_total = 0
self.log_learned_path_value_combos = 0
self.log_new_learned_values = []
self.persistence_file_name = AminerConfig.build_persistence_file_name(
aminer_config, self.__class__.__name__, persistence_id)
self.next_persist_time = None
self.known_values_set = set()
self.load_persistence_data()
PersistenceUtil.add_persistable_component(self)
def do_timer(self, trigger_time):
"""
Perform trigger actions and to determine the time for next invocation.
The caller may decide to invoke this method earlier than requested during the previous call. Classes implementing this method have
to handle such cases. Each class should try to limit the time spent in this method as it might delay trigger signals to other
components. For extensive compuational work or IO, a separate thread should be used.
@param trigger_time the time this trigger is invoked. This might be the current real time when invoked from real time
timers or the forensic log timescale time value.
@return the number of seconds when next invocation of this trigger is required.
"""
delta = self.next_persist_time - trigger_time
if delta <= 0:
self.repositioning_data_dict = {}
for log_stream_name, log_stream in self.log_streams_by_name.items(
):
repositioning_data = log_stream.get_repositioning_data()
if repositioning_data is not None:
self.repositioning_data_dict[
log_stream_name] = repositioning_data
PersistenceUtil.store_json(self.persistence_file_name,
self.repositioning_data_dict)
delta = self.aminer_config.config_properties.get(
KEY_PERSISTENCE_PERIOD, DEFAULT_PERSISTENCE_PERIOD)
self.next_persist_time = trigger_time + delta
logging.getLogger(DEBUG_LOG_NAME).debug(
'Repositioning data was persisted.')
return delta
def __init__(self,
aminer_config,
ruleset,
anomaly_event_handlers,
persistence_id='Default',
output_log_line=True):
"""
Initialize the detector. This will also trigger reading or creation of persistence storage location.
@param ruleset a list of MatchRule rules with appropriate CorrelationRules attached as actions.
"""
self.aminer_config = aminer_config
self.event_classification_ruleset = ruleset
self.anomaly_event_handlers = anomaly_event_handlers
self.next_persist_time = time.time(
) + self.aminer_config.config_properties.get(
AminerConfig.KEY_PERSISTENCE_PERIOD,
AminerConfig.DEFAULT_PERSISTENCE_PERIOD)
self.persistence_id = persistence_id
self.output_log_line = output_log_line
self.last_log_atom = None
event_correlation_set = set()
for rule in self.event_classification_ruleset:
if rule.match_action.artefact_a_rules is not None:
event_correlation_set |= set(
rule.match_action.artefact_a_rules)
if rule.match_action.artefact_b_rules is not None:
event_correlation_set |= set(
rule.match_action.artefact_b_rules)
self.event_correlation_ruleset = list(event_correlation_set)
self.persistence_file_name = AminerConfig.build_persistence_file_name(
aminer_config, self.__class__.__name__, persistence_id)
PersistenceUtil.add_persistable_component(self)
def __init__(self,
aminer_config,
target_path_list,
anomaly_event_handlers,
persistence_id='Default',
auto_include_flag=False,
output_log_line=True):
"""Initialize the detector. This will also trigger reading or creation of persistence storage location."""
self.target_path_list = target_path_list
self.anomaly_event_handlers = anomaly_event_handlers
self.auto_include_flag = auto_include_flag
self.output_log_line = output_log_line
self.aminer_config = aminer_config
self.next_persist_time = time.time(
) + self.aminer_config.config_properties.get(
KEY_PERSISTENCE_PERIOD, DEFAULT_PERSISTENCE_PERIOD)
self.persistence_id = persistence_id
self.log_success = 0
self.log_total = 0
self.log_learned_path_values = 0
self.log_new_learned_values = []
self.persistence_file_name = build_persistence_file_name(
aminer_config, self.__class__.__name__, persistence_id)
PersistenceUtil.add_persistable_component(self)
persistence_data = PersistenceUtil.load_json(
self.persistence_file_name)
if persistence_data is None:
self.known_values_set = set()
else:
self.known_values_set = set(persistence_data)
logging.getLogger(DEBUG_LOG_NAME).debug(
'%s loaded persistence data.', self.__class__.__name__)
def test2persist_multiple_objects_of_multiple_class(self):
"""In this test case multiple instances of multiple classes are to be persisted and loaded."""
description = "Test2PersistenceUtil"
new_match_path_detector = NewMatchPathDetector(
self.aminer_config, [self.stream_printer_event_handler],
'Default2', True)
self.analysis_context.register_component(new_match_path_detector,
description)
t = time.time()
log_atom_fixed_dme = LogAtom(self.fixed_dme.fixed_data,
ParserMatch(self.match_element_fixed_dme),
t, new_match_path_detector)
log_atom_decimal_integer_value_me = LogAtom(
self.match_context_decimal_integer_value_me.match_data,
ParserMatch(self.match_element_decimal_integer_value_me), t,
new_match_path_detector)
new_match_path_detector.receive_atom(log_atom_fixed_dme)
new_match_path_detector.receive_atom(log_atom_decimal_integer_value_me)
other_new_match_path_detector = NewMatchPathDetector(
self.aminer_config, [self.stream_printer_event_handler],
'otherDetector2', True)
self.analysis_context.register_component(other_new_match_path_detector,
description + "2")
log_atom_fixed_dme = LogAtom(self.fixed_dme.fixed_data,
ParserMatch(self.match_element_fixed_dme),
t, other_new_match_path_detector)
other_new_match_path_detector.receive_atom(log_atom_fixed_dme)
new_match_path_value_combo_detector = NewMatchPathValueComboDetector(
self.aminer_config, ['first/f1/s1'],
[self.stream_printer_event_handler], 'Default', False, True)
self.analysis_context.register_component(
new_match_path_value_combo_detector, description + "3")
log_atom_sequence_me = LogAtom(
self.fixed_dme.fixed_data,
ParserMatch(self.match_element_first_match_me), t,
new_match_path_value_combo_detector)
new_match_path_value_combo_detector.receive_atom(log_atom_sequence_me)
PersistenceUtil.persist_all()
persistence_data = PersistenceUtil.load_json(
new_match_path_detector.persistence_file_name)
self.assertTrue(persistence_data in ([
self.match_element_fixed_dme.get_path(),
self.match_element_decimal_integer_value_me.get_path()
], [
self.match_element_decimal_integer_value_me.get_path(),
self.match_element_fixed_dme.get_path()
]))
self.assertEqual(
PersistenceUtil.load_json(
other_new_match_path_detector.persistence_file_name),
[self.match_element_fixed_dme.get_path()])
self.assertEqual(
PersistenceUtil.load_json(
new_match_path_value_combo_detector.persistence_file_name),
([[log_atom_sequence_me.raw_data]]))
def __init__(self,
aminer_config,
anomaly_event_handlers,
id_path_list,
target_path_list=None,
persistence_id='Default',
auto_include_flag=False,
output_log_line=True,
ignore_list=None,
constraint_list=None):
"""
Initialize the detector. This will also trigger reading or creation of persistence storage location.
@param aminer_config configuration from analysis_context.
@param anomaly_event_handlers for handling events, e.g., print events to stdout.
@param id_path_list to specify group identifiers for which numeric ranges should be learned.
@param target_path_list parser paths of values to be analyzed. Multiple paths mean that all values occurring in these paths
are considered for value range generation.
@param persistence_id name of persistency document.
@param auto_include_flag specifies whether value ranges should be extended when values outside of ranges are observed.
@param output_log_line specifies whether the full parsed log atom should be provided in the output.
@param ignore_list list of paths that are not considered for analysis, i.e., events that contain one of these paths are
omitted.
@param constrain_list list of paths that have to be present in the log atom to be analyzed.
"""
self.target_path_list = target_path_list
self.anomaly_event_handlers = anomaly_event_handlers
self.auto_include_flag = auto_include_flag
self.next_persist_time = None
self.output_log_line = output_log_line
self.aminer_config = aminer_config
self.persistence_id = persistence_id
self.id_path_list = id_path_list
if constraint_list is None:
self.constraint_list = []
else:
self.constraint_list = set(constraint_list)
if ignore_list is None:
self.ignore_list = []
else:
self.ignore_list = set(ignore_list)
self.log_total = 0
self.log_success = 0
self.ranges_min = {}
self.ranges_max = {}
# Persisted data consists of min and max values for each identifier, i.e.,
# [["min", [<id1, id2, ...>], <min_value>], ["max", [<id1, id2, ...>], <max_value>]]
self.persistence_file_name = AminerConfig.build_persistence_file_name(
aminer_config, self.__class__.__name__, persistence_id)
PersistenceUtil.add_persistable_component(self)
persistence_data = PersistenceUtil.load_json(
self.persistence_file_name)
if persistence_data is not None:
for l in persistence_data:
if l[0] == 'min':
self.ranges_min[tuple(l[1])] = float(l[2])
elif l[0] == 'max':
self.ranges_max[tuple(l[1])] = float(l[2])
def do_persist(self):
"""Immediately write persistence data to storage."""
lst = []
for id_ev, charset in self.charsets.items():
lst.append([id_ev, list(charset)])
PersistenceUtil.store_json(self.persistence_file_name, lst)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).debug(
'%s persisted data.', self.__class__.__name__)
def do_persist(self):
"""Immediately write persistence data to storage."""
persist_data = []
for log_ev, freq in self.counts_prev.items():
persist_data.append((log_ev, freq))
PersistenceUtil.store_json(self.persistence_file_name, persist_data)
self.next_persist_time = None
logging.getLogger(DEBUG_LOG_NAME).debug('%s persisted data.', self.__class__.__name__)
def do_persist(self):
"""Immediately write persistence data to storage."""
persist_data = []
for log_ev, freqs in self.counts.items():
# Skip last count as the time window may not be complete yet and count thus too low
persist_data.append((log_ev, freqs[:-1]))
PersistenceUtil.store_json(self.persistence_file_name, persist_data)
logging.getLogger(DEBUG_LOG_NAME).debug('%s persisted data.',
self.__class__.__name__)
def __init__(self,
aminer_config,
target_path_list,
anomaly_event_handlers,
persistence_id='Default',
auto_include_flag=False,
default_interval=3600,
realert_interval=86400,
output_log_line=True):
"""
Initialize the detector. This will also trigger reading or creation of persistence storage location.
@param target_path_list to extract a source identification value from each logatom.
"""
self.target_path_list = target_path_list
self.anomaly_event_handlers = anomaly_event_handlers
self.auto_include_flag = auto_include_flag
self.default_interval = default_interval
self.realert_interval = realert_interval
# This timestamps is compared with timestamp values from log atoms for activation of alerting logic. The first timestamp from logs
# above this value will trigger alerting.
self.next_check_timestamp = 0
self.last_seen_timestamp = 0
self.output_log_line = output_log_line
self.aminer_config = aminer_config
self.next_persist_time = time.time(
) + self.aminer_config.config_properties.get(
KEY_PERSISTENCE_PERIOD, DEFAULT_PERSISTENCE_PERIOD)
self.persistence_id = persistence_id
self.log_success = 0
self.log_total = 0
self.log_learned_values = 0
self.log_new_learned_values = []
self.persistence_file_name = build_persistence_file_name(
aminer_config, self.__class__.__name__, persistence_id)
PersistenceUtil.add_persistable_component(self)
persistence_data = PersistenceUtil.load_json(
self.persistence_file_name)
self.expected_values_dict = {}
if persistence_data is not None:
for key in persistence_data:
value = persistence_data[key]
if self.target_path_list is not None: # skipcq: PTC-W0048
if value[3] != self.target_path_list:
continue
elif self.target_path_list is not None and value[
3] not in self.target_path_list:
continue
if value[1] != default_interval:
value[1] = default_interval
value[2] = value[0] + default_interval
self.expected_values_dict[key] = value
logging.getLogger(DEBUG_LOG_NAME).debug(
'%s loaded persistence data.', self.__class__.__name__)
self.analysis_string = 'Analysis.%s'
def do_persist(self):
"""Immediately write persistence data to storage."""
persistence_data = []
persistence_data.append(self.time_window_history)
persistence_data.append(self.prediction_history)
persistence_data.append(self.time_history)
persistence_data.append(self.result_list)
PersistenceUtil.store_json(self.persistence_file_name, persistence_data)
logging.getLogger(DEBUG_LOG_NAME).debug('%s persisted data.', self.__class__.__name__)
def __init__(self,
aminer_config,
anomaly_event_handlers,
timestamp_path,
analyze_path_list,
min_bin_elements,
min_bin_time,
debug_mode=False,
persistence_id='Default',
output_log_line=True):
"""
Initialize the detector. This will also trigger reading or creation of persistence storage location.
@param timestamp_path if not None, use this path value for timestamp based bins.
@param analyze_path_list list of match paths to analyze in this detector.
@param min_bin_elements evaluate the latest bin only after at least that number of elements was added to it.
@param min_bin_time evaluate the latest bin only when the first element is received after min_bin_time has elapsed.
@param debug_mode if true, generate an analysis report even when average of last bin was within expected range.
"""
self.anomaly_event_handlers = anomaly_event_handlers
self.timestamp_path = timestamp_path
self.min_bin_elements = min_bin_elements
self.min_bin_time = min_bin_time
self.debug_mode = debug_mode
self.next_persist_time = None
self.persistence_id = persistence_id
self.output_log_line = output_log_line
self.aminer_config = aminer_config
self.persistence_file_name = build_persistence_file_name(
aminer_config, self.__class__.__name__, persistence_id)
PersistenceUtil.add_persistable_component(self)
persistence_data = PersistenceUtil.load_json(
self.persistence_file_name)
self.stat_data = []
for path in analyze_path_list:
self.stat_data.append((
path,
[],
))
if persistence_data is not None:
for val in persistence_data:
if isinstance(val, str):
val = val.strip('[').strip(']').split(',', 2)
path = val[0].strip('"')
values = val[1].strip(' ').strip('[').strip(']')
else:
path = val[0]
values = val[1]
index = 0
for p, _ in self.stat_data:
if p == path:
break
index += 1
for value in values:
self.stat_data[index][1].append(value)
def __init__(self, aminer_config, anomaly_event_handlers, id_path_list=None, target_path_list=None, seq_len=3, persistence_id='Default',
auto_include_flag=False, output_log_line=True, ignore_list=None, constraint_list=None):
"""
Initialize the detector. This will also trigger reading or creation of persistence storage location.
@param aminer_config configuration from analysis_context.
@param target_path_list parser paths of values to be analyzed. Multiple paths mean that values are analyzed by their combined
occurrences. When no paths are specified, the events given by the full path list are analyzed.
@param anomaly_event_handlers for handling events, e.g., print events to stdout.
@param id_path_list one or more paths that specify the trace of the sequence detection, i.e., incorrect sequences that are
generated by interleaved events can be avoided when event sequence identifiers are available.
@param seq_len the length of the sequences to be learned (larger lengths increase precision, but may overfit the data).
@param persistence_id name of persistency document.
@param auto_include_flag specifies whether new frequency measurements override ground truth frequencies.
@param output_log_line specifies whether the full parsed log atom should be provided in the output.
@param ignore_list list of paths that are not considered for analysis, i.e., events that contain one of these paths are
omitted. The default value is [] as None is not iterable.
@param constrain_list list of paths that have to be present in the log atom to be analyzed.
"""
self.target_path_list = target_path_list
self.anomaly_event_handlers = anomaly_event_handlers
self.auto_include_flag = auto_include_flag
self.next_persist_time = None
self.output_log_line = output_log_line
self.aminer_config = aminer_config
self.persistence_id = persistence_id
self.id_path_list = id_path_list
if id_path_list is None:
self.id_path_list = []
self.constraint_list = constraint_list
if self.constraint_list is None:
self.constraint_list = []
self.ignore_list = ignore_list
if self.ignore_list is None:
self.ignore_list = []
self.seq_len = seq_len
self.sequences = set()
self.current_sequences = {}
self.log_total = 0
self.log_success = 0
self.log_learned = 0
self.log_learned_sequences = []
self.persistence_file_name = AminerConfig.build_persistence_file_name(aminer_config, self.__class__.__name__, persistence_id)
PersistenceUtil.add_persistable_component(self)
# Persisted data contains lists of sequences, i.e., [[<seq1_elem1>, <seq1_elem2>], [<seq2_elem1, ...], ...]
# Thereby, sequence elements may be tuples, i.e., combinations of values, or paths that define events.
persistence_data = PersistenceUtil.load_json(self.persistence_file_name)
if persistence_data is not None:
for sequence in persistence_data:
sequence_elem_tuple = []
for sequence_elem in sequence:
sequence_elem_tuple.append(tuple(sequence_elem))
self.sequences.add(tuple(sequence_elem_tuple))
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).debug('%s loaded persistence data.', self.__class__.__name__)
def do_persist(self):
"""Immediately write persistence data to storage."""
persistence_data = []
persistence_data.append(self.target_path_index_list)
persistence_data.append(self.period_length_list)
persistence_data.append(self.prediction_history)
PersistenceUtil.store_json(self.persistence_file_name,
persistence_data)
logging.getLogger(DEBUG_LOG_NAME).debug('%s persisted data.',
self.__class__.__name__)
def do_persist(self):
"""Immediately write persistence data to storage."""
persist_data = [[], []]
for match_value_tuple, time_list in self.appeared_time_list.items():
persist_data[0].append((match_value_tuple, time_list))
for match_value_tuple, counter in self.counter_reduce_time_intervals.items(
):
persist_data[1].append((match_value_tuple, counter))
PersistenceUtil.store_json(self.persistence_file_name, persist_data)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).debug(
'%s persisted data.', self.__class__.__name__)
def __init__(self, aminer_config, anomaly_event_handlers, target_path_list=None, window_size=600, confidence_factor=0.5,
persistence_id='Default', auto_include_flag=False, output_log_line=True, ignore_list=None, constraint_list=None):
"""
Initialize the detector. This will also trigger reading or creation of persistence storage location.
@param aminer_config configuration from analysis_context.
@param target_path_list parser paths of values to be analyzed. Multiple paths mean that values are analyzed by their combined
occurrences. When no paths are specified, the events given by the full path list are analyzed.
@param anomaly_event_handlers for handling events, e.g., print events to stdout.
@param window_size the length of the time window for counting in seconds.
@param confidence_factor defines range of tolerable deviation of measured frequency from ground truth frequency gt by
[gf * confidence_factor, gf / confidence_factor]. confidence_factor must be in range [0, 1].
@param persistence_id name of persistency document.
@param auto_include_flag specifies whether new frequency measurements override ground truth frequencies.
@param output_log_line specifies whether the full parsed log atom should be provided in the output.
@param ignore_list list of paths that are not considered for analysis, i.e., events that contain one of these paths are
omitted. The default value is [] as None is not iterable.
@param constrain_list list of paths that have to be present in the log atom to be analyzed.
"""
self.target_path_list = target_path_list
self.anomaly_event_handlers = anomaly_event_handlers
self.auto_include_flag = auto_include_flag
self.next_persist_time = None
self.output_log_line = output_log_line
self.aminer_config = aminer_config
self.persistence_id = persistence_id
self.constraint_list = constraint_list
if self.constraint_list is None:
self.constraint_list = []
self.ignore_list = ignore_list
if self.ignore_list is None:
self.ignore_list = []
self.window_size = window_size
if not 0 <= confidence_factor <= 1:
logging.getLogger(DEBUG_LOG_NAME).warning('confidence_factor must be in the range [0,1]!')
confidence_factor = 1
self.confidence_factor = confidence_factor
self.next_check_time = None
self.counts = {}
self.counts_prev = {}
self.log_total = 0
self.log_success = 0
self.log_windows = 0
self.persistence_file_name = build_persistence_file_name(aminer_config, self.__class__.__name__, persistence_id)
PersistenceUtil.add_persistable_component(self)
# Persisted data contains lists of event-frequency pairs, i.e., [[<ev1, ev2>, <freq>], [<ev1, ev2>, <freq>], ...]
persistence_data = PersistenceUtil.load_json(self.persistence_file_name)
if persistence_data is not None:
for entry in persistence_data:
log_event = entry[0]
frequency = entry[1]
self.counts_prev[tuple(log_event)] = frequency
logging.getLogger(DEBUG_LOG_NAME).debug('%s loaded persistence data.', self.__class__.__name__)
def do_persist(self):
"""Immediately write persistence data to storage."""
lst = []
for id_ev, v in self.ranges_min.items():
lst.append(['min', id_ev, v])
for id_ev, v in self.ranges_max.items():
lst.append(['max', id_ev, v])
PersistenceUtil.store_json(self.persistence_file_name, lst)
self.next_persist_time = None
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).debug(
'%s persisted data.', self.__class__.__name__)
def __init__(self,
aminer_config,
anomaly_event_handlers,
parallel_check_count,
persistence_id='Default',
record_count_before_event=10000,
output_log_line=True,
use_path_match=True,
use_value_match=True,
min_rule_attributes=1,
max_rule_attributes=5):
"""
Initialize the detector. This will also trigger reading or creation of persistence storage location.
@param parallel_check_count number of rule detection checks to run in parallel.
@param record_count_before_event number of events used to calculate statistics (i.e., window size)
@param min_rule_attributes minimum number of attributes forming a rule
@param max_rule_attributes maximum number of attributes forming a rule
@param use_path_match if true rules are build based on path existance
@param use_value_match if true rules are built based on actual values
"""
self.last_timestamp = 0.0
self.parallel_check_count = parallel_check_count
self.anomaly_event_handlers = anomaly_event_handlers
self.min_rule_attributes = min_rule_attributes
self.max_rule_attributes = max_rule_attributes
self.last_unhandled_match = None
self.total_records = 0
self.record_count_before_event = record_count_before_event
self.persistence_id = persistence_id
self.output_log_line = output_log_line
self.use_path_match = use_path_match
self.use_value_match = use_value_match
self.aminer_config = aminer_config
self.next_persist_time = time.time(
) + self.aminer_config.config_properties.get(
KEY_PERSISTENCE_PERIOD, DEFAULT_PERSISTENCE_PERIOD)
self.persistence_file_name = build_persistence_file_name(
aminer_config, self.__class__.__name__, persistence_id)
PersistenceUtil.add_persistable_component(self)
persistence_data = PersistenceUtil.load_json(
self.persistence_file_name)
if persistence_data is None:
self.feature_list = []
self.event_count_table = [
0
] * parallel_check_count * parallel_check_count * 2
self.event_delta_table = [
0
] * parallel_check_count * parallel_check_count * 2
else:
logging.getLogger(DEBUG_LOG_NAME).debug(
'%s loaded persistence data.', self.__class__.__name__)
def do_persist(self):
"""Immediately write persistence data to storage."""
tmp_list = []
tmp_list.append(self.time_window_history)
tmp_list.append(self.prediction_history)
tmp_list.append(self.time_history)
tmp_list.append(self.result_list)
PersistenceUtil.store_json(self.persistence_file_name, tmp_list)
self.next_persist_time = time.time() + self.aminer_config.config_properties.get(
KEY_PERSISTENCE_PERIOD, DEFAULT_PERSISTENCE_PERIOD)
logging.getLogger(DEBUG_LOG_NAME).debug('%s persisted data.', self.__class__.__name__)
def do_persist(self):
"""Immediately write persistence data to storage."""
persist_data = []
keys_1 = list(self.time_matrix.keys())
keys_2 = [list(self.time_matrix[key].keys()) for key in keys_1]
return_matrix = [[
self.time_matrix[keys_1[i]][keys_2[i][j]]
for j in range(len(keys_2[i]))
] for i in range(len(keys_1))]
persist_data.append(return_matrix)
persist_data.append(keys_1)
persist_data.append(keys_2)
PersistenceUtil.store_json(self.persistence_file_name, persist_data)
logging.getLogger(DEBUG_LOG_NAME).debug('%s persisted data.',
self.__class__.__name__)
def do_timer(self, trigger_time):
"""Check current ruleset should be persisted."""
if self.next_persist_time is None:
return self.aminer_config.config_properties.get(
AMinerConfig.KEY_PERSISTENCE_PERIOD,
AMinerConfig.DEFAULT_PERSISTENCE_PERIOD)
delta = self.next_persist_time - trigger_time
if delta < 0:
PersistenceUtil.store_json(self.persistence_file_name,
self.expected_values_dict)
self.next_persist_time = None
delta = self.aminer_config.config_properties.get(
AMinerConfig.KEY_PERSISTENCE_PERIOD,
AMinerConfig.DEFAULT_PERSISTENCE_PERIOD)
return delta
def __init__(self,
aminer_config,
target_path_list,
anomaly_event_handlers,
id_path_list,
min_allowed_time_diff,
persistence_id='Default',
allow_missing_values_flag=False,
auto_include_flag=False,
output_log_line=True):
"""
Initialize the detector. This will also trigger reading or creation of persistence storage location.
@param target_path_list the list of values to extract from each match to create the value combination to be checked.
@param id_path_list the list of pathes where id values can be stored in all relevant log event types.
@param min_allowed_time_diff the minimum amount of time in seconds after the first appearance of a log atom with a specific id
that is waited for other log atoms with the same id to occur. The maximum possible time to keep an incomplete combo
is 2*min_allowed_time_diff
@param allow_missing_values_flag when set to True, the detector will also use matches, where one of the pathes from target_path_list
does not refer to an existing parsed data object.
@param auto_include_flag when set to True, this detector will report a new value only the first time before including it
in the known values set automatically.
"""
self.target_path_list = target_path_list
self.anomaly_event_handlers = anomaly_event_handlers
self.id_path_list = id_path_list
self.min_allowed_time_diff = min_allowed_time_diff
self.allow_missing_values_flag = allow_missing_values_flag
self.auto_include_flag = auto_include_flag
self.output_log_line = output_log_line
self.aminer_config = aminer_config
self.persistence_id = persistence_id
self.log_success = 0
self.log_total = 0
self.log_learned_path_value_combos = 0
self.log_new_learned_values = []
self.persistence_file_name = build_persistence_file_name(
aminer_config, self.__class__.__name__, persistence_id)
self.next_persist_time = time.time(
) + self.aminer_config.config_properties.get(
KEY_PERSISTENCE_PERIOD, DEFAULT_PERSISTENCE_PERIOD)
self.load_persistence_data()
PersistenceUtil.add_persistable_component(self)
self.id_dict_current = {}
self.id_dict_old = {}
self.next_shift_time = None
def load_persistence_data(self):
"""Load the persistence data from storage."""
persistence_data = PersistenceUtil.load_json(self.persistence_file_name)
if persistence_data is not None:
# Set and tuples were stored as list of lists. Transform the inner lists to tuples to allow hash operation needed by set.
self.known_values_set = {tuple(record) for record in persistence_data}
logging.getLogger(DEBUG_LOG_NAME).debug('%s loaded persistence data.', self.__class__.__name__)
def load_persistence_data(self):
"""Load the persistence data from storage."""
persistence_data = PersistenceUtil.load_json(self.persistence_file_name)
if persistence_data is not None:
self.target_path_index_list = persistence_data[0]
self.period_length_list = persistence_data[1]
self.prediction_history = persistence_data[2]