def __init__(self,
aminer_config,
histogram_defs,
report_interval,
report_event_handlers,
reset_after_report_flag=True,
persistence_id='Default',
output_log_line=True):
"""Initialize the analysis component.
@param histogram_defs is a list of tuples containing the target property path to analyze and the BinDefinition to apply for
binning.
@param report_interval delay in seconds between creation of two reports. The parameter is applied to the parsed record data
time, not the system time. Hence reports can be delayed when no data is received."""
self.last_report_time = None
self.next_report_time = 0.0
self.histogram_data = []
for (path, bin_definition) in histogram_defs:
self.histogram_data.append(HistogramData(path, bin_definition))
self.report_interval = report_interval
self.report_event_handlers = report_event_handlers
self.reset_after_report_flag = reset_after_report_flag
self.persistence_id = persistence_id
self.next_persist_time = None
self.output_log_line = output_log_line
PersistencyUtil.add_persistable_component(self)
self.persistenceFileName = AMinerConfig.build_persistence_file_name(
aminer_config, 'HistogramAnalysis', persistence_id)
persistence_data = PersistencyUtil.load_json(self.persistenceFileName)
if persistence_data is not None:
raise Exception('No data reading, def merge yet')
def __init__(self, aminer_config, parallel_check_count, correlation_test_count, max_fail_count, anomaly_event_handlers,
persistence_id='Default', record_count_before_event=0x10000, output_log_line=True):
"""Initialize the detector. This will also trigger reading or creation of persistence storage location.
@param parallel_check_count number of rule detection checks to run in parallel.
@param correlation_test_count number of unit to perform on a rule under test.
@param max_fail_count maximal number of test failures so that rule is still eligible for reporting."""
self.last_timestamp = 0.0
self.parallel_check_count = parallel_check_count
self.correlation_test_count = correlation_test_count
self.max_fail_count = max_fail_count
self.anomaly_event_handlers = anomaly_event_handlers
self.max_rule_attributes = 5
self.last_unhandled_match = None
self.next_persist_time = None
self.total_records = 0
self.record_count_before_event = record_count_before_event
self.persistence_id = persistence_id
self.output_log_line = output_log_line
PersistencyUtil.add_persistable_component(self)
self.persistence_file_name = AMinerConfig.build_persistence_file_name(aminer_config, 'TimeCorrelationDetector', persistence_id)
persistence_data = PersistencyUtil.load_json(self.persistence_file_name)
if persistence_data is None:
self.feature_list = []
self.event_count_table = [0] * parallel_check_count * parallel_check_count * 2
self.event_delta_table = [0] * parallel_check_count * parallel_check_count * 2
def __init__(self,
aminer_config,
property_path,
bin_definition,
report_interval,
report_event_handlers,
reset_after_report_flag=True,
persistence_id='Default',
output_log_line=True):
"""Initialize the analysis component.
@param report_interval delay in seconds between creation of two reports. The parameter is applied to the parsed record data
time, not the system time. Hence reports can be delayed when no data is received."""
self.last_report_time = None
self.next_report_time = 0.0
self.property_path = property_path
self.bin_definition = bin_definition
self.histogram_data = {}
self.report_interval = report_interval
self.report_event_handlers = report_event_handlers
self.reset_after_report_flag = reset_after_report_flag
self.persistence_id = persistence_id
self.next_persist_time = None
self.output_log_line = output_log_line
PersistencyUtil.add_persistable_component(self)
self.persistence_file_name = AMinerConfig.build_persistence_file_name(
aminer_config, 'PathDependentHistogramAnalysis', persistence_id)
persistence_data = PersistencyUtil.load_json(
self.persistence_file_name)
if persistence_data is not None:
raise Exception('No data reading, def merge yet')
def __init__(self, aminer_config, anomaly_event_handlers, timestamp_path, analyze_path_list, min_bin_elements, min_bin_time,
sync_bins_flag=True, debug_mode=False, persistence_id='Default', output_log_line=True):
"""Initialize the detector. This will also trigger reading or creation of persistence storage location.
@param timestamp_path if not None, use this path value for timestamp based bins.
@param analyze_path_list list of match pathes to analyze in this detector.
@param min_bin_elements evaluate the latest bin only after at least that number of elements was added to it.
@param min_bin_time evaluate the latest bin only when the first element is received after minBinTime has elapsed.
@param sync_bins_flag if true the bins of all analyzed path values have to be filled enough to trigger analysis.
@param debug_mode if true, generate an analysis report even when average of last bin was within expected range."""
self.anomaly_event_handlers = anomaly_event_handlers
self.timestamp_path = timestamp_path
self.min_bin_elements = min_bin_elements
self.min_bin_time = min_bin_time
self.sync_bins_flag = sync_bins_flag
self.debug_mode = debug_mode
self.next_persist_time = None
self.persistence_id = persistence_id
self.output_log_line = output_log_line
PersistencyUtil.add_persistable_component(self)
self.persistence_file_name = AMinerConfig.build_persistence_file_name(aminer_config, 'MatchValueAverageChangeDetector',
persistence_id)
persistence_data = PersistencyUtil.load_json(self.persistence_file_name)
if persistence_data is None:
self.stat_data = []
for path in analyze_path_list:
self.stat_data.append((path, [],))
def __init__(self,
aminer_config,
target_path,
anomaly_event_handlers,
persistence_id='Default',
auto_include_flag=False,
default_interval=3600,
realert_interval=86400,
output_log_line=True):
"""Initialize the detector. This will also trigger reading or creation of persistence storage location.
@param target_path to extract a source identification value from each logatom."""
self.target_path = target_path
self.anomaly_event_handlers = anomaly_event_handlers
self.auto_include_flag = auto_include_flag
self.default_interval = default_interval
self.realert_interval = realert_interval
# This timestamps is compared with timestamp values from log atoms for activation of alerting logic. The first timestamp from logs
# above this value will trigger alerting.
self.next_check_timestamp = 0
self.last_seen_timestamp = 0
self.next_persist_time = None
self.output_log_line = output_log_line
self.aminer_config = aminer_config
self.persistence_id = persistence_id
PersistencyUtil.add_persistable_component(self)
self.persistence_file_name = AMinerConfig.build_persistence_file_name(
aminer_config, self.__class__.__name__, persistence_id)
persistence_data = PersistencyUtil.load_json(
self.persistence_file_name)
if persistence_data is None:
self.expected_values_dict = {}
else:
self.expected_values_dict = persistence_data
self.analysis_string = 'Analysis.%s'
def __init__(self,
aminer_config,
target_path_list,
anomaly_event_handlers,
persistence_id='Default',
auto_include_flag=False,
output_log_line=True):
"""Initialize the detector. This will also trigger reading or creation of persistence storage location."""
self.target_path_list = target_path_list
self.anomaly_event_handlers = anomaly_event_handlers
self.auto_include_flag = auto_include_flag
self.next_persist_time = None
self.output_log_line = output_log_line
self.aminer_config = aminer_config
self.persistence_id = persistence_id
PersistencyUtil.add_persistable_component(self)
self.persistence_file_name = AMinerConfig.build_persistence_file_name(
aminer_config, self.__class__.__name__, persistence_id)
persistence_data = PersistencyUtil.load_json(
self.persistence_file_name)
if persistence_data is None:
self.known_path_set = set()
else:
self.known_path_set = set(persistence_data)
def __init__(self,
aminer_config,
ruleset,
anomaly_event_handlers,
persistence_id='Default',
output_log_line=True):
"""Initialize the detector. This will also trigger reading or creation of persistence storage location.
@param ruleset a list of MatchRule rules with appropriate CorrelationRules attached as actions."""
self.event_classification_ruleset = ruleset
self.anomaly_event_handlers = anomaly_event_handlers
self.next_persist_time = time.time() + 600.0
self.persistence_id = persistence_id
self.output_log_line = output_log_line
self.last_log_atom = None
event_correlation_set = set()
for rule in self.event_classification_ruleset:
if rule.match_action.artefact_a_rules is not None:
event_correlation_set |= set(
rule.match_action.artefact_a_rules)
if rule.match_action.artefact_b_rules is not None:
event_correlation_set |= set(
rule.match_action.artefact_b_rules)
self.event_correlation_ruleset = list(event_correlation_set)
PersistencyUtil.add_persistable_component(self)
self.persistence_file_name = AMinerConfig.build_persistence_file_name(
aminer_config, 'TimeCorrelationViolationDetector', persistence_id)
def __init__(self,
aminer_config,
target_path_list,
anomaly_event_handlers,
persistence_id='Default',
allow_missing_values_flag=False,
auto_include_flag=False,
output_log_line=True):
"""Initialize the detector. This will also trigger reading or creation of persistence storage location.
@param target_path_list the list of values to extract from each match to create the value combination to be checked.
@param allow_missing_values_flag when set to True, the detector will also use matches, where one of the pathes from targetPathList
does not refer to an existing parsed data object.
@param auto_include_flag when set to True, this detector will report a new value only the first time before including it
in the known values set automatically."""
self.target_path_list = target_path_list
self.anomaly_event_handlers = anomaly_event_handlers
self.allow_missing_values_flag = allow_missing_values_flag
self.auto_include_flag = auto_include_flag
self.output_log_line = output_log_line
self.aminer_config = aminer_config
self.persistence_id = persistence_id
self.persistence_file_name = AMinerConfig.build_persistence_file_name(
aminer_config, self.__class__.__name__, persistence_id)
self.next_persist_time = None
self.known_values_set = set()
self.load_persistency_data()
PersistencyUtil.add_persistable_component(self)
def __init__(self,
aminer_config,
target_path_list,
anomaly_event_handlers,
id_path_list,
min_allowed_time_diff,
persistence_id='Default',
allow_missing_values_flag=False,
auto_include_flag=False,
output_log_line=True):
"""Initialize the detector. This will also trigger reading or creation of persistence storage location.
@param target_path_list the list of values to extract from each match to create the value combination to be checked.
@param id_path_list the list of pathes where id values can be stored in all relevant log event types.
@param min_allowed_time_diff the minimum amount of time in seconds after the first appearance of a log atom with a specific id
that is waited for other log atoms with the same id to occur. The maximum possible time to keep an incomplete combo
is 2*min_allowed_time_diff
@param allow_missing_values_flag when set to True, the detector will also use matches, where one of the pathes from targetPathList
does not refer to an existing parsed data object.
@param auto_include_flag when set to True, this detector will report a new value only the first time before including it
in the known values set automatically."""
self.target_path_list = target_path_list
self.anomaly_event_handlers = anomaly_event_handlers
self.id_path_list = id_path_list
self.min_allowed_time_diff = min_allowed_time_diff
self.allow_missing_values_flag = allow_missing_values_flag
self.auto_include_flag = auto_include_flag
self.output_log_line = output_log_line
self.aminer_config = aminer_config
self.persistence_id = persistence_id
self.persistence_file_name = AMinerConfig.build_persistence_file_name(
aminer_config, self.__class__.__name__, persistence_id)
self.next_persist_time = None
self.load_persistency_data()
PersistencyUtil.add_persistable_component(self)
self.id_dict_current = {}
self.id_dict_old = {}
self.next_shift_time = None