def identify_cloudbursts(current_skyline_app, plot_graphs=False, log=False):
"""
Find significant changes (cloudbursts) in metrics.
"""
current_skyline_app_logger = current_skyline_app + 'Log'
current_logger = logging.getLogger(current_skyline_app_logger)
child_process_pid = os.getpid()
function_str = '%s :: functions.luminosity.identify_cloudbursts' % current_skyline_app
if log:
current_logger.info('%s :: running for process_pid - %s for %s' % (
function_str, str(child_process_pid), metric))
start = timer()
full_uniques = '%sunique_metrics' % settings.FULL_NAMESPACE
unique_metrics = list(redis_conn_decoded.smembers(full_uniques))
timer_start_all = timer()
custom_algorithm = 'm66'
m66_algorithm_source = '%%s/custom_algorithms/m66.py' % root_path
custom_algorithms = {}
custom_algorithms[custom_algorithm] = {
'algorithm_source': m66_algorithm_source,
'algorithm_parameters': {
'nth_median': 6, 'sigma': 6, 'window': 5, 'return_anomalies': True,
'save_plots_to': False, 'save_plots_to_absolute_dir': False,
'filename_prefix': False
},
'max_execution_time': 1.0,
'consensus': 1,
'algorithms_allowed_in_consensus': ['m66'],
'run_3sigma_algorithms': False,
'run_before_3sigma': False,
'run_only_if_consensus': False,
'use_with': ['crucible', 'luminosity'],
'debug_logging': False,
}
m66_candidate_metrics = {}
align = True
truncate_last_datapoint = True
window = 4
summarize_intervalString = '15min'
summarize_func = 'median'
nth_median = 6
n_sigma = 6
custom_algorithm = 'median_6_6sigma'
m66_candidate_metrics = {}
found = 0
now_timestamp = int(time())
check_last = 3600
candidate_metrics = {}
for metric in unique_metrics:
metric_name = metric
if metric_name.startswith(settings.FULL_NAMESPACE):
base_name = metric_name.replace(settings.FULL_NAMESPACE, '', 1)
else:
base_name = metric_name
timeseries = []
timeseries = get_metric_timeseries(skyline_app, metric, False)
if not timeseries:
continue
if truncate_last_datapoint:
timeseries_length = len(timeseries)
timeseries = timeseries[1:(timeseries_length - 2)]
for custom_algorithm in list(custom_algorithms.keys()):
custom_algorithms_dict = custom_algorithms[custom_algorithm]
custom_algorithm_dict = {}
custom_algorithm_dict['debug_logging'] = False
debug_algorithm_logging = False
if debug_algorithms:
custom_algorithm_dict['debug_logging'] = True
debug_algorithm_logging = True
algorithm_source = '/opt/skyline/github/skyline/skyline/custom_algorithms/%s.py' % algorithm
custom_algorithm_dict['algorithm_source'] = algorithm_source
if LUMINOSITY_CLASSIFY_ANOMALIES_SAVE_PLOTS:
custom_algorithm_dict['algorithm_parameters'] = {
'window': window, 'c': 6.0, 'return_anomalies': True,
'realtime_analysis': False,
'save_plots_to': metric_training_data_dir,
'save_plots_to_absolute_dir': True,
'filename_prefix': 'luminosity.classify_anomaly',
'debug_logging': debug_algorithm_logging,
}
custom_algorithm_dict['max_execution_time'] = 10.0
else:
custom_algorithm_dict['algorithm_parameters'] = {
'window': window, 'c': 6.0, 'return_anomalies': True,
'realtime_analysis': False,
'debug_logging': debug_algorithm_logging,
}
custom_algorithm_dict['max_execution_time'] = 5.0
if algorithm == base_algorithm:
if current_skyline_app == 'webapp':
anomalous, anomalyScore, anomalies, anomalies_dict = run_custom_algorithm_on_timeseries(current_skyline_app, current_pid, base_name, timeseries, custom_algorithm, custom_algorithm_dict, debug_algorithms)
result, anomalyScore, anomalies = run_custom_algorithm_on_timeseries(skyline_app, current_pid, base_name, timeseries, custom_algorithm, custom_algorithm_dict, debug_algorithms)
if return_anomalies:
return (anomalous, anomalyScore, anomalies)
else:
return (anomalous, anomalyScore)
else:
rolling_df = pd.DataFrame(timeseries, columns=['date', 'value'])
rolling_df['date'] = pd.to_datetime(rolling_df['date'], unit='s')
datetime_index = pd.DatetimeIndex(rolling_df['date'].values)
rolling_df = rolling_df.set_index(datetime_index)
rolling_df.drop('date', axis=1, inplace=True)
original_rolling_df = rolling_df.copy()
# MinMax scale
rolling_df = (rolling_df - rolling_df.min()) / (rolling_df.max() - rolling_df.min())
window = 6
data = rolling_df['value'].tolist()
s = pd.Series(data)
rolling_median_s = s.rolling(window).median()
median = rolling_median_s.tolist()
data = median
s = pd.Series(data)
rolling_median_s = s.rolling(window).median()
median_2 = rolling_median_s.tolist()
data = median_2
s = pd.Series(data)
rolling_median_s = s.rolling(window).median()
median_3 = rolling_median_s.tolist()
data = median_3
s = pd.Series(data)
rolling_median_s = s.rolling(window).median()
median_4 = rolling_median_s.tolist()
data = median_4
s = pd.Series(data)
rolling_median_s = s.rolling(window).median()
median_5 = rolling_median_s.tolist()
data = median_5
s = pd.Series(data)
rolling_median_s = s.rolling(window).median()
median_6 = rolling_median_s.tolist()
data = median_6
s = pd.Series(data)
rolling_std_s = s.rolling(window).std()
rolling_df['std_median_6'] = rolling_std_s.tolist()
std_median_6 = rolling_df['std_median_6'].fillna(0).tolist()
metric_stddev = np.std(std_median_6)
std_median_6_6sigma = []
anomalies = False
for value in std_median_6:
if value > (metric_stddev * 6):
std_median_6_6sigma.append(1)
anomalies = True
else:
std_median_6_6sigma.append(0)
rolling_df['std_median_6_6sigma'] = std_median_6_6sigma
if anomalies:
last_trigger = None
current_triggers = []
anomalies = []
# Only tag anomalous if the 6sigma triggers for window
for index, item in enumerate(timeseries):
if std_median_6_6sigma[index] == 1:
current_triggers.append(index)
else:
if len(current_triggers) > (window / 2):
for trigger_index in current_triggers:
anomalies.append(timeseries[(trigger_index - (window * 3))])
current_triggers = []
if anomalies:
anomalies_data = []
anomalies_timestamps = [int(item[0]) for item in anomalies]
for item in timeseries:
if int(item[0]) in anomalies_timestamps:
anomalies_data.append(1)
else:
anomalies_data.append(0)
rolling_df['anomalies'] = anomalies_data
m66_candidate_metrics[base_name] = {}
m66_candidate_metrics[base_name][custom_algorithm] = {}
m66_candidate_metrics[base_name][custom_algorithm]['anomalies'] = anomalies
# rolling_df['value'].plot(figsize=(18, 6), title=base_name)
title = '%s - median 6 6-sigma persisted' % base_name
# rolling_df['std_median_6_6sigma'].plot(figsize=(18, 6), title=title)
plot(original_rolling_df['value'], anomaly=rolling_df['anomalies'], anomaly_color='red', title=title)
def run_selected_batch_algorithm(timeseries, metric_name,
run_negatives_present):
"""
Filter timeseries and run selected algorithm.
"""
try:
from settings import BATCH_PROCESSING_STALE_PERIOD
# @modified 20200816 - Feature #3678: SNAB - anomalyScore
# Renamed to avoid confusion
# STALE_PERIOD = int(BATCH_PROCESSING_STALE_PERIOD)
BATCH_PROCESSING_STALE_PERIOD = int(BATCH_PROCESSING_STALE_PERIOD)
except:
BATCH_PROCESSING_STALE_PERIOD = 86400
# Get rid of short series
if len(timeseries) < MIN_TOLERABLE_LENGTH:
raise TooShort()
# Get rid of stale series
# @modified 20200816 - Feature #3678: SNAB - anomalyScore
# Renamed to avoid confusion
# if time() - timeseries[-1][0] > BATCH_PROCESSING_STALE_PERIOD:
if time() - timeseries[-1][0] > BATCH_PROCESSING_STALE_PERIOD:
raise Stale()
# Get rid of boring series
if len(set(item[1] for item in
timeseries[-MAX_TOLERABLE_BOREDOM:])) == BOREDOM_SET_SIZE:
raise Boring()
# RUN_OPTIMIZED_WORKFLOW - replaces the original ensemble method:
# ensemble = [globals()[algorithm](timeseries) for algorithm in ALGORITHMS]
# which runs all timeseries through all ALGORITHMS
final_ensemble = []
number_of_algorithms_triggered = 0
number_of_algorithms_run = 0
number_of_algorithms = len(ALGORITHMS)
maximum_false_count = number_of_algorithms - CONSENSUS + 1
# logger.info('the maximum_false_count is %s, above which CONSENSUS cannot be achieved' % (str(maximum_false_count)))
consensus_possible = True
time_all_algorithms = False
algorithm_tmp_file_prefix = '%s/%s.' % (SKYLINE_TMP_DIR, skyline_app)
# @added 20200607 - Feature #3566: custom_algorithms
algorithms_run = []
custom_consensus_override = False
custom_consensus_values = []
run_3sigma_algorithms = True
run_3sigma_algorithms_overridden_by = []
custom_algorithm = None
# @modified 20200817 - Bug #3652: Handle multiple metrics in base_name conversion
# base_name = metric_name.replace(FULL_NAMESPACE, '', 1)
if metric_name.startswith(FULL_NAMESPACE):
base_name = metric_name.replace(FULL_NAMESPACE, '', 1)
else:
base_name = metric_name
if CUSTOM_ALGORITHMS:
custom_algorithms_to_run = {}
try:
custom_algorithms_to_run = get_custom_algorithms_to_run(
skyline_app, base_name, CUSTOM_ALGORITHMS,
DEBUG_CUSTOM_ALGORITHMS)
if DEBUG_CUSTOM_ALGORITHMS:
if custom_algorithms_to_run:
logger.debug(
'algorithms :: debug :: custom algorithms ARE RUN on %s'
% (str(base_name)))
except:
logger.error('error :: get_custom_algorithms_to_run :: %s' %
traceback.format_exc())
custom_algorithms_to_run = {}
for custom_algorithm in custom_algorithms_to_run:
if consensus_possible:
algorithm = custom_algorithm
debug_logging = False
try:
debug_logging = custom_algorithms_to_run[custom_algorithm][
'debug_logging']
except:
debug_logging = False
if DEBUG_CUSTOM_ALGORITHMS:
debug_logging = True
if send_algorithm_run_metrics:
algorithm_count_file = '%s%s.count' % (
algorithm_tmp_file_prefix, algorithm)
algorithm_timings_file = '%s%s.timings' % (
algorithm_tmp_file_prefix, algorithm)
run_algorithm = []
run_algorithm.append(algorithm)
number_of_algorithms += 1
number_of_algorithms_run += 1
if send_algorithm_run_metrics:
start = timer()
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
logger.debug(
'debug :: algorithms :: running custom algorithm %s on %s'
% (str(algorithm), str(base_name)))
start_debug_timer = timer()
run_custom_algorithm_on_timeseries = None
try:
from custom_algorithms import run_custom_algorithm_on_timeseries
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
logger.debug(
'debug :: algorithms :: loaded run_custom_algorithm_on_timeseries'
)
except:
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
logger.error(traceback.format_exc())
logger.error(
'error :: algorithms :: failed to load run_custom_algorithm_on_timeseries'
)
result = None
anomalyScore = None
if run_custom_algorithm_on_timeseries:
try:
result, anomalyScore = run_custom_algorithm_on_timeseries(
skyline_app, getpid(), base_name, timeseries,
custom_algorithm,
custom_algorithms_to_run[custom_algorithm],
DEBUG_CUSTOM_ALGORITHMS)
algorithm_result = [result]
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
logger.debug(
'debug :: algorithms :: run_custom_algorithm_on_timeseries run with result - %s, anomalyScore - %s'
% (str(result), str(anomalyScore)))
except:
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
logger.error(traceback.format_exc())
logger.error(
'error :: algorithms :: failed to run custom_algorithm %s on %s'
% (custom_algorithm, base_name))
result = None
algorithm_result = [None]
else:
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
logger.error(
'error :: debug :: algorithms :: run_custom_algorithm_on_timeseries was not loaded so was not run'
)
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
end_debug_timer = timer()
logger.debug(
'debug :: algorithms :: ran custom algorithm %s on %s with result of (%s, %s) in %.6f seconds'
% (str(algorithm), str(base_name), str(result),
str(anomalyScore),
(end_debug_timer - start_debug_timer)))
algorithms_run.append(algorithm)
if send_algorithm_run_metrics:
end = timer()
with open(algorithm_count_file, 'a') as f:
f.write('1\n')
with open(algorithm_timings_file, 'a') as f:
f.write('%.6f\n' % (end - start))
else:
algorithm_result = [None]
algorithms_run.append(algorithm)
if algorithm_result.count(True) == 1:
result = True
number_of_algorithms_triggered += 1
elif algorithm_result.count(False) == 1:
result = False
elif algorithm_result.count(None) == 1:
result = None
else:
result = False
final_ensemble.append(result)
custom_consensus = None
algorithms_allowed_in_consensus = []
# @added 20200605 - Feature #3566: custom_algorithms
# Allow only single or multiple custom algorithms to run and allow
# the a custom algorithm to specify not to run 3sigma aglorithms
custom_run_3sigma_algorithms = True
try:
custom_run_3sigma_algorithms = custom_algorithms_to_run[
custom_algorithm]['run_3sigma_algorithms']
except:
custom_run_3sigma_algorithms = True
if not custom_run_3sigma_algorithms and result:
run_3sigma_algorithms = False
run_3sigma_algorithms_overridden_by.append(custom_algorithm)
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
logger.debug(
'debug :: algorithms :: run_3sigma_algorithms is False on %s for %s'
% (custom_algorithm, base_name))
if result:
try:
custom_consensus = custom_algorithms_to_run[
custom_algorithm]['consensus']
if custom_consensus == 0:
custom_consensus = int(CONSENSUS)
else:
custom_consensus_values.append(custom_consensus)
except:
custom_consensus = int(CONSENSUS)
try:
algorithms_allowed_in_consensus = custom_algorithms_to_run[
custom_algorithm]['algorithms_allowed_in_consensus']
except:
algorithms_allowed_in_consensus = []
if custom_consensus == 1:
consensus_possible = False
custom_consensus_override = True
logger.info(
'algorithms :: overidding the CONSENSUS as custom algorithm %s overides on %s'
% (str(algorithm), str(base_name)))
# TODO - figure out how to handle consensus overrides if
# multiple custom algorithms are used
if DEBUG_CUSTOM_ALGORITHMS:
if not run_3sigma_algorithms:
logger.debug('algorithms :: not running 3 sigma algorithms')
if len(run_3sigma_algorithms_overridden_by) > 0:
logger.debug(
'algorithms :: run_3sigma_algorithms overridden by %s' %
(str(run_3sigma_algorithms_overridden_by)))
# @added 20200425 - Feature #3508: ionosphere.untrainable_metrics
# Added negatives_found
negatives_found = False
# @added 20200817 - Feature #3684: ROOMBA_BATCH_METRICS_CUSTOM_DURATIONS
# Feature #3650: ROOMBA_DO_NOT_PROCESS_BATCH_METRICS
# Feature #3480: batch_processing
# Feature #3678: SNAB - anomalyScore
# Allow for custom durations on namespaces
use_full_duration = int(FULL_DURATION) + 0
if ROOMBA_BATCH_METRICS_CUSTOM_DURATIONS:
for metric_namespace, custom_full_duration in ROOMBA_BATCH_METRICS_CUSTOM_DURATIONS:
if metric_namespace in base_name:
use_full_duration = custom_full_duration
detect_drop_off_cliff_trigger = False
for algorithm in ALGORITHMS:
# @modified 20200607 - Feature #3566: custom_algorithms
# Added run_3sigma_algorithms to allow only single or multiple custom
# algorithms to run and allow the a custom algorithm to specify not to
# run 3sigma aglorithms.
# if consensus_possible:
if consensus_possible and run_3sigma_algorithms:
if send_algorithm_run_metrics:
algorithm_count_file = '%s%s.count' % (
algorithm_tmp_file_prefix, algorithm)
algorithm_timings_file = '%s%s.timings' % (
algorithm_tmp_file_prefix, algorithm)
run_algorithm = []
run_algorithm.append(algorithm)
number_of_algorithms_run += 1
if send_algorithm_run_metrics:
start = timer()
try:
# @added 20200817 - Feature #3684: ROOMBA_BATCH_METRICS_CUSTOM_DURATIONS
# Feature #3650: ROOMBA_DO_NOT_PROCESS_BATCH_METRICS
# Feature #3480: batch_processing
# Feature #3678: SNAB - anomalyScore
# Allow for custom durations on namespaces
# algorithm_result = [globals()[test_algorithm](timeseries) for test_algorithm in run_algorithm]
algorithm_result = [
globals()[test_algorithm](timeseries, use_full_duration)
for test_algorithm in run_algorithm
]
except:
# logger.error('%s failed' % (algorithm))
algorithm_result = [None]
# @added 20200607 - Feature #3566: custom_algorithms
algorithms_run.append(algorithm)
if send_algorithm_run_metrics:
end = timer()
with open(algorithm_count_file, 'a') as f:
f.write('1\n')
with open(algorithm_timings_file, 'a') as f:
f.write('%.6f\n' % (end - start))
else:
algorithm_result = [None]
algorithms_run.append(algorithm)
if algorithm_result.count(True) == 1:
result = True
number_of_algorithms_triggered += 1
# logger.info('algorithm %s triggerred' % (str(algorithm)))
elif algorithm_result.count(False) == 1:
result = False
elif algorithm_result.count(None) == 1:
result = None
else:
result = False
final_ensemble.append(result)
if not RUN_OPTIMIZED_WORKFLOW:
continue
if time_all_algorithms:
continue
if ENABLE_ALL_ALGORITHMS_RUN_METRICS:
continue
# true_count = final_ensemble.count(True)
# false_count = final_ensemble.count(False)
# logger.info('current false_count %s' % (str(false_count)))
if final_ensemble.count(False) >= maximum_false_count:
consensus_possible = False
# logger.info('CONSENSUS cannot be reached as %s algorithms have already not been triggered' % (str(false_count)))
# skip_algorithms_count = number_of_algorithms - number_of_algorithms_run
# logger.info('skipping %s algorithms' % (str(skip_algorithms_count)))
# logger.info('final_ensemble: %s' % (str(final_ensemble)))
try:
# ensemble = [globals()[algorithm](timeseries) for algorithm in ALGORITHMS]
ensemble = final_ensemble
# @modified 20200607 - Feature #3566: custom_algorithms
# threshold = len(ensemble) - CONSENSUS
if custom_consensus_override:
threshold = len(ensemble) - 1
else:
threshold = len(ensemble) - CONSENSUS
if ensemble.count(False) <= threshold:
# @added 20200425 - Feature #3508: ionosphere.untrainable_metrics
# Only run a negatives_present check if it is anomalous, there
# is no need to check unless it is related to an anomaly
if run_negatives_present:
try:
# @added 20200817 - Feature #3684: ROOMBA_BATCH_METRICS_CUSTOM_DURATIONS
# Feature #3650: ROOMBA_DO_NOT_PROCESS_BATCH_METRICS
# Feature #3480: batch_processing
# Feature #3678: SNAB - anomalyScore
# Allow for custom durations on namespaces
# negatives_found = negatives_present(timeseries)
negatives_found = negatives_present(
timeseries, use_full_duration)
except:
logger.error('Algorithm error: negatives_present :: %s' %
traceback.format_exc())
negatives_found = False
# @modified 20200425 - Feature #3508: ionosphere.untrainable_metrics
# return True, ensemble, timeseries[-1][1]
# @modified 20200607 - Feature #3566: custom_algorithms
# Added algorithms_run
# return True, ensemble, timeseries[-1][1], negatives_found
# @modified 20200815 - Feature #3678: SNAB - anomalyScore
# Added the number_of_algorithms to calculate anomalyScore from
# return True, ensemble, timeseries[-1][1], negatives_found, algorithms_run
return True, ensemble, timeseries[-1][
1], negatives_found, algorithms_run, number_of_algorithms
# @modified 20200425 - Feature #3508: ionosphere.untrainable_metrics
# return False, ensemble, timeseries[-1][1]
# @modified 20200607 - Feature #3566: custom_algorithms
# Added algorithms_run
# @modified 20200815 - Feature #3678: SNAB - anomalyScore
# Added the number_of_algorithms to calculate anomalyScore from
# return False, ensemble, timeseries[-1][1], negatives_found, algorithms_run
return False, ensemble, timeseries[-1][
1], negatives_found, algorithms_run, number_of_algorithms
except:
logger.error('Algorithm error: %s' % traceback.format_exc())
# @modified 20200425 - Feature #3508: ionosphere.untrainable_metrics
# return False, [], 1
# @modified 20200607 - Feature #3566: custom_algorithms
# Added algorithms_run
# return False, ensemble, timeseries[-1][1], negatives_found, algorithms_run
# @modified 20200815 - Feature #3678: SNAB - anomalyScore
# Added the number_of_algorithms to calculate anomalyScore from
# return False, [], 1, negatives_found, algorithms_run
return False, [], 1, negatives_found, algorithms_run, 0
custom_algorithm_dict['max_execution_time'] = 10.0
debug_algorithms = False
timer_start = timer()
for base_name in current_base_names:
try:
timeseries = metrics_timeseries[base_name]['timeseries']
except KeyError:
continue
if not timeseries:
continue
result = None
anomalyScore = None
anomalies = []
try:
result, anomalyScore, anomalies = run_custom_algorithm_on_timeseries(skyline_app, current_pid, base_name, timeseries, custom_algorithm, custom_algorithm_dict, debug_algorithms)
if result:
print('%s - result: %s, anomalyScore: %s, anomalies: %s' % (
base_name, str(result), str(anomalyScore), str(len(anomalies))))
except Exception as e:
print('error :: failed to run custom_algorithm %s on %s - %s' % (
custom_algorithm, base_name, e))
triggered = False
if anomalies:
found += 1
found_level_shifts[base_name] = {}
found_level_shifts[base_name]['anomalyScore'] = anomalyScore
found_level_shifts[base_name]['anomalies'] = anomalies
more_analysis_metrics_timeseries[base_name] = {}
more_analysis_metrics_timeseries[base_name]['timeseries'] = timeseries
timer_end = timer()
def run_selected_algorithm(timeseries, metric_name,
second_order_resolution_seconds,
run_negatives_present):
"""
Run selected algorithms
"""
# @added 20200425 - Feature #3508: ionosphere.untrainable_metrics
# Added negatives_found for run_negatives_present
negatives_found = False
# @added 20200607 - Feature #3566: custom_algorithms
final_custom_ensemble = []
algorithms_run = []
custom_consensus_override = False
custom_consensus_values = []
run_3sigma_algorithms = True
run_3sigma_algorithms_overridden_by = []
custom_algorithm = None
# @added 20201125 - Feature #3848: custom_algorithms - run_before_3sigma parameter
run_custom_algorithm_after_3sigma = False
final_after_custom_ensemble = []
custom_algorithm_not_anomalous = False
if CUSTOM_ALGORITHMS:
base_name = metric_name.replace(FULL_NAMESPACE, '', 1)
custom_algorithms_to_run = {}
try:
custom_algorithms_to_run = get_custom_algorithms_to_run(
skyline_app, base_name, CUSTOM_ALGORITHMS,
DEBUG_CUSTOM_ALGORITHMS)
if DEBUG_CUSTOM_ALGORITHMS:
if custom_algorithms_to_run:
logger.debug(
'algorithms :: debug :: custom algorithms ARE RUN on %s'
% (str(base_name)))
except:
logger.error('error :: get_custom_algorithms_to_run :: %s' %
traceback.format_exc())
custom_algorithms_to_run = {}
for custom_algorithm in custom_algorithms_to_run:
debug_logging = False
try:
debug_logging = custom_algorithms_to_run[custom_algorithm][
'debug_logging']
except:
debug_logging = False
if DEBUG_CUSTOM_ALGORITHMS:
debug_logging = True
# @added 20201125 - Feature #3848: custom_algorithms - run_before_3sigma parameter
run_before_3sigma = True
try:
run_before_3sigma = custom_algorithms_to_run[custom_algorithm][
'run_before_3sigma']
except:
run_before_3sigma = True
if not run_before_3sigma:
run_custom_algorithm_after_3sigma = True
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
logger.debug(
'debug :: algorithms :: NOT running custom algorithm %s on %s BEFORE three-sigma algorithms'
% (str(custom_algorithm), str(base_name)))
continue
run_algorithm = []
run_algorithm.append(custom_algorithm)
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
logger.debug(
'debug :: algorithms :: running custom algorithm %s on %s'
% (str(custom_algorithm), str(base_name)))
start_debug_timer = timer()
run_custom_algorithm_on_timeseries = None
try:
from custom_algorithms import run_custom_algorithm_on_timeseries
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
logger.debug(
'debug :: algorithms :: loaded run_custom_algorithm_on_timeseries'
)
except:
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
logger.error(traceback.format_exc())
logger.error(
'error :: algorithms :: failed to load run_custom_algorithm_on_timeseries'
)
result = None
anomalyScore = None
if run_custom_algorithm_on_timeseries:
try:
result, anomalyScore = run_custom_algorithm_on_timeseries(
skyline_app, getpid(), base_name, timeseries,
custom_algorithm,
custom_algorithms_to_run[custom_algorithm],
DEBUG_CUSTOM_ALGORITHMS)
algorithm_result = [result]
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
logger.debug(
'debug :: algorithms :: run_custom_algorithm_on_timeseries run with result - %s, anomalyScore - %s'
% (str(result), str(anomalyScore)))
except:
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
logger.error(traceback.format_exc())
logger.error(
'error :: algorithms :: failed to run custom_algorithm %s on %s'
% (custom_algorithm, base_name))
result = None
algorithm_result = [None]
else:
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
logger.error(
'error :: debug :: algorithms :: run_custom_algorithm_on_timeseries was not loaded so was not run'
)
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
end_debug_timer = timer()
logger.debug(
'debug :: algorithms :: ran custom algorithm %s on %s with result of (%s, %s) in %.6f seconds'
% (str(custom_algorithm), str(base_name), str(result),
str(anomalyScore),
(end_debug_timer - start_debug_timer)))
algorithms_run.append(custom_algorithm)
if algorithm_result.count(True) == 1:
result = True
elif algorithm_result.count(False) == 1:
result = False
elif algorithm_result.count(None) == 1:
result = None
else:
result = False
final_custom_ensemble.append(result)
custom_consensus = None
algorithms_allowed_in_consensus = []
# @added 20200605 - Feature #3566: custom_algorithms
# Allow only single or multiple custom algorithms to run and allow
# the a custom algorithm to specify not to run 3sigma aglorithms
custom_run_3sigma_algorithms = True
try:
custom_run_3sigma_algorithms = custom_algorithms_to_run[
custom_algorithm]['run_3sigma_algorithms']
except:
custom_run_3sigma_algorithms = True
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
logger.debug(
'debug :: error - algorithms :: could not determine custom_run_3sigma_algorithms - default to True'
)
if not custom_run_3sigma_algorithms and result:
run_3sigma_algorithms = False
run_3sigma_algorithms_overridden_by.append(custom_algorithm)
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
logger.debug(
'debug :: algorithms :: run_3sigma_algorithms is False on %s for %s'
% (custom_algorithm, base_name))
else:
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
logger.debug(
'debug :: algorithms :: run_3sigma_algorithms will now be run on %s - %s'
% (base_name, str(custom_run_3sigma_algorithms)))
if result:
try:
custom_consensus = custom_algorithms_to_run[
custom_algorithm]['consensus']
if custom_consensus == 0:
custom_consensus = int(MIRAGE_CONSENSUS)
else:
custom_consensus_values.append(custom_consensus)
except:
custom_consensus = int(MIRAGE_CONSENSUS)
try:
algorithms_allowed_in_consensus = custom_algorithms_to_run[
custom_algorithm]['algorithms_allowed_in_consensus']
except:
algorithms_allowed_in_consensus = []
if custom_consensus == 1:
custom_consensus_override = True
logger.info(
'algorithms :: overidding the CONSENSUS as custom algorithm %s overides on %s'
% (str(custom_algorithm), str(base_name)))
# TODO - figure out how to handle consensus overrides if
# multiple custom algorithms are used
if DEBUG_CUSTOM_ALGORITHMS:
if not run_3sigma_algorithms:
logger.debug('algorithms :: not running 3 sigma algorithms')
if len(run_3sigma_algorithms_overridden_by) > 0:
logger.debug(
'algorithms :: run_3sigma_algorithms overridden by %s' %
(str(run_3sigma_algorithms_overridden_by)))
# @added 20200607 - Feature #3566: custom_algorithms
# @modified 20201120 - Feature #3566: custom_algorithms
# ensemble = []
ensemble = final_custom_ensemble
# @modified 20201125 - Feature #3848: custom_algorithms - run_before_3sigma parameter
# Check run_3sigma_algorithms as well to the conditional
# if not custom_consensus_override:
if run_3sigma_algorithms and not custom_consensus_override:
if DEBUG_CUSTOM_ALGORITHMS:
logger.debug('debug :: running three-sigma algorithms')
try:
ensemble = [
globals()[algorithm](timeseries,
second_order_resolution_seconds)
for algorithm in MIRAGE_ALGORITHMS
]
for algorithm in MIRAGE_ALGORITHMS:
algorithms_run.append(algorithm)
except:
logger.error('Algorithm error: %s' % traceback.format_exc())
# @modified 20200425 - Feature #3508: ionosphere.untrainable_metrics
# Added negatives_found
# return False, [], 1
# @modified 20200607 - Feature #3566: custom_algorithms
# Added algorithms_run
return False, [], 1, False, algorithms_run
# @added 20201124 - Feature #3566: custom_algorithms
if final_custom_ensemble:
ensemble = final_custom_ensemble + ensemble
else:
for algorithm in MIRAGE_ALGORITHMS:
ensemble.append(None)
# @added 20201120 - Feature #3566: custom_algorithms
# If 3sigma algorithms have not been run discard the MIRAGE_ALGORITHMS
# added above
if not run_3sigma_algorithms:
ensemble = final_custom_ensemble
# @added 20201125 - Feature #3848: custom_algorithms - run_before_3sigma parameter
if run_custom_algorithm_after_3sigma:
if DEBUG_CUSTOM_ALGORITHMS:
logger.debug(
'debug :: checking custom algorithms to run AFTER three-sigma algorithms'
)
for custom_algorithm in custom_algorithms_to_run:
debug_logging = False
try:
debug_logging = custom_algorithms_to_run[custom_algorithm][
'debug_logging']
except:
debug_logging = False
if DEBUG_CUSTOM_ALGORITHMS:
debug_logging = True
run_before_3sigma = True
try:
run_before_3sigma = custom_algorithms_to_run[custom_algorithm][
'run_before_3sigma']
except:
run_before_3sigma = True
if run_before_3sigma:
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
logger.debug(
'debug :: algorithms :: NOT running custom algorithm %s on %s AFTER three-sigma algorithms as run_before_3sigma is %s'
% (str(custom_algorithm), str(base_name),
str(run_before_3sigma)))
continue
try:
custom_consensus = custom_algorithms_to_run[custom_algorithm][
'consensus']
if custom_consensus == 0:
custom_consensus = int(MIRAGE_CONSENSUS)
else:
custom_consensus_values.append(custom_consensus)
except:
custom_consensus = int(MIRAGE_CONSENSUS)
run_only_if_consensus = False
try:
run_only_if_consensus = custom_algorithms_to_run[
custom_algorithm]['run_only_if_consensus']
except:
run_only_if_consensus = False
if run_only_if_consensus:
if ensemble.count(True) < int(MIRAGE_CONSENSUS):
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
logger.debug(
'debug :: algorithms :: NOT running custom algorithm %s on %s AFTER three-sigma algorithms as only %s three-sigma algorithms triggered - MIRAGE_CONSENSUS of %s not achieved'
%
(str(custom_algorithm), str(base_name),
str(ensemble.count(True)), str(MIRAGE_CONSENSUS)))
continue
else:
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
logger.debug(
'debug :: algorithms :: running custom algorithm %s on %s AFTER three-sigma algorithms as %s three-sigma algorithms triggered - MIRAGE_CONSENSUS of %s was achieved'
%
(str(custom_algorithm), str(base_name),
str(ensemble.count(True)), str(MIRAGE_CONSENSUS)))
run_algorithm = []
run_algorithm.append(custom_algorithm)
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
logger.debug(
'debug :: algorithms :: running custom algorithm %s on %s'
% (str(custom_algorithm), str(base_name)))
start_debug_timer = timer()
run_custom_algorithm_on_timeseries = None
try:
from custom_algorithms import run_custom_algorithm_on_timeseries
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
logger.debug(
'debug :: algorithms :: loaded run_custom_algorithm_on_timeseries'
)
except:
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
logger.error(traceback.format_exc())
logger.error(
'error :: algorithms :: failed to load run_custom_algorithm_on_timeseries'
)
result = None
anomalyScore = None
if run_custom_algorithm_on_timeseries:
try:
result, anomalyScore = run_custom_algorithm_on_timeseries(
skyline_app, getpid(), base_name, timeseries,
custom_algorithm,
custom_algorithms_to_run[custom_algorithm],
DEBUG_CUSTOM_ALGORITHMS)
algorithm_result = [result]
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
logger.debug(
'debug :: algorithms :: run_custom_algorithm_on_timeseries run with result - %s, anomalyScore - %s'
% (str(result), str(anomalyScore)))
except:
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
logger.error(traceback.format_exc())
logger.error(
'error :: algorithms :: failed to run custom_algorithm %s on %s'
% (custom_algorithm, base_name))
result = None
algorithm_result = [None]
else:
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
logger.error(
'error :: debug :: algorithms :: run_custom_algorithm_on_timeseries was not loaded so was not run'
)
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
end_debug_timer = timer()
logger.debug(
'debug :: algorithms :: ran custom algorithm %s on %s with result of (%s, %s) in %.6f seconds'
% (str(custom_algorithm), str(base_name), str(result),
str(anomalyScore),
(end_debug_timer - start_debug_timer)))
algorithms_run.append(custom_algorithm)
if algorithm_result.count(True) == 1:
result = True
elif algorithm_result.count(False) == 1:
result = False
elif algorithm_result.count(None) == 1:
result = None
else:
result = False
final_after_custom_ensemble.append(result)
algorithms_allowed_in_consensus = []
# custom_run_3sigma_algorithms = True does not need to be checked
# here as if three-sigma algorithms have run they have already run
# at this point, unlike above in the run_before_3sigma custom
# algorithms run
if result:
try:
algorithms_allowed_in_consensus = custom_algorithms_to_run[
custom_algorithm]['algorithms_allowed_in_consensus']
except:
algorithms_allowed_in_consensus = []
if custom_consensus == 1:
custom_consensus_override = True
logger.info(
'algorithms :: overidding the CONSENSUS as custom algorithm %s overides on %s'
% (str(custom_algorithm), str(base_name)))
else:
# @added 20201127 - Feature #3566: custom_algorithms
# Handle if the result is None
if result is None:
logger.warn(
'warning :: algorithms :: %s failed to run on %s' %
(str(custom_algorithm), str(base_name)))
else:
if custom_consensus == 1:
# hmmm we are required to hack threshold here
custom_algorithm_not_anomalous = True
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
logger.debug(
'debug :: algorithms :: %s did not trigger - custom_algorithm_not_anomalous set to identify as not anomalous'
% (str(custom_algorithm)))
for item in final_after_custom_ensemble:
ensemble.append(item)
# @modified 20200607 - Feature #3566: custom_algorithms
try:
# threshold = len(ensemble) - MIRAGE_CONSENSUS
if custom_consensus_override:
threshold = len(ensemble) - 1
else:
threshold = len(ensemble) - MIRAGE_CONSENSUS
# @modified 20201125 - Feature #3848: custom_algorithms - run_before_3sigma parameter
# Added and not custom_algorithm_not_anomalous
if ensemble.count(
False) <= threshold and not custom_algorithm_not_anomalous:
# @added 20200425 - Feature #3508: ionosphere.untrainable_metrics
# Only run a negatives_present check if it is anomalous, there
# is no need to check unless it is related to an anomaly
if run_negatives_present:
try:
negatives_found = negatives_present(timeseries)
if negatives_found:
number_of_negatives_found = len(negatives_found)
else:
number_of_negatives_found = 0
logger.info('%s negative values found for %s' %
(str(number_of_negatives_found), metric_name))
except:
logger.error('Algorithm error: negatives_present :: %s' %
traceback.format_exc())
negatives_found = False
if MIRAGE_ENABLE_SECOND_ORDER:
if is_anomalously_anomalous(metric_name, ensemble,
timeseries[-1][1]):
# @modified 20200425 - Feature #3508: ionosphere.untrainable_metrics
# Added negatives_found
# return True, ensemble, timeseries[-1][1]
# @modified 20200607 - Feature #3566: custom_algorithms
# Added algorithms_run
return True, ensemble, timeseries[-1][
1], negatives_found, algorithms_run
else:
# @modified 20200425 - Feature #3508: ionosphere.untrainable_metrics
# Added negatives_found
# return True, ensemble, timeseries[-1][1]
# @modified 20200607 - Feature #3566: custom_algorithms
# Added algorithms_run
return True, ensemble, timeseries[-1][
1], negatives_found, algorithms_run
# @modified 20200425 - Feature #3508: ionosphere.untrainable_metrics
# Added negatives_found
# return False, ensemble, timeseries[-1][1]
# @modified 20200607 - Feature #3566: custom_algorithms
# Added algorithms_run
return False, ensemble, timeseries[-1][
1], negatives_found, algorithms_run
except:
logger.error('Algorithm error: %s' % traceback.format_exc())
# @modified 20200425 - Feature #3508: ionosphere.untrainable_metrics
# Added negatives_found
# return False, [], 1
# @modified 20200607 - Feature #3566: custom_algorithms
# Added algorithms_run
return False, [], 1, False, algorithms_run
def classify_anomalies(i, classify_anomalies_set, start_timestamp,
classify_for):
logger = logging.getLogger(skyline_app_logger)
debug_algorithms = False
logger.info('classify_anomalies :: with start_timestamp - %s' %
str(start_timestamp))
start_classify_anomalies = timer()
def mysql_insert(insert):
"""
Insert data into mysql table
:param insert: the insert string
:type insert: str
:return: int
:rtype: int or boolean
- **Example usage**::
query = 'insert into host (host) VALUES (\'this_host\')'
result = self.mysql_insert(query)
.. note::
- If the MySQL query fails a boolean will be returned not a tuple
* ``False``
* ``None``
"""
try:
cnx = mysql.connector.connect(**config)
except mysql.connector.Error as err:
logger.error('error :: classify_anomalies :: mysql error - %s' %
str(err))
logger.error(
'error :: classify_anomalies :: failed to connect to mysql')
raise
if cnx:
try:
cursor = cnx.cursor()
cursor.execute(insert)
inserted_id = cursor.rowcount
# Make sure data is committed to the database
cnx.commit()
cursor.close()
cnx.close()
return inserted_id
except mysql.connector.Error as err:
logger.error(
'error :: classify_anomalies :: failed to insert record - mysql error - %s'
% str(err))
cnx.close()
raise
else:
cnx.close()
return False
return False
# Handle luminosity running with multiple processes
def manage_processing_key(current_pid, base_name, timestamp, classify_for,
action):
result = False
processing_key = 'luminosity.classify_anomalies.processing.%s.%s' % (
str(timestamp), str(base_name))
if action == 'add':
key_exists = None
try:
key_exists = redis_conn_decoded.get(processing_key)
if key_exists:
result = False
return result
except:
logger.error(traceback.format_exc())
logger.error(
'error :: classify_anomalies :: failed to query Redis for %s'
% (processing_key))
try:
data = {'pid': current_pid, 'timestamp': int(time())}
redis_conn.setex(processing_key, classify_for, str(data))
result = True
logger.info(
'classify_anomalies :: managing %s added %s with %s' %
(str(base_name), processing_key, str(data)))
except:
logger.error(traceback.format_exc())
logger.error(
'error :: classify_anomalies :: failed to create key %s' %
(processing_key))
if action == 'remove':
try:
redis_conn.delete(processing_key)
result = True
except:
logger.error(traceback.format_exc())
logger.error(
'error :: classify_anomalies :: failed to remove key %s' %
(processing_key))
return result
classify_anomalies_list = []
for classify_anomaly in classify_anomalies_set:
classify_anomalies_list.append(literal_eval(classify_anomaly))
if classify_anomalies_list:
classify_anomalies_list = sorted(classify_anomalies_list,
key=lambda x: x[2],
reverse=False)
current_pid = getpid()
anomalies_proceessed = 0
for classify_anomaly in classify_anomalies_list:
anomaly_data_dict = classify_anomaly[3]
base_name = anomaly_data_dict['metric']
timestamp = anomaly_data_dict['timestamp']
# logger.debug('debug :: classify_anomalies :: %s' % str(classify_anomaly))
anomalies_proceessed += 1
# Handle luminosity running with multiple processes
manage_metric = False
try:
manage_metric = manage_processing_key(current_pid, base_name,
timestamp, classify_for,
'add')
except:
logger.error(traceback.format_exc())
logger.error(
'error :: classify_anomalies :: failed to run manage_processing_key'
)
if not manage_metric:
logger.info(
'classify_anomalies :: skipping as processing key exists for %s'
% base_name)
continue
# Remove anomaly if not classified in 1800 seconds
if (int(time()) - 1800) > int(anomaly_data_dict['added_at']):
logger.info(
'classify_anomalies :: anomaly not classified in 1800 seocnds, removing from luminosity.classify_anomalies'
)
try:
redis_conn.srem('luminosity.classify_anomalies',
str(classify_anomaly))
logger.info(
'classify_anomalies :: removed %s, %s, %s item from luminosity.classify_anomalies Redis set'
% (base_name, str(timestamp), anomaly_data_dict['app']))
except:
logger.error(traceback.format_exc())
logger.error(
'error :: classify_anomalies :: failed to remove %s, %s, %s item from luminosity.classify_anomalies Redis set'
% (base_name, str(timestamp), anomaly_data_dict['app']))
try:
manage_processing_key(current_pid, base_name, timestamp,
classify_for, 'remove')
except:
logger.error(traceback.format_exc())
logger.error(
'error :: classify_anomalies :: failed to run manage_processing_key - %s'
% base_name)
continue
metric_timeseries_dir = base_name.replace('.', '/')
metric_training_data_dir = '%s/%s/%s' % (
settings.IONOSPHERE_DATA_FOLDER, timestamp, metric_timeseries_dir)
anomaly_json = '%s/%s.json' % (metric_training_data_dir, base_name)
timeseries = []
# Try load training data
if os.path.isfile(anomaly_json):
logger.info('classify_anomalies :: anomaly_json found - %s' %
anomaly_json)
try:
with open((anomaly_json), 'r') as f:
raw_timeseries = f.read()
timeseries_array_str = str(raw_timeseries).replace(
'(', '[').replace(')', ']')
del raw_timeseries
timeseries = literal_eval(timeseries_array_str)
del timeseries_array_str
except:
logger.error(traceback.format_exc())
logger.error(
'error :: classify_anomalies :: could not create timeseries from anomaly json %s'
% anomaly_json)
logger.info(
'classify_anomalies :: timeseries from anomaly_json has %s datapoints'
% str(len(timeseries)))
else:
logger.info(
'classify_anomalies :: no anomaly_json not found removing %s from luminosity.classify_anomalies Redis set'
% (base_name))
try:
redis_conn.srem('luminosity.classify_anomalies',
str(classify_anomaly))
logger.info(
'classify_anomalies :: removed %s, %s, %s item from luminosity.classify_anomalies Redis set'
% (base_name, str(timestamp), anomaly_data_dict['app']))
except:
logger.error(traceback.format_exc())
logger.error(
'error :: classify_anomalies :: failed to remove %s, %s, %s item from luminosity.classify_anomalies Redis set'
% (base_name, str(timestamp), anomaly_data_dict['app']))
if not timeseries:
try:
manage_processing_key(current_pid, base_name, timestamp,
classify_for, 'remove')
except:
logger.error(traceback.format_exc())
logger.error(
'error :: classify_anomalies :: failed to run manage_processing_key - %s'
% base_name)
continue
# Classify anomaly or continue classifying metric
window = 5
window_timestamps = [ts for ts, value in timeseries[-window:]]
algorithms_to_process = len(LUMINOSITY_CLASSIFY_ANOMALY_ALGORITHMS)
algorithms_processed = 0
algorithm_results = {}
algorithms_processed_key = 'luminosity.classify_anomalies.algorithms_processed.%s.%s' % (
str(timestamp), str(base_name))
try:
algorithm_results = redis_conn_decoded.get(
algorithms_processed_key)
if not algorithm_results:
algorithm_results = {}
else:
algorithm_results = literal_eval(algorithm_results)
except:
logger.error(traceback.format_exc())
logger.error(
'error :: classify_anomalies :: failed to query Redis for %s' %
(algorithms_processed_key))
if not algorithm_results:
for algorithm in LUMINOSITY_CLASSIFY_ANOMALY_ALGORITHMS:
algorithm_results[algorithm] = {}
algorithm_results[algorithm]['processed'] = False
algorithm_results[algorithm]['result'] = None
try:
redis_conn.setex(algorithms_processed_key, 300,
str(algorithm_results))
except:
logger.error(traceback.format_exc())
logger.error(
'error :: classify_anomalies :: failed to set Redis key %s'
% (algorithms_processed_key))
for algorithm in LUMINOSITY_CLASSIFY_ANOMALY_ALGORITHMS:
if algorithm_results[algorithm]['processed']:
algorithms_processed += 1
logger.info(
'classify_anomalies :: %s at %s already processed with %s with result %s'
% (str(base_name), str(timestamp), algorithm,
str(algorithm_results[algorithm]['result'])))
continue
custom_algorithm = algorithm
custom_algorithm_dict = {}
custom_algorithm_dict['debug_logging'] = False
debug_algorithm_logging = False
if debug_algorithms:
custom_algorithm_dict['debug_logging'] = True
debug_algorithm_logging = True
algorithm_source = '/opt/skyline/github/skyline/skyline/custom_algorithms/%s.py' % algorithm
custom_algorithm_dict['algorithm_source'] = algorithm_source
if LUMINOSITY_CLASSIFY_ANOMALIES_SAVE_PLOTS:
custom_algorithm_dict['algorithm_parameters'] = {
'window': window,
'c': 6.0,
'return_anomalies': True,
'realtime_analysis': False,
'save_plots_to': metric_training_data_dir,
'save_plots_to_absolute_dir': True,
'filename_prefix': 'luminosity.classify_anomaly',
'debug_logging': debug_algorithm_logging,
}
custom_algorithm_dict['max_execution_time'] = 10.0
else:
custom_algorithm_dict['algorithm_parameters'] = {
'window': window,
'c': 6.0,
'return_anomalies': True,
'realtime_analysis': False,
'debug_logging': debug_algorithm_logging,
}
custom_algorithm_dict['max_execution_time'] = 5.0
result = None
anomalyScore = None
anomalies = []
try:
result, anomalyScore, anomalies = run_custom_algorithm_on_timeseries(
skyline_app, current_pid, base_name, timeseries,
custom_algorithm, custom_algorithm_dict, debug_algorithms)
logger.info(
'classify_anomalies :: run_custom_algorithm_on_timeseries run %s on %s with result - %s, anomalyScore - %s'
% (custom_algorithm, base_name, str(result),
str(anomalyScore)))
except:
logger.error(traceback.format_exc())
logger.error(
'error :: classify_anomalies :: failed to run custom_algorithm %s on %s'
% (custom_algorithm, base_name))
triggered = False
if anomalies:
anomalies.reverse()
for ts, value in anomalies:
if ts in window_timestamps:
triggered = True
break
if ts < window_timestamps[0]:
break
if triggered:
logger.info(
'classify_anomalies :: %s triggered on %s within the window at %s'
% (custom_algorithm, base_name, str(ts)))
else:
logger.info(
'classify_anomalies :: %s did not trigger on %s within the window'
% (custom_algorithm, base_name))
algorithm_results[algorithm]['processed'] = True
algorithm_results[algorithm]['result'] = triggered
try:
redis_conn.setex(algorithms_processed_key, 300,
str(algorithm_results))
except:
logger.error(traceback.format_exc())
logger.error(
'error :: classify_anomalies :: failed to set Redis key %s'
% (algorithms_processed_key))
algorithms_processed += 1
time_now = time()
runtime = time_now - start_timestamp
if runtime >= (classify_for - 0.3):
logger.info(
'classify_anomalies :: stopping before timeout is reached')
break
time_now = time()
runtime = time_now - start_timestamp
if runtime >= (classify_for - 0.3):
logger.info(
'classify_anomalies :: stopping before timeout is reached')
break
anomaly_types = []
results_recorded = False
if algorithms_processed == algorithms_to_process:
for algorithm in LUMINOSITY_CLASSIFY_ANOMALY_ALGORITHMS:
if algorithm_results[algorithm]['result']:
anomaly_types.append(algorithm)
if not anomaly_types:
results_recorded = True
else:
logger.info(
'classify_anomalies :: anomaly_types identified for %s - %s'
% (base_name, str(anomaly_types)))
anomaly_id = 0
if anomaly_types:
try:
anomaly_id = get_anomaly_id(skyline_app, base_name, timestamp)
except:
logger.error(
'error :: classify_anomalies :: get_anomaly_id failed to determine id'
)
anomaly_id = 0
logger.info('classify_anomalies :: anomaly_id: %s' % (str(anomaly_id)))
type_data = []
if anomaly_id:
query = 'SELECT id,algorithm,type FROM anomaly_types'
try:
results = mysql_select(skyline_app, query)
except:
logger.error(traceback.format_exc())
logger.error(
'error :: querying MySQL - SELECT id,type FROM anomaly_types'
)
db_anomaly_types = {}
for id, associated_algorithm, anomaly_type in results:
db_anomaly_types[associated_algorithm] = {}
db_anomaly_types[associated_algorithm]['id'] = id
db_anomaly_types[associated_algorithm]['type'] = anomaly_type
metric_id = 0
query = 'SELECT id FROM metrics WHERE metric=\'%s\'' % base_name
try:
results = mysql_select(skyline_app, query)
for item in results:
metric_id = item[0]
break
except:
logger.error(traceback.format_exc())
logger.error(
'error :: classify_anomalies :: querying MySQL - SELECT id FROM metrics WHERE metric=\'%s\''
% base_name)
type_data = []
for anomaly_type in anomaly_types:
type_data.append(int(db_anomaly_types[anomaly_type]['id']))
logger.info('classify_anomalies :: type_data: %s' % (str(type_data)))
classification_exists = None
if type_data and anomaly_id:
query = 'SELECT metric_id FROM anomalies_type WHERE id=%s' % anomaly_id
try:
results = mysql_select(skyline_app, query)
for item in results:
classification_exists = item[0]
break
except:
logger.error(traceback.format_exc())
logger.error(
'error :: classify_anomalies :: querying MySQL - SELECT metric_id FROM anomalies_type WHERE id=%s'
% anomaly_id)
if classification_exists:
try:
redis_conn.srem('luminosity.classify_anomalies',
str(classify_anomaly))
logger.info(
'classify_anomalies :: results already recorded for metric_id %s so removed %s, %s, %s item from luminosity.classify_anomalies Redis set'
% (str(classification_exists), base_name, str(timestamp),
anomaly_data_dict['app']))
except:
logger.error(traceback.format_exc())
logger.error(
'error :: classify_anomalies :: after results recorded failed to remove %s, %s, %s item from luminosity.classify_anomalies Redis set'
% (base_name, str(timestamp), anomaly_data_dict['app']))
type_data = None
if type_data:
type_data_str = ''
for id in type_data:
if type_data_str == '':
type_data_str = '%s' % str(id)
else:
type_data_str = '%s,%s' % (type_data_str, str(id))
ins_values = '(%s,%s,\'%s\')' % (str(anomaly_id), str(metric_id),
type_data_str)
values_string = 'INSERT INTO anomalies_type (id, metric_id, type) VALUES %s' % ins_values
try:
results_recorded = mysql_insert(values_string)
logger.debug('debug :: classify_anomalies :: INSERT: %s' %
(str(values_string)))
logger.debug(
'debug :: classify_anomalies :: results_recorded: %s' %
(str(results_recorded)))
except Exception as e:
# Handle a process updating on SystemExit
if 'Duplicate entry' in str(e):
results_recorded = True
logger.info(
'classify_anomalies :: a entry already exists in anomalies_type for anomaly id %s on %s, OK'
% (str(anomaly_id), str(base_name)))
else:
logger.error(traceback.format_exc())
logger.error('error :: MySQL insert - %s' %
str(values_string))
results_recorded = 0
if results_recorded:
logger.info(
'classify_anomalies :: added %s row to anomalies_type for anomaly id %s on %s - %s'
% (str(results_recorded), str(anomaly_id), base_name,
str(type_data)))
if results_recorded:
try:
redis_conn.srem('luminosity.classify_anomalies',
str(classify_anomaly))
logger.info(
'classify_anomalies :: results recorded so removed %s, %s, %s item from luminosity.classify_anomalies Redis set'
% (base_name, str(timestamp), anomaly_data_dict['app']))
except:
logger.error(traceback.format_exc())
logger.error(
'error :: classify_anomalies :: after results recorded failed to remove %s, %s, %s item from luminosity.classify_anomalies Redis set'
% (base_name, str(timestamp), anomaly_data_dict['app']))
try:
manage_processing_key(current_pid, base_name, timestamp,
classify_for, 'remove')
except:
logger.error(traceback.format_exc())
logger.error(
'error :: classify_anomalies :: failed to run manage_processing_key - %s'
% base_name)
end_classify_anomalies = timer()
logger.info(
'classify_anomalies :: %s anomalies were processed, took %.6f seconds'
% (str(anomalies_proceessed),
(end_classify_anomalies - start_classify_anomalies)))
return
def thunder_check_horizon_metrics_received(self):
"""
Determine any significant change in the number of metrics recieved by the
workers and sends a thunder event when necessary.
:param self: the self object
:type self: object
:return: success
:rtype: boolean
"""
function_str = 'functions.thunder.thunder_check_horizon_metrics_recieved'
check_app = 'horizon'
event_type = 'worker.metrics_received'
base_name = 'skyline.%s.%s.%s' % (check_app, this_host, event_type)
success = True
now = int(time())
try:
expiry = int(settings.THUNDER_CHECKS[check_app][event_type]['expiry'])
except Exception as e:
logger.error(
'error :: %s :: failed to determine the expiry for %s %s check - %s'
% (function_str, check_app, event_type, e))
expiry = 900
check_dict = {}
cache_key = 'thunder.%s.%s' % (check_app, event_type)
try:
check_dict = self.redis_conn_decoded.hgetall(cache_key)
except Exception as e:
logger.error('error :: %s :: could not get the Redis %s key - %s' %
(function_str, cache_key, e))
value = None
timestamp = None
if check_dict:
try:
value = float(check_dict['value'])
timestamp = int(float(check_dict['timestamp']))
logger.info('thunder/rolling :: %s :: %s.%s value from dict: %s' %
(function_str, check_app, event_type, str(value)))
except Exception as e:
logger.error(traceback.format_exc())
logger.error(
'error :: thunder/rolling :: %s :: could not determine %s.%s value from dict - %s'
% (function_str, check_app, event_type, e))
# Determine if a thunder alert has been sent for this check
check_thunder_alert = None
cache_key = 'thunder.alert.%s.%s' % (check_app, event_type)
try:
check_thunder_alert = self.redis_conn_decoded.get(cache_key)
except Exception as e:
logger.error(traceback.format_exc())
logger.error(
'error :: thunder/rolling :: %s :: failed to get %s Redis key - %s'
% (function_str, cache_key, e))
if not check_thunder_alert:
check_thunder_alert = check_thunder_failover_key(self, cache_key)
if not check_thunder_alert:
# If worker has not updated the thunder.horizon.worker.metrics_received
# Redis in 5 minutes, alert
alert_no_metrics_recieved_reported = False
if timestamp:
if timestamp < (int(time()) - 299):
alert_no_metrics_recieved_reported = True
if alert_no_metrics_recieved_reported:
level = 'alert'
message = '%s - Horizon worker not reporting metrics_received' % level
status = '%s no count reported for 5 minutes' % base_name
thunder_event = {
'level': level,
'event_type': event_type,
'message': message,
'app': check_app,
'metric': base_name,
'source': 'thunder',
'timestamp': time(),
'expiry': expiry,
'data': {
'status': status
}
}
submitted = None
try:
submitted = thunder_send_event(skyline_app,
thunder_event,
log=True)
except Exception as e:
logger.error(
'error :: thunder/rolling :: %s :: thunder_send_event failed - %s'
% (function_str, e))
if submitted:
logger.info(
'thunder/rolling :: %s :: %s %s not reported in 5 minutes thunder_send_event submitted'
% (function_str, check_app, event_type))
return False
else:
logger.error(
'error :: thunder/rolling :: %s :: %s %s not reported in 5 minutes thunder_send_event failed'
% (function_str, check_app, event_type))
# Get the timeseries
metric_name = '%s%s' % (settings.FULL_NAMESPACE, base_name)
timeseries = []
try:
timeseries = get_metric_timeseries(skyline_app, metric_name)
except Exception as e:
logger.error(
'error :: %s :: get_metric_timeseries failed for %s - %s' %
(function_str, metric_name, e))
send_alert = False
anomalous = None
anomalyScore = None
if timeseries:
percent = 20
try:
percent = float(settings.THUNDER_CHECKS[check_app][event_type]
['significant_change_percentage'])
except Exception as e:
logger.error(
'error :: %s :: failed to determine significant_change_percentage for %s %s check - %s'
% (function_str, check_app, event_type, e))
percent = 20
window = 600
try:
window = float(settings.THUNDER_CHECKS[check_app][event_type]
['significant_change_window'])
except Exception as e:
logger.error(
'error :: %s :: failed to determine significant_change_window for %s %s check - %s'
% (function_str, check_app, event_type, e))
window = 600
period = 3600
try:
period = float(settings.THUNDER_CHECKS[check_app][event_type]
['significant_change_over'])
except Exception as e:
logger.error(
'error :: %s :: failed to determine significant_change_over for %s %s check - %s'
% (function_str, check_app, event_type, e))
period = 3600
times_in_a_row = 5
try:
times_in_a_row = int(
float(settings.THUNDER_CHECKS[check_app][event_type]
['times_in_a_row']))
except Exception as e:
logger.error(
'error :: %s :: failed to determine times_in_a_row for %s %s check - %s'
% (function_str, check_app, event_type, e))
times_in_a_row = 5
custom_algorithm_dict = {}
algorithm_source = '/opt/skyline/github/skyline/skyline/custom_algorithms/significant_change_window_percent_sustained.py'
try:
algorithm_source = settings.THUNDER_CHECKS[check_app][event_type][
'algorithm_source']
except Exception as e:
logger.error(
'error :: %s :: failed to determine algorithm_source for %s %s check - %s'
% (function_str, check_app, event_type, e))
algorithm_source = '/opt/skyline/github/skyline/skyline/custom_algorithms/significant_change_window_percent_sustained.py'
custom_algorithm_dict['algorithm_source'] = algorithm_source
custom_algorithm_dict['max_execution_time'] = 10.0
algorithm_parameters = {
'percent': percent,
'window': window,
'period': period,
'times_in_a_row': times_in_a_row,
'return_percent_as_anomalyScore': True,
'debug_logging': True,
}
custom_algorithm_dict['algorithm_parameters'] = algorithm_parameters
custom_algorithm = 'significant_change_window_percent_sustained'
anomalous = None
anomalyScore = None
try:
anomalous, anomalyScore = run_custom_algorithm_on_timeseries(
skyline_app, getpid(), base_name, timeseries, custom_algorithm,
custom_algorithm_dict, False)
except Exception as e:
logger.error(
'error :: %s :: failed to determine the expiry for %s %s check - %s'
% (function_str, check_app, event_type, e))
if anomalous:
send_alert = True
success = False
logger.warn(
'warning :: thunder/rolling :: %s :: %s.%s is anomalous latest value is %s %% different from median of windows in period'
% (function_str, check_app, event_type, str(anomalyScore)))
else:
logger.info(
'thunder/rolling :: %s :: %s.%s not anomalous latest value only %s %% different from median of windows in period'
% (function_str, check_app, event_type, str(anomalyScore)))
# If not anomalous and an alert has been sent, send a recovery notice
if check_thunder_alert and anomalous is False:
level = 'notice'
message = '%s - Horizon worker.metrics_received has recovered' % level
status = '%s is now %.2f %% different from the median windows (%s seconds) of the last period (%s seconds)' % (
base_name, anomalyScore, str(window), str(period))
thunder_event = {
'level': level,
'event_type': event_type,
'message': message,
'app': check_app,
'metric': base_name,
'source': 'thunder',
'timestamp': now,
'expiry': 59,
'data': {
'status': status
}
}
try:
submitted = thunder_send_event(skyline_app,
thunder_event,
log=True)
except Exception as e:
logger.error('error :: %s :: thunder_send_event failed - %s' %
(function_str, e))
if submitted:
logger.info('%s :: %s %s thunder_send_event submitted' %
(function_str, check_app, event_type))
# Remove alert key
cache_key = 'thunder.alert.%s.%s' % (check_app, event_type)
try:
self.redis_conn.delete(cache_key)
except Exception as e:
logger.error(traceback.format_exc())
logger.error(
'error :: thunder/rolling :: %s :: failed to delete %s Redis key - %s'
% (function_str, cache_key, e))
else:
logger.error('error :: %s :: %s %s thunder_send_event failed' %
(function_str, check_app, event_type))
return success
# If anomalous
if send_alert and anomalous and not check_thunder_alert:
level = 'alert'
message = '%s - Horizon worker.metrics_received has changed significantly' % level
status = '%s is %.2f %% different from the median windows (%s seconds) of the last period (%s seconds)' % (
base_name, anomalyScore, str(window), str(period))
try:
expiry = int(
settings.THUNDER_CHECKS[check_app][event_type]['expiry'])
except Exception as e:
logger.error(
'error :: %s :: failed to determine the expiry for %s %s check - %s'
% (function_str, check_app, event_type, e))
expiry = 900
thunder_event = {
'level': level,
'event_type': event_type,
'message': message,
'app': check_app,
'metric': base_name,
'source': 'thunder',
'timestamp': time(),
'expiry': expiry,
'data': {
'status': status
}
}
try:
submitted = thunder_send_event(skyline_app,
thunder_event,
log=True)
except Exception as e:
logger.error('error :: %s :: thunder_send_event failed - %s' %
(function_str, e))
if submitted:
logger.info(
'%s :: Horizon worker.metrics_received thunder_send_event submitted'
% (function_str))
else:
logger.error(
'error :: %s :: Horizon worker.metrics_received thunder_send_event failed'
% (function_str))
return success
def run_selected_algorithm(timeseries, metric_name, second_order_resolution_seconds, run_negatives_present, triggered_algorithms):
"""
Run selected algorithms
"""
# @added 20200425 - Feature #3508: ionosphere.untrainable_metrics
# Added negatives_found for run_negatives_present
negatives_found = False
# @added 20200607 - Feature #3566: custom_algorithms
final_custom_ensemble = []
algorithms_run = []
custom_consensus_override = False
custom_consensus_values = []
run_3sigma_algorithms = True
run_3sigma_algorithms_overridden_by = []
custom_algorithm = None
# @added 20201125 - Feature #3848: custom_algorithms - run_before_3sigma parameter
run_custom_algorithm_after_3sigma = False
final_after_custom_ensemble = []
custom_algorithm_not_anomalous = False
# @added 20211125 - Feature #3566: custom_algorithms
custom_algorithms_run = []
custom_algorithms_run_results = []
# @added 20220111 - Feature #3566: custom_algorithms
# Set default values
base_name = metric_name.replace(FULL_NAMESPACE, '', 1)
custom_algorithms_to_run = {}
if CUSTOM_ALGORITHMS:
base_name = metric_name.replace(FULL_NAMESPACE, '', 1)
custom_algorithms_to_run = {}
try:
custom_algorithms_to_run = get_custom_algorithms_to_run(skyline_app, base_name, CUSTOM_ALGORITHMS, DEBUG_CUSTOM_ALGORITHMS)
if DEBUG_CUSTOM_ALGORITHMS:
if custom_algorithms_to_run:
logger.debug('mirage_algorithms :: debug :: custom algorithms ARE RUN on %s' % (str(base_name)))
except:
logger.error('error :: get_custom_algorithms_to_run :: %s' % traceback.format_exc())
custom_algorithms_to_run = {}
for custom_algorithm in list(custom_algorithms_to_run.keys()):
debug_logging = False
try:
debug_logging = custom_algorithms_to_run[custom_algorithm]['debug_logging']
except:
debug_logging = False
if DEBUG_CUSTOM_ALGORITHMS:
debug_logging = True
# @modified 20210304 - Feature #3642: Anomaly type classification
# Feature #3970: custom_algorithm - adtk_level_shift
# Added triggered_algorithms
if custom_algorithm == 'adtk_level_shift':
if 'adtk_level_shift' not in triggered_algorithms:
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
logger.debug('debug :: custom_algorithms :: NOT running custom algorithm %s on %s as was not in triggered_algorithms' % (
str(custom_algorithm), str(base_name)))
continue
# @added 20201125 - Feature #3848: custom_algorithms - run_before_3sigma parameter
run_before_3sigma = True
try:
run_before_3sigma = custom_algorithms_to_run[custom_algorithm]['run_before_3sigma']
except:
run_before_3sigma = True
if not run_before_3sigma:
run_custom_algorithm_after_3sigma = True
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
logger.debug('debug :: mirage_algorithms :: NOT running custom algorithm %s on %s BEFORE three-sigma algorithms' % (
str(custom_algorithm), str(base_name)))
continue
run_algorithm = []
run_algorithm.append(custom_algorithm)
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
logger.debug('debug :: mirage_algorithms :: running custom algorithm %s on %s' % (
str(custom_algorithm), str(base_name)))
start_debug_timer = timer()
run_custom_algorithm_on_timeseries = None
try:
from custom_algorithms import run_custom_algorithm_on_timeseries
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
logger.debug('debug :: mirage_algorithms :: loaded run_custom_algorithm_on_timeseries')
except:
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
logger.error(traceback.format_exc())
logger.error('error :: mirage_algorithms :: failed to load run_custom_algorithm_on_timeseries')
result = None
anomalyScore = None
use_debug_logging = False
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
use_debug_logging = True
if run_custom_algorithm_on_timeseries:
# @added 20211125 - Feature #3566: custom_algorithms
custom_algorithms_run.append(custom_algorithm)
try:
result, anomalyScore = run_custom_algorithm_on_timeseries(skyline_app, getpid(), base_name, timeseries, custom_algorithm, custom_algorithms_to_run[custom_algorithm], use_debug_logging)
algorithm_result = [result]
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
logger.debug('debug :: mirage_algorithms :: run_custom_algorithm_on_timeseries run with result - %s, anomalyScore - %s' % (
str(result), str(anomalyScore)))
# @added 20211125 - Feature #3566: custom_algorithms
custom_algorithms_run_results.append(result)
except:
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
logger.error(traceback.format_exc())
logger.error('error :: mirage_algorithms :: failed to run custom_algorithm %s on %s' % (
custom_algorithm, base_name))
result = None
algorithm_result = [None]
# @added 20211125 - Feature #3566: custom_algorithms
custom_algorithms_run_results.append(False)
else:
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
logger.error('error :: debug :: mirage_algorithms :: run_custom_algorithm_on_timeseries was not loaded so was not run')
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
end_debug_timer = timer()
logger.debug('debug :: mirage_algorithms :: ran custom algorithm %s on %s with result of (%s, %s) in %.6f seconds' % (
str(custom_algorithm), str(base_name),
str(result), str(anomalyScore),
(end_debug_timer - start_debug_timer)))
algorithms_run.append(custom_algorithm)
if algorithm_result.count(True) == 1:
result = True
elif algorithm_result.count(False) == 1:
result = False
elif algorithm_result.count(None) == 1:
result = None
else:
result = False
final_custom_ensemble.append(result)
custom_consensus = None
algorithms_allowed_in_consensus = []
# @added 20200605 - Feature #3566: custom_algorithms
# Allow only single or multiple custom algorithms to run and allow
# the a custom algorithm to specify not to run 3sigma aglorithms
custom_run_3sigma_algorithms = True
try:
custom_run_3sigma_algorithms = custom_algorithms_to_run[custom_algorithm]['run_3sigma_algorithms']
except:
custom_run_3sigma_algorithms = True
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
logger.debug('debug :: error - algorithms :: could not determine custom_run_3sigma_algorithms - default to True')
if not custom_run_3sigma_algorithms and result:
run_3sigma_algorithms = False
run_3sigma_algorithms_overridden_by.append(custom_algorithm)
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
logger.debug('debug :: mirage_algorithms :: run_3sigma_algorithms is False on %s for %s' % (
custom_algorithm, base_name))
else:
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
logger.debug('debug :: mirage_algorithms :: run_3sigma_algorithms will now be run on %s - %s' % (
base_name, str(custom_run_3sigma_algorithms)))
if result:
try:
custom_consensus = custom_algorithms_to_run[custom_algorithm]['consensus']
if custom_consensus == 0:
custom_consensus = int(MIRAGE_CONSENSUS)
else:
custom_consensus_values.append(custom_consensus)
except:
custom_consensus = int(MIRAGE_CONSENSUS)
try:
algorithms_allowed_in_consensus = custom_algorithms_to_run[custom_algorithm]['algorithms_allowed_in_consensus']
except:
algorithms_allowed_in_consensus = []
if custom_consensus == 1:
custom_consensus_override = True
logger.info('mirage_algorithms :: overidding the CONSENSUS as custom algorithm %s overides on %s' % (
str(custom_algorithm), str(base_name)))
# TODO - figure out how to handle consensus overrides if
# multiple custom algorithms are used
if DEBUG_CUSTOM_ALGORITHMS:
if not run_3sigma_algorithms:
logger.debug('mirage_algorithms :: not running 3 sigma algorithms')
if len(run_3sigma_algorithms_overridden_by) > 0:
logger.debug('mirage_algorithms :: run_3sigma_algorithms overridden by %s' % (
str(run_3sigma_algorithms_overridden_by)))
# @added 20200607 - Feature #3566: custom_algorithms
# @modified 20201120 - Feature #3566: custom_algorithms
# ensemble = []
ensemble = final_custom_ensemble
# @added 20211125 - Feature #3566: custom_algorithms
# If custom_algorithms were run and did not trigger reset the consensus
if run_3sigma_algorithms:
if len(custom_algorithms_run) > 0:
none_count = custom_algorithms_run_results.count(None)
false_count = custom_algorithms_run_results.count(False)
not_anomalous_count = none_count + false_count
if len(custom_algorithms_run) == not_anomalous_count:
custom_consensus_override = False
custom_consensus = int(MIRAGE_CONSENSUS)
ensemble = []
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
logger.debug('debug :: mirage-algorithms :: reset ensemble, custom_consensus and custom_consensus_override after custom_algorithms all calcuated False')
# @modified 20201125 - Feature #3848: custom_algorithms - run_before_3sigma parameter
# Check run_3sigma_algorithms as well to the conditional
# if not custom_consensus_override:
if run_3sigma_algorithms and not custom_consensus_override:
if DEBUG_CUSTOM_ALGORITHMS:
logger.debug('debug :: mirage_algorithms :: running three-sigma algorithms')
try:
logger.info('mirage_algorithms :: running three-sigma algorithms')
ensemble = [globals()[algorithm](timeseries, second_order_resolution_seconds) for algorithm in MIRAGE_ALGORITHMS]
for algorithm in MIRAGE_ALGORITHMS:
algorithms_run.append(algorithm)
except:
logger.error('Algorithm error: %s' % traceback.format_exc())
# @modified 20200425 - Feature #3508: ionosphere.untrainable_metrics
# Added negatives_found
# return False, [], 1
# @modified 20200607 - Feature #3566: custom_algorithms
# Added algorithms_run
return False, [], 1, False, algorithms_run
# @added 20201124 - Feature #3566: custom_algorithms
if final_custom_ensemble:
ensemble = final_custom_ensemble + ensemble
logger.info('mirage_algorithms :: %s - True count of %s after 3-sigma algorithms' % (
str(base_name), str(ensemble.count(True))))
else:
for algorithm in MIRAGE_ALGORITHMS:
ensemble.append(None)
# @added 20201120 - Feature #3566: custom_algorithms
# If 3sigma algorithms have not been run discard the MIRAGE_ALGORITHMS
# added above
if not run_3sigma_algorithms:
ensemble = final_custom_ensemble
# @added 20211104 - Bug #4308: matrixprofile - fN on big drops
# Branch #3068: SNAB
ensemble_pre_custom_algorithms_true_count = ensemble.count(True)
ensemble_pre_custom_algorithms = list(ensemble)
# if ensemble_pre_custom_algorithms_true_count >= 7:
# skyline_matrixprofile_override = True
check_trigger_history = False
trigger_history_override = 0
try:
trigger_history_override = custom_algorithms_to_run[custom_algorithm]['trigger_history_override']
except KeyError:
trigger_history_override = 0
redis_conn_decoded = None
if ensemble_pre_custom_algorithms_true_count >= MIRAGE_CONSENSUS and trigger_history_override:
from ast import literal_eval
from skyline_functions import get_redis_conn_decoded
try:
redis_conn_decoded = get_redis_conn_decoded(skyline_app)
except Exception as err:
logger.error(traceback.format_exc())
logger.error('error :: mirage_algorithms :: get_redis_conn_decoded failed - %s' % (
str(err)))
check_trigger_history = True
# @added 20201125 - Feature #3848: custom_algorithms - run_before_3sigma parameter
if run_custom_algorithm_after_3sigma:
if DEBUG_CUSTOM_ALGORITHMS:
logger.debug('debug :: checking custom algorithms to run AFTER three-sigma algorithms')
for custom_algorithm in list(custom_algorithms_to_run.keys()):
debug_logging = False
try:
debug_logging = custom_algorithms_to_run[custom_algorithm]['debug_logging']
except:
debug_logging = False
if DEBUG_CUSTOM_ALGORITHMS:
debug_logging = True
run_before_3sigma = True
try:
run_before_3sigma = custom_algorithms_to_run[custom_algorithm]['run_before_3sigma']
except:
run_before_3sigma = True
if run_before_3sigma:
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
logger.debug('debug :: mirage_algorithms :: NOT running custom algorithm %s on %s AFTER three-sigma algorithms as run_before_3sigma is %s' % (
str(custom_algorithm), str(base_name),
str(run_before_3sigma)))
continue
try:
custom_consensus = custom_algorithms_to_run[custom_algorithm]['consensus']
if custom_consensus == 0:
custom_consensus = int(MIRAGE_CONSENSUS)
else:
custom_consensus_values.append(custom_consensus)
except:
custom_consensus = int(MIRAGE_CONSENSUS)
run_only_if_consensus = False
try:
run_only_if_consensus = custom_algorithms_to_run[custom_algorithm]['run_only_if_consensus']
except:
run_only_if_consensus = False
if run_only_if_consensus:
if ensemble.count(True) < int(MIRAGE_CONSENSUS):
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
logger.debug('debug :: mirage_algorithms :: NOT running custom algorithm %s on %s AFTER three-sigma algorithms as only %s three-sigma algorithms triggered - MIRAGE_CONSENSUS of %s not achieved' % (
str(custom_algorithm), str(base_name),
str(ensemble.count(True)), str(MIRAGE_CONSENSUS)))
continue
else:
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
logger.debug('debug :: mirage_algorithms :: running custom algorithm %s on %s AFTER three-sigma algorithms as %s three-sigma algorithms triggered - MIRAGE_CONSENSUS of %s was achieved' % (
str(custom_algorithm), str(base_name),
str(ensemble.count(True)), str(MIRAGE_CONSENSUS)))
run_algorithm = []
run_algorithm.append(custom_algorithm)
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
logger.debug('debug :: mirage_algorithms :: running custom algorithm %s on %s' % (
str(custom_algorithm), str(base_name)))
start_debug_timer = timer()
run_custom_algorithm_on_timeseries = None
try:
from custom_algorithms import run_custom_algorithm_on_timeseries
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
logger.debug('debug :: mirage_algorithms :: loaded run_custom_algorithm_on_timeseries')
except:
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
logger.error(traceback.format_exc())
logger.error('error :: mirage_algorithms :: failed to load run_custom_algorithm_on_timeseries')
result = None
anomalyScore = None
use_debug_logging = False
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
use_debug_logging = True
if run_custom_algorithm_on_timeseries:
try:
result, anomalyScore = run_custom_algorithm_on_timeseries(skyline_app, getpid(), base_name, timeseries, custom_algorithm, custom_algorithms_to_run[custom_algorithm], use_debug_logging)
algorithm_result = [result]
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
logger.debug('debug :: mirage_algorithms :: run_custom_algorithm_on_timeseries run with result - %s, anomalyScore - %s' % (
str(result), str(anomalyScore)))
logger.info('mirage_algorithms :: metric: %s, custom_algorithm: %s, result: %s, anomalyScore: %s' % (
base_name, custom_algorithm, str(result), str(anomalyScore)))
except Exception as err:
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
logger.error(traceback.format_exc())
logger.error('error :: mirage_algorithms :: failed to run custom_algorithm %s on %s - %s' % (
custom_algorithm, base_name, str(err)))
result = None
algorithm_result = [None]
else:
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
logger.error('error :: debug :: mirage_algorithms :: run_custom_algorithm_on_timeseries was not loaded so was not run')
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
end_debug_timer = timer()
logger.debug('debug :: mirage_algorithms :: ran custom algorithm %s on %s with result of (%s, %s) in %.6f seconds' % (
str(custom_algorithm), str(base_name),
str(result), str(anomalyScore),
(end_debug_timer - start_debug_timer)))
algorithms_run.append(custom_algorithm)
if algorithm_result.count(True) == 1:
result = True
elif algorithm_result.count(False) == 1:
result = False
elif algorithm_result.count(None) == 1:
result = None
else:
result = False
final_after_custom_ensemble.append(result)
algorithms_allowed_in_consensus = []
# custom_run_3sigma_algorithms = True does not need to be checked
# here as if three-sigma algorithms have run they have already run
# at this point, unlike above in the run_before_3sigma custom
# algorithms run
if result:
try:
algorithms_allowed_in_consensus = custom_algorithms_to_run[custom_algorithm]['algorithms_allowed_in_consensus']
except:
algorithms_allowed_in_consensus = []
if custom_consensus == 1:
custom_consensus_override = True
logger.info('mirage_algorithms :: overriding the CONSENSUS as custom algorithm %s overides on %s' % (
str(custom_algorithm), str(base_name)))
else:
# @added 20201127 - Feature #3566: custom_algorithms
# Handle if the result is None
if result is None:
logger.warning('warning :: mirage_algorithms :: %s failed to run on %s' % (
str(custom_algorithm), str(base_name)))
else:
if custom_consensus == 1:
# hmmm we are required to hack threshold here
custom_algorithm_not_anomalous = True
if DEBUG_CUSTOM_ALGORITHMS or debug_logging:
logger.debug('debug :: mirage_algorithms :: %s did not trigger - custom_algorithm_not_anomalous set to identify as not anomalous' % (
str(custom_algorithm)))
# @added 20211104 - Bug #4308: matrixprofile - fN on big drops
# Branch #3068: SNAB
if custom_algorithm == 'skyline_matrixprofile' and check_trigger_history:
trigger_history = {}
try:
raw_trigger_history = redis_conn_decoded.hget('mirage.trigger_history', base_name)
if raw_trigger_history:
trigger_history = literal_eval(raw_trigger_history)
except Exception as err:
logger.error(traceback.format_exc())
logger.error('error :: mirage_algorithms :: failed to evaluate data from mirage.trigger_history Redis hash key - %s' % (
str(err)))
metric_resolution = 60
if trigger_history:
try:
from functions.timeseries.determine_data_frequency import determine_data_frequency
metric_resolution = determine_data_frequency(skyline_app, timeseries, False)
except Exception as err:
logger.error(traceback.format_exc())
logger.error('error :: mirage_algorithms :: determine_data_frequency failed - %s' % (
str(err)))
recent_trigger_history = {}
last_timestamp = int(timeseries[-1][0])
oldest_trigger_timestamp = last_timestamp - (metric_resolution * 4)
# Self clean trigger_history
for trigger_timestamp in list(trigger_history.keys()):
if trigger_timestamp < oldest_trigger_timestamp:
continue
recent_trigger_history[trigger_timestamp] = trigger_history[trigger_timestamp]
tmp_final_ensemble = ensemble + final_after_custom_ensemble
trigger_dict = {
'count': ensemble_pre_custom_algorithms_true_count,
'ensemble': ensemble_pre_custom_algorithms,
'final_ensemble': tmp_final_ensemble,
'final_ensemble_count': tmp_final_ensemble.count(True),
'algorithms_run': algorithms_run,
'value': timeseries[-1][1],
'resolution': metric_resolution
}
recent_trigger_history[last_timestamp] = trigger_dict
try:
redis_conn_decoded.hset('mirage.trigger_history', base_name, str(recent_trigger_history))
except Exception as err:
logger.error(traceback.format_exc())
logger.error('error :: mirage_algorithms :: failed to set key in mirage.trigger_history Redis hash key - %s' % (
str(err)))
recent_trigger_history_count = len(list(recent_trigger_history.keys()))
if recent_trigger_history_count >= trigger_history_override and custom_consensus == 1:
logger.info('mirage_algorithms :: %s overriding %s - recent_trigger_history_count breached with %s recent triggers' % (
str(base_name), custom_algorithm,
str(recent_trigger_history_count)))
custom_consensus_override = False
custom_algorithm_not_anomalous = False
logger.info('mirage_algorithms :: overriding %s result: %s, not anomalous custom_consensus of 1' % (
custom_algorithm, str(result)))
logger.info('mirage_algorithms :: custom_consensus_override set to False as original 3-sigma True count is %s along with recent_trigger_history_count breach' % (
str(ensemble_pre_custom_algorithms_true_count)))
new_algorithms_run = []
for algorithm_run in algorithms_run:
if algorithm_run == custom_algorithm:
algorithm_run = '%s (override - %s recent 3-sigma triggers)' % (
custom_algorithm, str(recent_trigger_history_count))
new_algorithms_run.append(algorithm_run)
algorithms_run = list(new_algorithms_run)
for item in final_after_custom_ensemble:
ensemble.append(item)
# @modified 20200607 - Feature #3566: custom_algorithms
try:
# threshold = len(ensemble) - MIRAGE_CONSENSUS
if custom_consensus_override:
threshold = len(ensemble) - 1
else:
threshold = len(ensemble) - MIRAGE_CONSENSUS
# @modified 20201125 - Feature #3848: custom_algorithms - run_before_3sigma parameter
# Added and not custom_algorithm_not_anomalous
if ensemble.count(False) <= threshold and not custom_algorithm_not_anomalous:
# @added 20200425 - Feature #3508: ionosphere.untrainable_metrics
# Only run a negatives_present check if it is anomalous, there
# is no need to check unless it is related to an anomaly
if run_negatives_present:
try:
negatives_found = negatives_present(timeseries)
if negatives_found:
number_of_negatives_found = len(negatives_found)
else:
number_of_negatives_found = 0
logger.info('%s negative values found for %s' % (
str(number_of_negatives_found), metric_name))
except:
logger.error('Algorithm error: negatives_present :: %s' % traceback.format_exc())
negatives_found = False
if MIRAGE_ENABLE_SECOND_ORDER:
if is_anomalously_anomalous(metric_name, ensemble, timeseries[-1][1]):
# @modified 20200425 - Feature #3508: ionosphere.untrainable_metrics
# Added negatives_found
# return True, ensemble, timeseries[-1][1]
# @modified 20200607 - Feature #3566: custom_algorithms
# Added algorithms_run
return True, ensemble, timeseries[-1][1], negatives_found, algorithms_run
else:
# @modified 20200425 - Feature #3508: ionosphere.untrainable_metrics
# Added negatives_found
# return True, ensemble, timeseries[-1][1]
# @modified 20200607 - Feature #3566: custom_algorithms
# Added algorithms_run
return True, ensemble, timeseries[-1][1], negatives_found, algorithms_run
# @modified 20200425 - Feature #3508: ionosphere.untrainable_metrics
# Added negatives_found
# return False, ensemble, timeseries[-1][1]
# @modified 20200607 - Feature #3566: custom_algorithms
# Added algorithms_run
return False, ensemble, timeseries[-1][1], negatives_found, algorithms_run
except:
logger.error('Algorithm error: %s' % traceback.format_exc())
# @modified 20200425 - Feature #3508: ionosphere.untrainable_metrics
# Added negatives_found
# return False, [], 1
# @modified 20200607 - Feature #3566: custom_algorithms
# Added algorithms_run
return False, [], 1, False, algorithms_run