def arundo_adtk(data_path):
data = get_data(data_path)
data['date'] = data['timestamp'].apply(
lambda i: datetime.fromtimestamp(i)) # 时间转换
s_train = data[['date', 'value']]
# 设置索引项
s_train = s_train.set_index('date')
s_train = validate_series(s_train)
print(s_train)
# plot(s_train)
# STL分解+离群点检测
steps = [("deseasonal", STLDecomposition(freq=20)),
("quantile_ad", QuantileAD(high=0.9997, low=0.005))]
pipeline = Pipeline(steps)
anomalies = pipeline.fit_detect(s_train)
print(anomalies)
# plot(s_train, anomaly_pred=anomalies, ap_color='red', ap_marker_on_curve=True)
# 绘制检测结果]
known_anomalies = data.loc[data['label'] == 1]
known_anomalies = known_anomalies[['date', 'label']]
known_anomalies = known_anomalies.set_index('date')
known_anomalies = to_events(known_anomalies)
print(known_anomalies)
plot(s_train,
anomaly_true=known_anomalies,
anomaly_pred=anomalies,
ap_color='red',
ap_marker_on_curve=True,
at_color="orange")
plt.savefig(img_path + "arundo_adtk.png", dpi=1000)
plt.show()
def get_missing_stats_data(filtered_data):
## plotting the outliers for each node and saving the stats of gaps, filled sequences to a csv file
s1 = None
outlier_list = []
try:
list_dict = []
list_cols = list(filtered_data.columns)
list_cols.remove('date')
for col in list_cols:
d1 = {}
sample = filtered_data[col]
sample = sample.dropna()
if not sample.empty and sample.shape[0] > 10:
sample.index = pd.to_datetime(sample.index)
s1 = sample.resample('H').mean()
d1['site'] = col.split("n=")[0]
node_id = col.split("n=")[1].split("_")[0]
d1['depth'] = col.split("n=")[1].split("_d=")[1]
s = validate_series(s1)
iqr_ad = InterQuartileRangeAD(c=1.5)
anomalies = iqr_ad.fit_detect(s)
plot(s,
anomaly=anomalies,
ts_linewidth=1,
ts_markersize=3,
anomaly_markersize=5,
anomaly_color='red',
anomaly_tag="marker")
plt.figure(figsize=(20, 10))
fig = plt.gcf()
plt.xlabel('date')
plt.ylabel('soil moisture')
plt.title("SITE = " + d1['site'] + ", NODE_ID = " + node_id +
", DEPTH = " + d1['depth'])
fig.savefig(hourly_visualisations + col + ".png")
plt.close('all')
d1['node_id'] = node_id
d1['total_rows'] = s1.shape[0]
d1['start'] = s1.index[0].date()
d1['end'] = s1.index[-1].date()
max_long_seq, longest_gap, num_gaps = find_longest_seq_count(
s1)
d1['nulls'] = s1.isnull().sum()
d1['filled_percentage'] = 1 - d1['nulls'] / float(
d1['total_rows'])
d1['longest_gap'] = longest_gap
d1['longest_seq'] = max_long_seq
d1['num_gaps'] = num_gaps
list_dict.append(d1)
with open(hourly_node_wise_data_path, 'w', newline="") as f:
writer = csv.DictWriter(f, [
'site', 'node_id', 'depth', 'start', 'end', 'nulls',
'total_rows', 'filled_percentage', 'longest_gap',
'longest_seq', 'num_gaps'
])
writer.writeheader()
writer.writerows(list_dict)
except Exception as e:
print(e)
def anomaly_plot(data, anomaly_true, anomaly_pred):
"""
Plot time series and/or anomalies.
"""
plot(data,
anomaly={
"anomaly_true": anomaly_true,
"anomaly_pred": anomaly_pred
},
ts_linewidth=1,
ts_markersize=3,
anomaly_color={
"anomaly_true": 'blue',
"anomaly_pred": 'red'
},
anomaly_alpha=0.3,
curve_group='all')
plt.show()
def checkOutlier(data):
dataCopy = copy.deepcopy(data)
# 修改为时间序列索引
dataCopy['时间'] = pd.to_datetime(dataCopy['时间'], format="%Y%m%d%H%M%S")
dataCopy.set_index("时间", inplace=True)
dataCopy = validate_series(dataCopy)
iqr_ad = InterQuartileRangeAD(c=1.5)
anomalies = iqr_ad.fit_detect(dataCopy)
# 可视化异常图，并保存到本地
for i, index in enumerate(indexes):
axes = plot(dataCopy[index],
anomaly=anomalies[index],
ts_linewidth=1,
ts_markersize=3,
anomaly_markersize=5,
anomaly_color='red',
anomaly_tag="marker")
axes[0].set_title(index_names[i], fontsize='xx-large')
axes[0].legend(['Normal', 'Anomaly'], loc='best', fontsize='x-large')
axes[0].set_xlabel('time', fontsize='x-large')
axes[0].set_ylabel('value', fontsize='x-large')
axes[0].figure.savefig(index + ".png")
logging.log(logging.DEBUG, 'The abnormal curve has been drawn.')
# 修改数据并统计异常比率
rows, columns = anomalies.shape
count = 0
for row in range(rows):
for col in range(columns):
if anomalies.iloc[row, col]:
count += 1
data.iloc[row, col + 1] = math.nan
if count != 0:
logging.log(logging.DEBUG, 'There are no outliers.')
outlierRate = float(count) / (rows * columns)
return data, outlierRate
def m66(current_skyline_app, parent_pid, timeseries, algorithm_parameters):
"""
A time series data points are anomalous if the 6th median is 6 standard
deviations (six-sigma) from the time series 6th median standard deviation
and persists for x_windows, where `x_windows = int(window / 2)`.
This algorithm finds SIGNIFICANT cahngepoints in a time series, similar to
PELT and Bayesian Online Changepoint Detection, however it is more robust to
instaneous outliers and more conditionally selective of changepoints.
:param current_skyline_app: the Skyline app executing the algorithm. This
will be passed to the algorithm by Skyline. This is **required** for
error handling and logging. You do not have to worry about handling the
argument in the scope of the custom algorithm itself, but the algorithm
must accept it as the first agrument.
:param parent_pid: the parent pid which is executing the algorithm, this is
**required** for error handling and logging. You do not have to worry
about handling this argument in the scope of algorithm, but the
algorithm must accept it as the second argument.
:param timeseries: the time series as a list e.g. ``[[1578916800.0, 29.0],
[1578920400.0, 55.0], ... [1580353200.0, 55.0]]``
:param algorithm_parameters: a dictionary of any required parameters for the
custom_algorithm and algorithm itself for example:
``algorithm_parameters={
'nth_median': 6,
'sigma': 6,
'window': 5,
'return_anomalies' = True,
}``
:type current_skyline_app: str
:type parent_pid: int
:type timeseries: list
:type algorithm_parameters: dict
:return: True, False or Non
:rtype: boolean
Example CUSTOM_ALGORITHMS configuration:
'm66': {
'namespaces': [
'skyline.analyzer.run_time', 'skyline.analyzer.total_metrics',
'skyline.analyzer.exceptions'
],
'algorithm_source': '/opt/skyline/github/skyline/skyline/custom_algorithms/m66.py',
'algorithm_parameters': {
'nth_median': 6, 'sigma': 6, 'window': 5, 'resolution': 60,
'minimum_sparsity': 0, 'determine_duration': False,
'return_anomalies': True, 'save_plots_to': False,
'save_plots_to_absolute_dir': False, 'filename_prefix': False
},
'max_execution_time': 1.0
'consensus': 1,
'algorithms_allowed_in_consensus': ['m66'],
'run_3sigma_algorithms': False,
'run_before_3sigma': False,
'run_only_if_consensus': False,
'use_with': ['crucible', 'luminosity'],
'debug_logging': False,
},
"""
# You MUST define the algorithm_name
algorithm_name = 'm66'
# Define the default state of None and None, anomalous does not default to
# False as that is not correct, False is only correct if the algorithm
# determines the data point is not anomalous. The same is true for the
# anomalyScore.
anomalous = None
anomalyScore = None
return_anomalies = False
anomalies = []
anomalies_dict = {}
anomalies_dict['algorithm'] = algorithm_name
realtime_analysis = False
current_logger = None
dev_null = None
# If you wanted to log, you can but this should only be done during
# testing and development
def get_log(current_skyline_app):
current_skyline_app_logger = current_skyline_app + 'Log'
current_logger = logging.getLogger(current_skyline_app_logger)
return current_logger
start = timer()
# Use the algorithm_parameters to determine the sample_period
debug_logging = None
try:
debug_logging = algorithm_parameters['debug_logging']
except:
debug_logging = False
if debug_logging:
try:
current_logger = get_log(current_skyline_app)
current_logger.debug(
'debug :: %s :: debug_logging enabled with algorithm_parameters - %s'
% (algorithm_name, str(algorithm_parameters)))
except Exception as e:
# This except pattern MUST be used in ALL custom algortihms to
# facilitate the traceback from any errors. The algorithm we want to
# run super fast and without spamming the log with lots of errors.
# But we do not want the function returning and not reporting
# anything to the log, so the pythonic except is used to "sample" any
# algorithm errors to a tmp file and report once per run rather than
# spewing tons of errors into the log e.g. analyzer.log
dev_null = e
record_algorithm_error(current_skyline_app, parent_pid,
algorithm_name, traceback.format_exc())
# Return None and None as the algorithm could not determine True or False
del dev_null
if current_skyline_app == 'webapp':
return (anomalous, anomalyScore, anomalies, anomalies_dict)
if return_anomalies:
return (anomalous, anomalyScore, anomalies)
return (anomalous, anomalyScore)
# Allow the m66 parameters to be passed in the algorithm_parameters
window = 6
try:
window = algorithm_parameters['window']
except KeyError:
window = 6
except Exception as e:
record_algorithm_error(current_skyline_app, parent_pid, algorithm_name,
traceback.format_exc())
dev_null = e
nth_median = 6
try:
nth_median = algorithm_parameters['nth_median']
except KeyError:
nth_median = 6
except Exception as e:
record_algorithm_error(current_skyline_app, parent_pid, algorithm_name,
traceback.format_exc())
dev_null = e
n_sigma = 6
try:
n_sigma = algorithm_parameters['sigma']
except KeyError:
n_sigma = 6
except Exception as e:
record_algorithm_error(current_skyline_app, parent_pid, algorithm_name,
traceback.format_exc())
dev_null = e
resolution = 0
try:
resolution = algorithm_parameters['resolution']
except KeyError:
resolution = 0
except Exception as e:
record_algorithm_error(current_skyline_app, parent_pid, algorithm_name,
traceback.format_exc())
dev_null = e
determine_duration = False
try:
determine_duration = algorithm_parameters['determine_duration']
except KeyError:
determine_duration = False
except Exception as e:
record_algorithm_error(current_skyline_app, parent_pid, algorithm_name,
traceback.format_exc())
dev_null = e
minimum_sparsity = 0
try:
minimum_sparsity = algorithm_parameters['minimum_sparsity']
except KeyError:
minimum_sparsity = 0
except Exception as e:
record_algorithm_error(current_skyline_app, parent_pid, algorithm_name,
traceback.format_exc())
dev_null = e
shift_to_start_of_window = True
try:
shift_to_start_of_window = algorithm_parameters[
'shift_to_start_of_window']
except KeyError:
shift_to_start_of_window = True
except Exception as e:
record_algorithm_error(current_skyline_app, parent_pid, algorithm_name,
traceback.format_exc())
dev_null = e
save_plots_to = False
try:
save_plots_to = algorithm_parameters['save_plots_to']
except KeyError:
save_plots_to = False
except Exception as e:
record_algorithm_error(current_skyline_app, parent_pid, algorithm_name,
traceback.format_exc())
dev_null = e
save_plots_to_absolute_dir = False
try:
save_plots_to_absolute_dir = algorithm_parameters[
'save_plots_to_absolute_dir']
except KeyError:
save_plots_to_absolute_dir = False
except Exception as e:
record_algorithm_error(current_skyline_app, parent_pid, algorithm_name,
traceback.format_exc())
dev_null = e
filename_prefix = False
try:
filename_prefix = algorithm_parameters['filename_prefix']
except KeyError:
filename_prefix = False
except Exception as e:
record_algorithm_error(current_skyline_app, parent_pid, algorithm_name,
traceback.format_exc())
dev_null = e
if debug_logging:
current_logger.debug('debug :: algorithm_parameters :: %s' %
(str(algorithm_parameters)))
return_anomalies = False
try:
return_anomalies = algorithm_parameters['return_anomalies']
except KeyError:
return_anomalies = False
except Exception as e:
record_algorithm_error(current_skyline_app, parent_pid, algorithm_name,
traceback.format_exc())
dev_null = e
try:
realtime_analysis = algorithm_parameters['realtime_analysis']
except KeyError:
realtime_analysis = False
except Exception as e:
record_algorithm_error(current_skyline_app, parent_pid, algorithm_name,
traceback.format_exc())
dev_null = e
save_plots_to = False
try:
save_plots_to = algorithm_parameters['save_plots_to']
except KeyError:
save_plots_to = False
except Exception as e:
record_algorithm_error(current_skyline_app, parent_pid, algorithm_name,
traceback.format_exc())
dev_null = e
save_plots_to_absolute_dir = False
try:
save_plots_to_absolute_dir = algorithm_parameters[
'save_plots_to_absolute_dir']
except KeyError:
save_plots_to = False
except Exception as e:
record_algorithm_error(current_skyline_app, parent_pid, algorithm_name,
traceback.format_exc())
dev_null = e
filename_prefix = False
try:
filename_prefix = algorithm_parameters['filename_prefix']
except KeyError:
filename_prefix = False
except Exception as e:
record_algorithm_error(current_skyline_app, parent_pid, algorithm_name,
traceback.format_exc())
dev_null = e
try:
base_name = algorithm_parameters['base_name']
except Exception as e:
# This except pattern MUST be used in ALL custom algortihms to
# facilitate the traceback from any errors. The algorithm we want to
# run super fast and without spamming the log with lots of errors.
# But we do not want the function returning and not reporting
# anything to the log, so the pythonic except is used to "sample" any
# algorithm errors to a tmp file and report once per run rather than
# spewing tons of errors into the log e.g. analyzer.log
record_algorithm_error(current_skyline_app, parent_pid, algorithm_name,
traceback.format_exc())
# Return None and None as the algorithm could not determine True or False
dev_null = e
del dev_null
if current_skyline_app == 'webapp':
return (anomalous, anomalyScore, anomalies, anomalies_dict)
if return_anomalies:
return (False, None, anomalies)
return (False, None)
if debug_logging:
current_logger.debug('debug :: %s :: base_name - %s' %
(algorithm_name, str(base_name)))
anomalies_dict['metric'] = base_name
anomalies_dict['anomalies'] = {}
use_bottleneck = True
if save_plots_to:
use_bottleneck = False
if use_bottleneck:
import bottleneck as bn
# ALWAYS WRAP YOUR ALGORITHM IN try and the BELOW except
try:
start_preprocessing = timer()
# INFO: Sorting time series of 10079 data points took 0.002215 seconds
timeseries = sorted(timeseries, key=lambda x: x[0])
if debug_logging:
current_logger.debug('debug :: %s :: time series of length - %s' %
(algorithm_name, str(len(timeseries))))
# Testing the data to ensure it meets minimum requirements, in the case
# of Skyline's use of the m66 algorithm this means that:
# - the time series must have at least 75% of its full_duration
do_not_use_sparse_data = False
if current_skyline_app == 'luminosity':
do_not_use_sparse_data = True
if minimum_sparsity == 0:
do_not_use_sparse_data = False
total_period = 0
total_datapoints = 0
calculate_variables = False
if do_not_use_sparse_data:
calculate_variables = True
if determine_duration:
calculate_variables = True
if calculate_variables:
try:
start_timestamp = int(timeseries[0][0])
end_timestamp = int(timeseries[-1][0])
total_period = end_timestamp - start_timestamp
total_datapoints = len(timeseries)
except SystemExit as e:
if debug_logging:
current_logger.debug(
'debug_logging :: %s :: SystemExit called, exiting - %s'
% (algorithm_name, e))
if current_skyline_app == 'webapp':
return (anomalous, anomalyScore, anomalies, anomalies_dict)
if return_anomalies:
return (anomalous, anomalyScore, anomalies)
return (anomalous, anomalyScore)
except:
traceback_msg = traceback.format_exc()
record_algorithm_error(current_skyline_app, parent_pid,
algorithm_name, traceback_msg)
if debug_logging:
current_logger.error(traceback_msg)
current_logger.error(
'error :: debug_logging :: %s :: failed to determine total_period and total_datapoints'
% (algorithm_name))
timeseries = []
if not timeseries:
if current_skyline_app == 'webapp':
return (anomalous, anomalyScore, anomalies, anomalies_dict)
if return_anomalies:
return (anomalous, anomalyScore, anomalies)
return (anomalous, anomalyScore)
if current_skyline_app == 'analyzer':
# Default for analyzer at required period to 18 hours
period_required = int(FULL_DURATION * 0.75)
else:
# Determine from timeseries
if total_period < FULL_DURATION:
period_required = int(FULL_DURATION * 0.75)
else:
period_required = int(total_period * 0.75)
if determine_duration:
period_required = int(total_period * 0.75)
if do_not_use_sparse_data:
# If the time series does not have 75% of its full_duration it does
# not have sufficient data to sample
try:
if total_period < period_required:
if debug_logging:
current_logger.debug(
'debug :: %s :: time series does not have sufficient data'
% (algorithm_name))
if current_skyline_app == 'webapp':
return (anomalous, anomalyScore, anomalies,
anomalies_dict)
if return_anomalies:
return (anomalous, anomalyScore, anomalies)
return (anomalous, anomalyScore)
except SystemExit as e:
if debug_logging:
current_logger.debug(
'debug_logging :: %s :: SystemExit called, exiting - %s'
% (algorithm_name, e))
if current_skyline_app == 'webapp':
return (anomalous, anomalyScore, anomalies, anomalies_dict)
if return_anomalies:
return (anomalous, anomalyScore, anomalies)
return (anomalous, anomalyScore)
except:
traceback_msg = traceback.format_exc()
record_algorithm_error(current_skyline_app, parent_pid,
algorithm_name, traceback_msg)
if debug_logging:
current_logger.error(traceback_msg)
current_logger.error(
'error :: debug_logging :: %s :: falied to determine if time series has sufficient data'
% (algorithm_name))
if current_skyline_app == 'webapp':
return (anomalous, anomalyScore, anomalies, anomalies_dict)
if return_anomalies:
return (anomalous, anomalyScore, anomalies)
return (anomalous, anomalyScore)
# If the time series does not have 75% of its full_duration
# datapoints it does not have sufficient data to sample
# Determine resolution from the last 30 data points
# INFO took 0.002060 seconds
if not resolution:
resolution_timestamps = []
metric_resolution = False
for metric_datapoint in timeseries[-30:]:
timestamp = int(metric_datapoint[0])
resolution_timestamps.append(timestamp)
timestamp_resolutions = []
if resolution_timestamps:
last_timestamp = None
for timestamp in resolution_timestamps:
if last_timestamp:
resolution = timestamp - last_timestamp
timestamp_resolutions.append(resolution)
last_timestamp = timestamp
else:
last_timestamp = timestamp
try:
del resolution_timestamps
except:
pass
if timestamp_resolutions:
try:
timestamp_resolutions_count = Counter(
timestamp_resolutions)
ordered_timestamp_resolutions_count = timestamp_resolutions_count.most_common(
)
metric_resolution = int(
ordered_timestamp_resolutions_count[0][0])
except SystemExit as e:
if debug_logging:
current_logger.debug(
'debug_logging :: %s :: SystemExit called, exiting - %s'
% (algorithm_name, e))
if current_skyline_app == 'webapp':
return (anomalous, anomalyScore, anomalies,
anomalies_dict)
if return_anomalies:
return (anomalous, anomalyScore, anomalies)
return (anomalous, anomalyScore)
except:
traceback_msg = traceback.format_exc()
record_algorithm_error(current_skyline_app, parent_pid,
algorithm_name, traceback_msg)
if debug_logging:
current_logger.error(traceback_msg)
current_logger.error(
'error :: debug_logging :: %s :: failed to determine if time series has sufficient data'
% (algorithm_name))
try:
del timestamp_resolutions
except:
pass
else:
metric_resolution = resolution
minimum_datapoints = None
if metric_resolution:
minimum_datapoints = int(period_required / metric_resolution)
if minimum_datapoints:
if total_datapoints < minimum_datapoints:
if debug_logging:
current_logger.debug(
'debug :: %s :: time series does not have sufficient data, minimum_datapoints required is %s and time series has %s'
% (algorithm_name, str(minimum_datapoints),
str(total_datapoints)))
if current_skyline_app == 'webapp':
return (anomalous, anomalyScore, anomalies,
anomalies_dict)
if return_anomalies:
return (anomalous, anomalyScore, anomalies)
return (anomalous, anomalyScore)
# Is the time series fully populated?
# full_duration_datapoints = int(full_duration / metric_resolution)
total_period_datapoints = int(total_period / metric_resolution)
# minimum_percentage_sparsity = 95
minimum_percentage_sparsity = 90
sparsity = int(total_datapoints / (total_period_datapoints / 100))
if sparsity < minimum_percentage_sparsity:
if debug_logging:
current_logger.debug(
'debug :: %s :: time series does not have sufficient data, minimum_percentage_sparsity required is %s and time series has %s'
% (algorithm_name, str(minimum_percentage_sparsity),
str(sparsity)))
if current_skyline_app == 'webapp':
return (anomalous, anomalyScore, anomalies, anomalies_dict)
if return_anomalies:
return (anomalous, anomalyScore, anomalies)
return (anomalous, anomalyScore)
if len(set(item[1] for item in timeseries)) == 1:
if debug_logging:
current_logger.debug(
'debug :: %s :: time series does not have sufficient variability, all the values are the same'
% algorithm_name)
anomalous = False
anomalyScore = 0.0
if current_skyline_app == 'webapp':
return (anomalous, anomalyScore, anomalies, anomalies_dict)
if return_anomalies:
return (anomalous, anomalyScore, anomalies)
return (anomalous, anomalyScore)
end_preprocessing = timer()
preprocessing_runtime = end_preprocessing - start_preprocessing
if debug_logging:
current_logger.debug(
'debug :: %s :: preprocessing took %.6f seconds' %
(algorithm_name, preprocessing_runtime))
if not timeseries:
if debug_logging:
current_logger.debug('debug :: %s :: m66 not run as no data' %
(algorithm_name))
anomalies = []
if current_skyline_app == 'webapp':
return (anomalous, anomalyScore, anomalies, anomalies_dict)
if return_anomalies:
return (anomalous, anomalyScore, anomalies)
return (anomalous, anomalyScore)
if debug_logging:
current_logger.debug('debug :: %s :: timeseries length: %s' %
(algorithm_name, str(len(timeseries))))
anomalies_dict['timestamp'] = int(timeseries[-1][0])
anomalies_dict['from_timestamp'] = int(timeseries[0][0])
start_analysis = timer()
try:
# bottleneck is used because it is much faster
# pd dataframe method (1445 data point - 24hrs): took 0.077915 seconds
# bottleneck method (1445 data point - 24hrs): took 0.005692 seconds
# numpy and pandas rolling
# 2021-07-30 12:37:31 :: 2827897 :: cloudbursts :: find_cloudbursts completed on 1530 metrics in 136.93 seconds
# 2021-07-30 12:44:53 :: 2855884 :: cloudbursts :: find_cloudbursts completed on 1530 metrics in 148.82 seconds
# 2021-07-30 12:48:41 :: 2870822 :: cloudbursts :: find_cloudbursts completed on 1530 metrics in 145.62 seconds
# 2021-07-30 12:55:00 :: 2893634 :: cloudbursts :: find_cloudbursts completed on 1530 metrics in 139.00 seconds
# 2021-07-30 12:59:31 :: 2910443 :: cloudbursts :: find_cloudbursts completed on 1530 metrics in 144.80 seconds
# 2021-07-30 13:02:31 :: 2922928 :: cloudbursts :: find_cloudbursts completed on 1530 metrics in 143.35 seconds
# 2021-07-30 14:12:56 :: 3132457 :: cloudbursts :: find_cloudbursts completed on 1530 metrics in 129.25 seconds
# 2021-07-30 14:22:35 :: 3164370 :: cloudbursts :: find_cloudbursts completed on 1530 metrics in 125.72 seconds
# 2021-07-30 14:28:24 :: 3179687 :: cloudbursts :: find_cloudbursts completed on 1530 metrics in 222.43 seconds
# 2021-07-30 14:33:45 :: 3179687 :: cloudbursts :: find_cloudbursts completed on 1530 metrics in 244.00 seconds
# 2021-07-30 14:36:27 :: 3214047 :: cloudbursts :: find_cloudbursts completed on 1530 metrics in 141.10 seconds
# numpy and bottleneck
# 2021-07-30 16:41:52 :: 3585162 :: cloudbursts :: find_cloudbursts completed on 1530 metrics in 73.92 seconds
# 2021-07-30 16:46:46 :: 3585162 :: cloudbursts :: find_cloudbursts completed on 1530 metrics in 68.84 seconds
# 2021-07-30 16:51:48 :: 3585162 :: cloudbursts :: find_cloudbursts completed on 1530 metrics in 70.55 seconds
# numpy and bottleneck (passing resolution and not calculating in m66)
# 2021-07-30 16:57:46 :: 3643253 :: cloudbursts :: find_cloudbursts completed on 1530 metrics in 65.59 seconds
if use_bottleneck:
if len(timeseries) < 10:
if current_skyline_app == 'webapp':
return (anomalous, anomalyScore, anomalies,
anomalies_dict)
if return_anomalies:
return (anomalous, anomalyScore, anomalies)
return (anomalous, anomalyScore)
x_np = np.asarray([x[1] for x in timeseries])
# Fast Min-Max scaling
data = (x_np - x_np.min()) / (x_np.max() - x_np.min())
# m66 - calculate to nth_median
median_count = 0
while median_count < nth_median:
median_count += 1
rolling_median_s = bn.move_median(data, window=window)
median = rolling_median_s.tolist()
data = median
if median_count == nth_median:
break
# m66 - calculate the moving standard deviation for the
# nth_median array
rolling_std_s = bn.move_std(data, window=window)
std_nth_median_array = np.nan_to_num(rolling_std_s,
copy=False,
nan=0.0,
posinf=None,
neginf=None)
std_nth_median = std_nth_median_array.tolist()
if debug_logging:
current_logger.debug(
'debug :: %s :: std_nth_median calculated with bn' %
(algorithm_name))
else:
df = pd.DataFrame(timeseries, columns=['date', 'value'])
df['date'] = pd.to_datetime(df['date'], unit='s')
datetime_index = pd.DatetimeIndex(df['date'].values)
df = df.set_index(datetime_index)
df.drop('date', axis=1, inplace=True)
original_df = df.copy()
# MinMax scale
df = (df - df.min()) / (df.max() - df.min())
# window = 6
data = df['value'].tolist()
if len(data) < 10:
if current_skyline_app == 'webapp':
return (anomalous, anomalyScore, anomalies,
anomalies_dict)
if return_anomalies:
return (anomalous, anomalyScore, anomalies)
return (anomalous, anomalyScore)
# m66 - calculate to nth_median
median_count = 0
while median_count < nth_median:
median_count += 1
s = pd.Series(data)
rolling_median_s = s.rolling(window).median()
median = rolling_median_s.tolist()
data = median
if median_count == nth_median:
break
# m66 - calculate the moving standard deviation for the
# nth_median array
s = pd.Series(data)
rolling_std_s = s.rolling(window).std()
nth_median_column = 'std_nth_median_%s' % str(nth_median)
df[nth_median_column] = rolling_std_s.tolist()
std_nth_median = df[nth_median_column].fillna(0).tolist()
# m66 - calculate the standard deviation for the entire nth_median
# array
metric_stddev = np.std(std_nth_median)
std_nth_median_n_sigma = []
anomalies_found = False
for value in std_nth_median:
# m66 - if the value in the 6th median array is > six-sigma of
# the metric_stddev the datapoint is anomalous
if value > (metric_stddev * n_sigma):
std_nth_median_n_sigma.append(1)
anomalies_found = True
else:
std_nth_median_n_sigma.append(0)
std_nth_median_n_sigma_column = 'std_median_%s_%s_sigma' % (
str(nth_median), str(n_sigma))
if not use_bottleneck:
df[std_nth_median_n_sigma_column] = std_nth_median_n_sigma
anomalies = []
# m66 - only label anomalous if the n_sigma triggers are persisted
# for (window / 2)
if anomalies_found:
current_triggers = []
for index, item in enumerate(timeseries):
if std_nth_median_n_sigma[index] == 1:
current_triggers.append(index)
else:
if len(current_triggers) > int(window / 2):
for trigger_index in current_triggers:
# Shift the anomaly back to the beginning of the
# window
if shift_to_start_of_window:
anomalies.append(
timeseries[(trigger_index -
(window * int(
(nth_median / 2))))])
else:
anomalies.append(timeseries[trigger_index])
current_triggers = []
# Process any remaining current_triggers
if len(current_triggers) > int(window / 2):
for trigger_index in current_triggers:
# Shift the anomaly back to the beginning of the
# window
if shift_to_start_of_window:
anomalies.append(
timeseries[(trigger_index - (window * int(
(nth_median / 2))))])
else:
anomalies.append(timeseries[trigger_index])
if not anomalies:
anomalous = False
if anomalies:
anomalous = True
anomalies_data = []
anomaly_timestamps = [int(item[0]) for item in anomalies]
for item in timeseries:
if int(item[0]) in anomaly_timestamps:
anomalies_data.append(1)
else:
anomalies_data.append(0)
if not use_bottleneck:
df['anomalies'] = anomalies_data
anomalies_list = []
for ts, value in timeseries:
if int(ts) in anomaly_timestamps:
anomalies_list.append([int(ts), value])
anomalies_dict['anomalies'][int(ts)] = value
if anomalies and save_plots_to:
try:
from adtk.visualization import plot
metric_dir = base_name.replace('.', '/')
timestamp_dir = str(int(timeseries[-1][0]))
save_path = '%s/%s/%s/%s' % (save_plots_to, algorithm_name,
metric_dir, timestamp_dir)
if save_plots_to_absolute_dir:
save_path = '%s' % save_plots_to
anomalies_dict['file_path'] = save_path
save_to_file = '%s/%s.%s.png' % (save_path, algorithm_name,
base_name)
if filename_prefix:
save_to_file = '%s/%s.%s.%s.png' % (
save_path, filename_prefix, algorithm_name,
base_name)
save_to_path = os_path_dirname(save_to_file)
title = '%s\n%s - median %s %s-sigma persisted (window=%s)' % (
base_name, algorithm_name, str(nth_median),
str(n_sigma), str(window))
if not os_path_exists(save_to_path):
try:
mkdir_p(save_to_path)
except Exception as e:
current_logger.error(
'error :: %s :: failed to create dir - %s - %s'
% (algorithm_name, save_to_path, e))
if os_path_exists(save_to_path):
try:
plot(original_df['value'],
anomaly=df['anomalies'],
anomaly_color='red',
title=title,
save_to_file=save_to_file)
if debug_logging:
current_logger.debug(
'debug :: %s :: plot saved to - %s' %
(algorithm_name, save_to_file))
anomalies_dict['image'] = save_to_file
except Exception as e:
current_logger.error(
'error :: %s :: failed to plot - %s - %s' %
(algorithm_name, base_name, e))
anomalies_file = '%s/%s.%s.anomalies_list.txt' % (
save_path, algorithm_name, base_name)
with open(anomalies_file, 'w') as fh:
fh.write(str(anomalies_list))
# os.chmod(anomalies_file, mode=0o644)
data_file = '%s/data.txt' % (save_path)
with open(data_file, 'w') as fh:
fh.write(str(anomalies_dict))
except SystemExit as e:
if debug_logging:
current_logger.debug(
'debug_logging :: %s :: SystemExit called during save plot, exiting - %s'
% (algorithm_name, e))
if current_skyline_app == 'webapp':
return (anomalous, anomalyScore, anomalies,
anomalies_dict)
if return_anomalies:
return (anomalous, anomalyScore, anomalies)
return (anomalous, anomalyScore)
except Exception as e:
traceback_msg = traceback.format_exc()
record_algorithm_error(current_skyline_app, parent_pid,
algorithm_name, traceback_msg)
if debug_logging:
current_logger.error(traceback_msg)
current_logger.error(
'error :: %s :: failed to plot or save anomalies file - %s - %s'
% (algorithm_name, base_name, e))
try:
del df
except:
pass
except SystemExit as e:
if debug_logging:
current_logger.debug(
'debug_logging :: %s :: SystemExit called, during analysis, exiting - %s'
% (algorithm_name, e))
if current_skyline_app == 'webapp':
return (anomalous, anomalyScore, anomalies, anomalies_dict)
if return_anomalies:
return (anomalous, anomalyScore, anomalies)
return (anomalous, anomalyScore)
except:
traceback_msg = traceback.format_exc()
record_algorithm_error(current_skyline_app, parent_pid,
algorithm_name, traceback_msg)
if debug_logging:
current_logger.error(traceback_msg)
current_logger.error(
'error :: debug_logging :: %s :: failed to run on ts' %
(algorithm_name))
if current_skyline_app == 'webapp':
return (anomalous, anomalyScore, anomalies, anomalies_dict)
if return_anomalies:
return (anomalous, anomalyScore, anomalies)
return (anomalous, anomalyScore)
end_analysis = timer()
analysis_runtime = end_analysis - start_analysis
if debug_logging:
current_logger.debug(
'debug :: analysis with %s took %.6f seconds' %
(algorithm_name, analysis_runtime))
if anomalous:
anomalyScore = 1.0
else:
anomalyScore = 0.0
if debug_logging:
current_logger.info(
'%s :: anomalous - %s, anomalyScore - %s' %
(algorithm_name, str(anomalous), str(anomalyScore)))
if debug_logging:
end = timer()
processing_runtime = end - start
current_logger.info('%s :: completed in %.6f seconds' %
(algorithm_name, processing_runtime))
try:
del timeseries
except:
pass
if current_skyline_app == 'webapp':
return (anomalous, anomalyScore, anomalies, anomalies_dict)
if return_anomalies:
return (anomalous, anomalyScore, anomalies)
return (anomalous, anomalyScore)
except SystemExit as e:
if debug_logging:
current_logger.debug(
'debug_logging :: %s :: SystemExit called (before StopIteration), exiting - %s'
% (algorithm_name, e))
if current_skyline_app == 'webapp':
return (anomalous, anomalyScore, anomalies, anomalies_dict)
if return_anomalies:
return (anomalous, anomalyScore, anomalies)
return (anomalous, anomalyScore)
except StopIteration:
# This except pattern MUST be used in ALL custom algortihms to
# facilitate the traceback from any errors. The algorithm we want to
# run super fast and without spamming the log with lots of errors.
# But we do not want the function returning and not reporting
# anything to the log, so the pythonic except is used to "sample" any
# algorithm errors to a tmp file and report once per run rather than
# spewing tons of errors into the log e.g. analyzer.log
if current_skyline_app == 'webapp':
return (anomalous, anomalyScore, anomalies, anomalies_dict)
if return_anomalies:
return (False, None, anomalies)
return (False, None)
except:
record_algorithm_error(current_skyline_app, parent_pid, algorithm_name,
traceback.format_exc())
# Return None and None as the algorithm could not determine True or False
if current_skyline_app == 'webapp':
return (anomalous, anomalyScore, anomalies, anomalies_dict)
if return_anomalies:
return (False, None, anomalies)
return (False, None)
if current_skyline_app == 'webapp':
return (anomalous, anomalyScore, anomalies, anomalies_dict)
if return_anomalies:
return (anomalous, anomalyScore, anomalies)
return (anomalous, anomalyScore)
a = model.fit_detect(df.iloc[:, i])
pd.testing.assert_series_equal(a, a_true.iloc[:, i], check_dtype=False)
a = model.fit_detect(df)
pd.testing.assert_frame_equal(a, a_true, check_dtype=False)
if __name__ == '__main__':
logins_per_week_filter_new = [
20.539924073521295, 4.709207794178407, 10.567975521943765,
4.654008712203323, 12.540417874779518, 8.68124128274528,
14.510354653083763, 18.506005440002554, 8.653713996141803,
2.6067706565147417, 8.568764060326458, 6.497231646713441,
2.657897098432535, 2.4346225766375302, 0.6283913785856533,
0.2832893392222861, 0.5798709739436374, 0.6129681116308826,
1.1067186413836192, 1.409769359194367, 1.638155640892966
]
from adtk.visualization import plot
from adtk.detector import PersistAD
from adtk.data import validate_series
s = validate_series(logins_per_week_filter_new)
persist_ad = PersistAD(c=0.5,
side='negative',
agg='median',
lower_threshold=15.0)
persist_ad.window = 4
anomalies = persist_ad.fit_detect(s)
plot(s, anomaly=anomalies, anomaly_color='red')
from adtk.data import validate_series
from adtk.visualization import plot
from adtk.detector import PersistAD
from adtk.transformer import DoubleRollingAggregate
from adtk.pipe import Pipeline
##############################################################################################################
##Import and Validate the Dataset
s_train = pd.read_csv("./anomalies.csv",
index_col="timestamp",
parse_dates=True,
squeeze=True)
s_train = validate_series(s_train)
print(s_train)
plot(
s_train) ##plot Function Draws a Chart but The Chart Is in JPEG Format !!!
##############################################################################################################
##PersistAD Detects Spikes (Extremely Abnormal Values)
##High Tolerance Model
persist_ad = PersistAD(
agg='mean', side='both',
c=6) ##Side Parameter Filters Positive and Negative Sided Anomalies
anomalies_1 = persist_ad.fit_detect(s_train, return_list=True)
plot(s_train, anomaly=anomalies_1, anomaly_color="red", anomaly_tag="marker")
for i in anomalies_1:
print(i)
print(len(anomalies_1))
df.drop(['date'], axis=1, inplace=True)
df.head()
s_train = df
# the same for the label
df = pd.DataFrame(dti, columns=['date'])
df[1] = (Y)
df['datetime'] = pd.to_datetime(df['date'])
df = df.set_index('datetime')
df.drop(['date'], axis=1, inplace=True)
df.head()
from adtk.data import to_events
known_anomalies = to_events(df)
from adtk.visualization import plot
plot(s_train, anomaly_true=known_anomalies)
plt.plot(Y)
from adtk.detector import SeasonalAD
seasonal_ad = SeasonalAD()
anomalies = seasonal_ad.fit_detect(s_train)
plot(s_train, anomaly_pred=anomalies, ap_color='red', ap_marker_on_curve=True)
from adtk.detector import LevelShiftAD
levelshift_ad = LevelShiftAD()
anomalies = levelshift_ad.fit_detect(s_train)
plot(s_train, anomaly_pred=anomalies, ap_color='red', ap_marker_on_curve=True)
from adtk.detector import MinClusterDetector
from sklearn.cluster import KMeans
min_cluster_detector = MinClusterDetector(KMeans(n_clusters=3))
#from adtk.detector import GeneralizedESDTestAD
#esd_ad = GeneralizedESDTestAD(alpha=0.3)
#anomalies = esd_ad.fit_detect(traincl)
#anomalies = esd_ad.fit(trainredox)
#q=plot(traincl,title='Generalized Extreme studentized Deviate Test on Redox', anomaly_pred=anomalies, ts_linewidth=2, ts_markersize=3, ap_markersize=5, ap_color='red', ap_marker_on_curve=True);
#m=esd_ad.fit_predict(testredox,anomalies)
from adtk.detector import AutoregressionAD
autoregression_ad = AutoregressionAD(c=4.0)
anomalies = autoregression_ad.fit(trainredox) #Fitting the model
m1 = autoregression_ad.fit_predict(testredox, anomalies) #predicting the model
anomaliesde = autoregression_ad.fit_detect(testredox)
plot(testredox,
anomaly_pred=anomaliesde,
ts_linewidth=1,
ap_color='red',
ap_marker_on_curve=True)
from adtk.detector import AutoregressionAD
autoregression_ad1 = AutoregressionAD(c=4.0)
anomalies1 = autoregression_ad.fit(traincl2) #Fitting the model
m = autoregression_ad1.fit_predict(testcl2,
anomalies1) ##predicting the model.
anomaliesde = autoregression_ad.fit_detect(testcl2)
plot(testcl2,
anomaly_pred=anomaliesde,
ts_linewidth=1,
ap_color='red',
ap_marker_on_curve=True)
def panorama_plot_anomalies(base_name,
from_timestamp=None,
until_timestamp=None):
"""
Create a plot of the metric with its anomalies and return the anomalies dict
and the path and filename
:param base_name: the name of the metric
:param from_timestamp: the from timestamp
:param until_timestamp: the until timestamp
:type base_name: str
:type from_timestamp: int
:type until_timestamp: int
:return: (anomalies_dict, path and file)
:rtype: tuple
"""
function_str = 'panorama_plot_anomalies'
logger.info('%s - base_name: %s, from_timestamp: %s, until_timestamp: %s' %
(function_str, str(base_name), str(from_timestamp),
str(until_timestamp)))
if not until_timestamp:
until_timestamp = int(time())
save_to_file = '%s/panorama_anomalies_plot.%s.%s.%s.png' % (
settings.SKYLINE_TMP_DIR, base_name, str(from_timestamp),
str(until_timestamp))
try:
metric_id = get_metric_id_from_base_name(skyline_app, base_name)
logger.info('%s - %s with metric id:%s' %
(function_str, str(base_name), str(metric_id)))
except Exception as err:
logger.error(traceback.format_exc())
logger.error(
'error :: %s :: failed to determine metric id for %s - %s' %
(function_str, base_name, err))
raise
try:
anomalies_dict = get_anomalies(skyline_app,
metric_id,
params={'latest': False})
except Exception as err:
logger.error(traceback.format_exc())
logger.error(
'error :: %s :: failed to determine anomalies for %s - %s' %
(function_str, base_name, err))
raise
if from_timestamp and anomalies_dict:
for anomaly_id in list(anomalies_dict.keys()):
if anomalies_dict[anomaly_id]['anomaly_timestamp'] < from_timestamp:
del anomalies_dict[anomaly_id]
if until_timestamp and anomalies_dict:
for anomaly_id in list(anomalies_dict.keys()):
if anomalies_dict[anomaly_id][
'anomaly_timestamp'] > until_timestamp:
del anomalies_dict[anomaly_id]
if os.path.isfile(save_to_file):
return anomalies_dict, save_to_file
if not from_timestamp and anomalies_dict:
first_anomaly_id = list(anomalies_dict.keys())[-1]
first_anomaly_timestamp = anomalies_dict[first_anomaly_id][
'anomaly_timestamp']
from_timestamp = first_anomaly_timestamp - (86400 * 7)
logger.info(
'%s :: the from_timestamp was not passed, calculated from the anomalies_dict as %s'
% (function_str, str(from_timestamp)))
if not from_timestamp and not anomalies_dict:
logger.info(
'%s :: the from_timestamp was not passed and no anomalies found for %s'
% (function_str, base_name))
from_timestamp = until_timestamp - (86400 * 7)
metrics_functions = {}
metrics_functions[base_name] = {}
metrics_functions[base_name]['functions'] = None
try:
metrics_timeseries = get_metrics_timeseries(skyline_app,
metrics_functions,
from_timestamp,
until_timestamp,
log=False)
except Exception as err:
logger.error(traceback.format_exc())
logger.error('error :: %s :: get_metrics_timeseries failed - %s' %
(function_str, err))
raise
try:
timeseries = metrics_timeseries[base_name]['timeseries']
# Truncate the first and last timestamp, just in case they are not
# filled buckets
timeseries = timeseries[1:-1]
except Exception as err:
logger.error(traceback.format_exc())
logger.error('error :: %s :: failed to get timeseries for %s - %s' %
(function_str, base_name, err))
raise
unaligned_anomaly_timestamps = []
for anomaly_id in list(anomalies_dict.keys()):
unaligned_anomaly_timestamps.append(
anomalies_dict[anomaly_id]['anomaly_timestamp'])
# Align anomalies to timeseries resolution
resolution = determine_data_frequency(skyline_app, timeseries, False)
anomaly_timestamps = []
for ts in unaligned_anomaly_timestamps:
anomaly_timestamps.append(int(int(ts) // resolution * resolution))
try:
df = pd.DataFrame(timeseries, columns=['date', 'value'])
df['date'] = pd.to_datetime(df['date'], unit='s')
datetime_index = pd.DatetimeIndex(df['date'].values)
df = df.set_index(datetime_index)
df.drop('date', axis=1, inplace=True)
anomalies_data = []
for item in timeseries:
if int(item[0]) in anomaly_timestamps:
anomalies_data.append(1)
else:
anomalies_data.append(0)
df['anomalies'] = anomalies_data
title = '%s\n%s anomalies' % (base_name, str(len(anomaly_timestamps)))
plot(df['value'],
anomaly=df['anomalies'],
anomaly_color='red',
title=title,
save_to_file=save_to_file)
except Exception as err:
logger.error(traceback.format_exc())
logger.error('error :: %s :: failed to plot anomalies for %s - %s' %
(function_str, base_name, err))
raise
if not os.path.isfile(save_to_file):
return anomalies_dict, None
return anomalies_dict, save_to_file
m = esd_ad.fit_predict(
testdata2,
anomalies) #predicting on the test redox and Cl_2 with pca algorithm
pca_ad = PcaAD(k=2)
anomalies1 = pca_ad.fit(
traindata1) #training redox and PH with pca algorithm
m1 = pca_ad.fit_predict(
testdata1,
anomalies1) #predicting on the test redox and pH with PCA algorithm
f = pd.merge(
pd.DataFrame(m),
pd.DataFrame(m1),
left_index=True,
right_index=True,
how='outer'
) #Merging the anomalies in both the algorithm in same data frame
k = OrAggregator().aggregate(f) #ORing the result of both
allk = pd.concat([allk, k])
ConfusionMatrix = confusion_matrix(
df2['EVENT'].astype(bool)[4 * 1440:(index + 5 * 1440)], allk.astype(bool))
print(ConfusionMatrix)
target_names = ['False', 'True']
print(
classification_report(df2['EVENT'].astype(bool)[4 * 1440:(index +
5 * 1440)],
allk.astype(bool),
target_names=target_names))
plot(allk, title='Aggregated Anomalies plot', ts_color='red', ts_linewidth='3')
plot(df2['EVENT'], title='Events plot', ts_color='green', ts_linewidth='3')
def get_cloudburst_plot(cloudburst_id, base_name, shift, all_in_period=False):
"""
Create a plot of the cloudburst and return the path and filename
:param cloudburst_id: the cloudburt id
:param base_name: the name of the metric
:param shift: the number of indice to shift the plot
:type cloudburst_id: int
:type base_name: str
:type shift: int
:return: path and file
:rtype: str
"""
function_str = 'get_cloudburst_plot'
logger.info(
'get_cloudburst_plot - cloudburst_id: %s, base_name: %s' % (
str(cloudburst_id), str(base_name)))
save_to_file = '%s/cloudburst_id.%s.%s.shift.%s.png' % (
settings.SKYLINE_TMP_DIR, str(cloudburst_id), base_name, str(shift))
if all_in_period:
save_to_file = '%s/cloudburst_id.%s.all.%s.shift.%s.png' % (
settings.SKYLINE_TMP_DIR, str(cloudburst_id), base_name, str(shift))
cloudburst_dict = {}
try:
cloudburst_dict = get_cloudburst_row(skyline_app, cloudburst_id)
except Exception as err:
logger.error(traceback.format_exc())
logger.error('error :: %s :: get_cloudburst_row failed - %s' % (
function_str, err))
raise
if not cloudburst_dict:
logger.error('error :: %s :: no cloudburst_dict - %s' % function_str)
return None, None
if os.path.isfile(save_to_file):
return cloudburst_dict, save_to_file
try:
from_timestamp = cloudburst_dict['from_timestamp']
until_timestamp = from_timestamp + cloudburst_dict['full_duration']
resolution = cloudburst_dict['resolution']
except Exception as err:
logger.error(traceback.format_exc())
logger.error('error :: %s :: failed parse values from cloudburst_dict - %s' % (
function_str, err))
raise
metrics_functions = {}
metrics_functions[base_name] = {}
metrics_functions[base_name]['functions'] = None
if resolution > 60:
resolution_minutes = int(resolution / 60)
summarize_intervalString = '%smin' % str(resolution_minutes)
summarize_func = 'median'
metrics_functions[base_name]['functions'] = {'summarize': {'intervalString': summarize_intervalString, 'func': summarize_func}}
try:
metrics_timeseries = get_metrics_timeseries(skyline_app, metrics_functions, from_timestamp, until_timestamp, log=False)
except Exception as err:
logger.error(traceback.format_exc())
logger.error('error :: %s :: get_metrics_timeseries failed - %s' % (
function_str, err))
raise
try:
timeseries = metrics_timeseries[base_name]['timeseries']
timeseries_length = len(timeseries)
timeseries = timeseries[1:(timeseries_length - 2)]
except Exception as err:
logger.error(traceback.format_exc())
logger.error('error :: %s :: failed to determine timeseries - %s' % (
function_str, err))
raise
anomalies_in_period = []
if all_in_period:
try:
engine, fail_msg, trace = get_engine(skyline_app)
except Exception as err:
trace = traceback.format_exc()
logger.error(trace)
fail_msg = 'error :: %s :: could not get a MySQL engine - %s' % (function_str, err)
logger.error('%s' % fail_msg)
if engine:
engine_disposal(skyline_app, engine)
raise
try:
cloudburst_table, log_msg, trace = cloudburst_table_meta(skyline_app, engine)
except Exception as err:
logger.error(traceback.format_exc())
logger.error('error :: %s :: failed to get cloudburst_table meta for cloudburst id %s - %s' % (
function_str, str(cloudburst_id), err))
if engine:
engine_disposal(engine)
raise
try:
connection = engine.connect()
stmt = select([cloudburst_table]).\
where(cloudburst_table.c.metric_id == cloudburst_dict['metric_id']).\
where(cloudburst_table.c.timestamp >= from_timestamp).\
where(cloudburst_table.c.timestamp <= until_timestamp).\
where(cloudburst_table.c.id != cloudburst_id)
result = connection.execute(stmt)
for row in result:
anomalies_in_period.append([row['timestamp'], row['end']])
connection.close()
except Exception as err:
logger.error(traceback.format_exc())
logger.error('error :: %s :: could not get cloudburst row for cloudburst id %s - %s' % (
function_str, str(cloudburst_id), err))
if engine:
engine_disposal(engine)
raise
if engine:
engine_disposal(skyline_app, engine)
anomalies = []
if anomalies_in_period:
logger.info(
'get_cloudburst_plot - adding %s all_in_period anomalies to cloudburst plot' % (
str(len(anomalies_in_period))))
for period_anomalies in anomalies_in_period:
new_anomalies = [item for item in timeseries if int(item[0]) >= period_anomalies[0] and int(item[0]) <= period_anomalies[1]]
if new_anomalies:
anomalies = anomalies + new_anomalies
try:
cloudburst_anomalies = [item for item in timeseries if int(item[0]) >= cloudburst_dict['timestamp'] and int(item[0]) <= cloudburst_dict['end']]
anomalies = anomalies + cloudburst_anomalies
df = pd.DataFrame(timeseries, columns=['date', 'value'])
df['date'] = pd.to_datetime(df['date'], unit='s')
datetime_index = pd.DatetimeIndex(df['date'].values)
df = df.set_index(datetime_index)
df.drop('date', axis=1, inplace=True)
anomalies_data = []
# @modified 20210831
# Align periods
# anomaly_timestamps = [int(item[0]) for item in anomalies]
# anomaly_timestamps = [(int(item[0]) + (resolution * 2)) for item in anomalies]
# anomaly_timestamps = [(int(item[0]) + (resolution * 6)) for item in anomalies]
# anomaly_timestamps = [(int(item[0]) + (resolution * 4)) for item in anomalies]
# anomaly_timestamps = [(int(item[0]) + (resolution * 3)) for item in anomalies]
anomaly_timestamps = [(int(item[0]) + (resolution * shift)) for item in anomalies]
for item in timeseries:
if int(item[0]) in anomaly_timestamps:
anomalies_data.append(1)
else:
anomalies_data.append(0)
df['anomalies'] = anomalies_data
title = '%s\ncloudburst id: %s' % (base_name, str(cloudburst_id))
if all_in_period:
title = '%s (all in period)' % title
plot(df['value'], anomaly=df['anomalies'], anomaly_color='red', title=title, save_to_file=save_to_file)
except Exception as err:
logger.error(traceback.format_exc())
logger.error('error :: %s :: failed to plot cloudburst - %s' % (
function_str, err))
raise
if not os.path.isfile(save_to_file):
return cloudburst_dict, None
return cloudburst_dict, save_to_file
df7 = df1[['Scaled Cl_2']]
#df8=df1[['Scaled Leit']]
c = df1['EVENT']
#from adtk.detector import PcaAD
#pca_ad = PcaAD(k=1)
#anomalies= pca_ad.fit_detect(df2)
#p=plot(df2, anomaly_pred=anomalies, ts_linewidth=2, ts_markersize=3, ap_color='red', ap_alpha=0.3, curve_group='all');
from adtk.detector import GeneralizedESDTestAD
esd_ad = GeneralizedESDTestAD(alpha=0.3)
anomalies = esd_ad.fit_detect(df2)
q = plot(df2,
anomaly_pred=anomalies,
ts_linewidth=2,
ts_markersize=3,
ap_markersize=5,
ap_color='red',
ap_marker_on_curve=True)
from adtk.detector import GeneralizedESDTestAD
esd_ad = GeneralizedESDTestAD(alpha=0.3)
anomalies1 = esd_ad.fit_detect(df7)
q = plot(df7,
anomaly_pred=anomalies1,
ts_linewidth=2,
ts_markersize=3,
ap_markersize=5,
ap_color='red',
ap_marker_on_curve=True)
"""
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
data_range = pd.date_range(pd.to_datetime("2020/7/11 10:30:00"), end= pd.to_datetime("2020/7/11 18:00:00"), freq="30T")
x = np.linspace(0, len(data_range), len(data_range))
data = pd.DataFrame({"time": data_range, "value": x})
data.set_index(["time"], inplace=True)
data.iloc[3:4] = 7
data.iloc[10:11] = 3
from adtk.transformer import DoubleRollingAggregate, RollingAggregate
from adtk.detector import ThresholdAD
from adtk.pipe import Pipenet
# 构建异常检测流水线, DoubleRollingAggregate两个滑动窗口并排移动，没有交叉，步长为1，初始的时候左窗口的右边界在数组外，右窗口的左边界在第一个元素
step = {"abs_skipe_change": {"model": DoubleRollingAggregate(agg="mean", window=(1, 1), center=False, diff="l1"),
"input": "original"},
"positive_change": {"model": ThresholdAD(low=0, high=4),
"input": "abs_skipe_change"}
}
mypipenet = Pipenet(steps=step)
anomalies = mypipenet.fit_detect(data, return_list=True, return_intermediate=True)
print(anomalies)
from adtk.visualization import plot
plot(data, anomaly=anomalies, anomaly_color='red', ts_markersize=10, anomaly_markersize=15, ts_linewidth=3, anomaly_alpha=1)
plt.show()
TimeBins = validate_series(TimeBins)
#persist_ad = PersistAD(window=7, c=3, side='both')
#anomalies1 = persist_ad.fit_detect(TimeBins)
#plot(TimeBins, anomaly=anomalies1, ts_linewidth=1, ts_markersize=3, anomaly_color='red', figsize=(20,10), anomaly_tag="marker", anomaly_markersize=5)
#customized_detector = CustomizedDetectorHD(detect_func=Detector_prive)
#anomalies = customized_detector.detect(TimeBins)
#threshold_ad = ThresholdAD(high=150, low=0)
#anomalies = threshold_ad.detect(TimeBins)
#plot(TimeBins, anomaly=anomalies, ts_linewidth=1, ts_markersize=5, anomaly_color='red', anomaly_alpha=0.3, curve_group='all');
outlier_detector = OutlierDetector(
LocalOutlierFactor(n_neighbors=1, p=1, contamination=0.05))
anomalies = outlier_detector.fit_detect(TimeBins)
plot(TimeBins,
anomaly=anomalies,
ts_linewidth=1,
ts_markersize=5,
anomaly_color='red',
anomaly_alpha=0.3,
curve_group='all')
plt.ylim(top=460)
plt.savefig('%d_%d.pdf' % Input2 % elem, bbox_inches='tight')
plt.close()
del TimeBins
del rslt_df
del boolean_condition
def plot_anomalies(current_skyline_app, metric, timeseries, anomalies, title,
output_file):
"""
Create a plot of a timeseries with anomalies and return the path and filename
:param current_skyline_app: skyline_app
:param metric: the name of the metric
:param timeseries: the timeseries to plot
:param anomalies: the anomaly timestamps
:param title: the plot title
:param output_file: the full path and filename (including .png extension) to
save to plot as
:type current_skyline_app: str
:type metric: str
:type timeseries: list
:type anomalies: list
:type title: str
:type output_file: str
:return: output_file
:rtype: str
"""
function_str = 'plot_anomalies'
current_skyline_app_logger = current_skyline_app + 'Log'
current_logger = logging.getLogger(current_skyline_app_logger)
if os.path.isfile(output_file):
current_logger.info('%s :: %s :: plot of %s with %s anomalies exists' %
(str(current_skyline_app), function_str, metric,
str(len(anomalies))))
return output_file
current_logger.info(
'%s :: %s :: plotting %s with %s anomalies' %
(str(current_skyline_app), function_str, metric, str(len(anomalies))))
anomalies_data = []
last_timestamp = None
for item in timeseries:
anomaly_in_period = 0
if not last_timestamp:
last_timestamp = int(item[0])
anomalies_data.append(anomaly_in_period)
continue
for anomaly_ts in anomalies:
if anomaly_ts < last_timestamp:
continue
if anomaly_ts > item[0]:
continue
if anomaly_ts in list(range(last_timestamp, int(item[0]))):
anomaly_in_period = 1
break
anomalies_data.append(anomaly_in_period)
last_timestamp = int(item[0])
try:
df = pd.DataFrame(timeseries, columns=['date', 'value'])
df['date'] = pd.to_datetime(df['date'], unit='s')
datetime_index = pd.DatetimeIndex(df['date'].values)
df = df.set_index(datetime_index)
df.drop('date', axis=1, inplace=True)
df['anomalies'] = anomalies_data
plot(df['value'],
anomaly=df['anomalies'],
anomaly_color='red',
title=title,
save_to_file=output_file)
except Exception as err:
current_logger.error(traceback.format_exc())
current_logger.error('error :: %s :: failed to plot anomalies - %s' %
(function_str, err))
if not os.path.isfile(output_file):
current_logger.error(
'error :: %s :: %s :: plotting %s with %s anomalies failed not output_file exists'
% (str(current_skyline_app), function_str, metric,
str(len(anomalies))))
return None
current_logger.info('%s :: %s :: plotted %s with %s anomalies to %s' %
(str(current_skyline_app), function_str, metric,
str(len(anomalies)), output_file))
return output_file
if anomalies:
anomalies_data = []
anomalies_timestamps = [int(item[0]) for item in anomalies]
for item in timeseries:
if int(item[0]) in anomalies_timestamps:
anomalies_data.append(1)
else:
anomalies_data.append(0)
rolling_df['anomalies'] = anomalies_data
m66_candidate_metrics[base_name] = {}
m66_candidate_metrics[base_name][custom_algorithm] = {}
m66_candidate_metrics[base_name][custom_algorithm]['anomalies'] = anomalies
# rolling_df['value'].plot(figsize=(18, 6), title=base_name)
title = '%s - median 6 6-sigma persisted' % base_name
# rolling_df['std_median_6_6sigma'].plot(figsize=(18, 6), title=title)
plot(original_rolling_df['value'], anomaly=rolling_df['anomalies'], anomaly_color='red', title=title)
except Exception as e:
print('error: %s' % e)
timer_end = timer()
print('median_6_6sigma analysis of %s metrics took %.6f seconds, significant changes now detected on %s metrics' % (
str(len(current_base_names)), (timer_end - timer_start), str(len(m66_candidate_metrics))))
timer_end_all = timer()
print('%s metrics analysed with m66, took %.6f seconds - %s metrics found with significant changes' % (
str(len(metrics)), (timer_end_all - timer_start_all), str(len(m66_candidate_metrics))))
# Try m66 on the 3 months
from_timestamp = 1618660800 - (86400 * 7)
until_timestamp = from_timestamp + (((86400 * 7) * 4) * 3)
metrics_to_do = list(metrics)
more_analysis_metrics_timeseries = {}
timer_start_all = timer()
def identify_cloudbursts(current_skyline_app, plot_graphs=False, log=False):
"""
Find significant changes (cloudbursts) in metrics.
"""
current_skyline_app_logger = current_skyline_app + 'Log'
current_logger = logging.getLogger(current_skyline_app_logger)
child_process_pid = os.getpid()
function_str = '%s :: functions.luminosity.identify_cloudbursts' % current_skyline_app
if log:
current_logger.info('%s :: running for process_pid - %s for %s' % (
function_str, str(child_process_pid), metric))
start = timer()
full_uniques = '%sunique_metrics' % settings.FULL_NAMESPACE
unique_metrics = list(redis_conn_decoded.smembers(full_uniques))
timer_start_all = timer()
custom_algorithm = 'm66'
m66_algorithm_source = '%%s/custom_algorithms/m66.py' % root_path
custom_algorithms = {}
custom_algorithms[custom_algorithm] = {
'algorithm_source': m66_algorithm_source,
'algorithm_parameters': {
'nth_median': 6, 'sigma': 6, 'window': 5, 'return_anomalies': True,
'save_plots_to': False, 'save_plots_to_absolute_dir': False,
'filename_prefix': False
},
'max_execution_time': 1.0,
'consensus': 1,
'algorithms_allowed_in_consensus': ['m66'],
'run_3sigma_algorithms': False,
'run_before_3sigma': False,
'run_only_if_consensus': False,
'use_with': ['crucible', 'luminosity'],
'debug_logging': False,
}
m66_candidate_metrics = {}
align = True
truncate_last_datapoint = True
window = 4
summarize_intervalString = '15min'
summarize_func = 'median'
nth_median = 6
n_sigma = 6
custom_algorithm = 'median_6_6sigma'
m66_candidate_metrics = {}
found = 0
now_timestamp = int(time())
check_last = 3600
candidate_metrics = {}
for metric in unique_metrics:
metric_name = metric
if metric_name.startswith(settings.FULL_NAMESPACE):
base_name = metric_name.replace(settings.FULL_NAMESPACE, '', 1)
else:
base_name = metric_name
timeseries = []
timeseries = get_metric_timeseries(skyline_app, metric, False)
if not timeseries:
continue
if truncate_last_datapoint:
timeseries_length = len(timeseries)
timeseries = timeseries[1:(timeseries_length - 2)]
for custom_algorithm in list(custom_algorithms.keys()):
custom_algorithms_dict = custom_algorithms[custom_algorithm]
custom_algorithm_dict = {}
custom_algorithm_dict['debug_logging'] = False
debug_algorithm_logging = False
if debug_algorithms:
custom_algorithm_dict['debug_logging'] = True
debug_algorithm_logging = True
algorithm_source = '/opt/skyline/github/skyline/skyline/custom_algorithms/%s.py' % algorithm
custom_algorithm_dict['algorithm_source'] = algorithm_source
if LUMINOSITY_CLASSIFY_ANOMALIES_SAVE_PLOTS:
custom_algorithm_dict['algorithm_parameters'] = {
'window': window, 'c': 6.0, 'return_anomalies': True,
'realtime_analysis': False,
'save_plots_to': metric_training_data_dir,
'save_plots_to_absolute_dir': True,
'filename_prefix': 'luminosity.classify_anomaly',
'debug_logging': debug_algorithm_logging,
}
custom_algorithm_dict['max_execution_time'] = 10.0
else:
custom_algorithm_dict['algorithm_parameters'] = {
'window': window, 'c': 6.0, 'return_anomalies': True,
'realtime_analysis': False,
'debug_logging': debug_algorithm_logging,
}
custom_algorithm_dict['max_execution_time'] = 5.0
if algorithm == base_algorithm:
if current_skyline_app == 'webapp':
anomalous, anomalyScore, anomalies, anomalies_dict = run_custom_algorithm_on_timeseries(current_skyline_app, current_pid, base_name, timeseries, custom_algorithm, custom_algorithm_dict, debug_algorithms)
result, anomalyScore, anomalies = run_custom_algorithm_on_timeseries(skyline_app, current_pid, base_name, timeseries, custom_algorithm, custom_algorithm_dict, debug_algorithms)
if return_anomalies:
return (anomalous, anomalyScore, anomalies)
else:
return (anomalous, anomalyScore)
else:
rolling_df = pd.DataFrame(timeseries, columns=['date', 'value'])
rolling_df['date'] = pd.to_datetime(rolling_df['date'], unit='s')
datetime_index = pd.DatetimeIndex(rolling_df['date'].values)
rolling_df = rolling_df.set_index(datetime_index)
rolling_df.drop('date', axis=1, inplace=True)
original_rolling_df = rolling_df.copy()
# MinMax scale
rolling_df = (rolling_df - rolling_df.min()) / (rolling_df.max() - rolling_df.min())
window = 6
data = rolling_df['value'].tolist()
s = pd.Series(data)
rolling_median_s = s.rolling(window).median()
median = rolling_median_s.tolist()
data = median
s = pd.Series(data)
rolling_median_s = s.rolling(window).median()
median_2 = rolling_median_s.tolist()
data = median_2
s = pd.Series(data)
rolling_median_s = s.rolling(window).median()
median_3 = rolling_median_s.tolist()
data = median_3
s = pd.Series(data)
rolling_median_s = s.rolling(window).median()
median_4 = rolling_median_s.tolist()
data = median_4
s = pd.Series(data)
rolling_median_s = s.rolling(window).median()
median_5 = rolling_median_s.tolist()
data = median_5
s = pd.Series(data)
rolling_median_s = s.rolling(window).median()
median_6 = rolling_median_s.tolist()
data = median_6
s = pd.Series(data)
rolling_std_s = s.rolling(window).std()
rolling_df['std_median_6'] = rolling_std_s.tolist()
std_median_6 = rolling_df['std_median_6'].fillna(0).tolist()
metric_stddev = np.std(std_median_6)
std_median_6_6sigma = []
anomalies = False
for value in std_median_6:
if value > (metric_stddev * 6):
std_median_6_6sigma.append(1)
anomalies = True
else:
std_median_6_6sigma.append(0)
rolling_df['std_median_6_6sigma'] = std_median_6_6sigma
if anomalies:
last_trigger = None
current_triggers = []
anomalies = []
# Only tag anomalous if the 6sigma triggers for window
for index, item in enumerate(timeseries):
if std_median_6_6sigma[index] == 1:
current_triggers.append(index)
else:
if len(current_triggers) > (window / 2):
for trigger_index in current_triggers:
anomalies.append(timeseries[(trigger_index - (window * 3))])
current_triggers = []
if anomalies:
anomalies_data = []
anomalies_timestamps = [int(item[0]) for item in anomalies]
for item in timeseries:
if int(item[0]) in anomalies_timestamps:
anomalies_data.append(1)
else:
anomalies_data.append(0)
rolling_df['anomalies'] = anomalies_data
m66_candidate_metrics[base_name] = {}
m66_candidate_metrics[base_name][custom_algorithm] = {}
m66_candidate_metrics[base_name][custom_algorithm]['anomalies'] = anomalies
# rolling_df['value'].plot(figsize=(18, 6), title=base_name)
title = '%s - median 6 6-sigma persisted' % base_name
# rolling_df['std_median_6_6sigma'].plot(figsize=(18, 6), title=title)
plot(original_rolling_df['value'], anomaly=rolling_df['anomalies'], anomaly_color='red', title=title)
df7 = df1[['Scaled Cl_2']]
#df8=df1[['Scaled Leit']]
c = df1['EVENT']
#from adtk.detector import PcaAD
#pca_ad = PcaAD(k=1)
#anomalies= pca_ad.fit_detect(df2)
#p=plot(df2, anomaly_pred=anomalies, ts_linewidth=2, ts_markersize=3, ap_color='red', ap_alpha=0.3, curve_group='all');
from adtk.detector import GeneralizedESDTestAD
esd_ad = GeneralizedESDTestAD(alpha=0.3)
anomalies = esd_ad.fit_detect(df2)
q = plot(df2,
title='Generalized Extreme studentized Deviate Test on Redox',
anomaly_pred=anomalies,
ts_linewidth=2,
ts_markersize=3,
ap_markersize=5,
ap_color='red',
ap_marker_on_curve=True)
from adtk.detector import GeneralizedESDTestAD
esd_ad = GeneralizedESDTestAD(alpha=0.3)
anomalies1 = esd_ad.fit_detect(df7)
q = plot(df7,
title='Generalized Extreme studentized Deviate Test on Cl_2',
anomaly_pred=anomalies1,
ts_linewidth=3,
ts_markersize=3,
ap_markersize=5,
ap_color='red',
ap_marker_on_curve=True)
def adtk_level_shift(current_skyline_app, parent_pid, timeseries, algorithm_parameters):
"""
A timeseries is anomalous if a level shift occurs in a 5 window period bound
by a factor of 9 of the normal range based on historical interquartile range.
:param current_skyline_app: the Skyline app executing the algorithm. This
will be passed to the algorithm by Skyline. This is **required** for
error handling and logging. You do not have to worry about handling the
argument in the scope of the custom algorithm itself, but the algorithm
must accept it as the first agrument.
:param parent_pid: the parent pid which is executing the algorithm, this is
**required** for error handling and logging. You do not have to worry
about handling this argument in the scope of algorithm, but the
algorithm must accept it as the second argument.
:param timeseries: the time series as a list e.g. ``[[1578916800.0, 29.0],
[1578920400.0, 55.0], ... [1580353200.0, 55.0]]``
:param algorithm_parameters: a dictionary of any required parameters for the
custom_algorithm and algorithm itself. For the matrixprofile custom
algorithm the following parameters are required, example:
``algorithm_parameters={
'c': 9.0,
'run_every': 5,
'side': 'both',
'window': 5
}``
:type current_skyline_app: str
:type parent_pid: int
:type timeseries: list
:type algorithm_parameters: dict
:return: True, False or Non
:rtype: boolean
Performance is of paramount importance in Skyline, especially in terms of
computational complexity, along with execution time and CPU usage. The
adtk LevelShiftAD algortihm is not O(n) and it is not fast either, not when
compared to the normal three-sigma triggered algorithms. However it is
useful if you care about detecting all level shifts. The normal three-sigma
triggered algorithms do not always detect a level shift, especially if the
level shift does not breach the three-sigma limits. Therefore you may find
over time that you encounter alerts that contain level shifts that you
thought should have been detected. On these types of metrics and events,
the adtk LevelShiftAD algortihm can be implemented to detect and alert on
these. It is not recommended to run on all your metrics as it would
immediately triple the analyzer runtime every if only run every 5 windows/
minutes.
Due to the computational complexity and long run time of the adtk
LevelShiftAD algorithm on the size of timeseries data used by Skyline, if
you consider the following timings of all three-sigma triggered algorithms
and compare them to the to the adtk_level_shift results in the last 2 rows
of the below log, it is clear that the running adtk_level_shift on all
metrics is probably not desirable, even if it is possible to do, it is very
noisy.
2021-03-06 10:46:38 :: 1582754 :: algorithm run count - histogram_bins run 567 times
2021-03-06 10:46:38 :: 1582754 :: algorithm timings count - histogram_bins has 567 timings
2021-03-06 10:46:38 :: 1582754 :: algorithm timing - histogram_bins - total: 1.051136 - median: 0.001430
2021-03-06 10:46:38 :: 1582754 :: algorithm run count - first_hour_average run 567 times
2021-03-06 10:46:38 :: 1582754 :: algorithm timings count - first_hour_average has 567 timings
2021-03-06 10:46:38 :: 1582754 :: algorithm timing - first_hour_average - total: 1.322432 - median: 0.001835
2021-03-06 10:46:38 :: 1582754 :: algorithm run count - stddev_from_average run 567 times
2021-03-06 10:46:38 :: 1582754 :: algorithm timings count - stddev_from_average has 567 timings
2021-03-06 10:46:38 :: 1582754 :: algorithm timing - stddev_from_average - total: 1.097290 - median: 0.001641
2021-03-06 10:46:38 :: 1582754 :: algorithm run count - grubbs run 567 times
2021-03-06 10:46:38 :: 1582754 :: algorithm timings count - grubbs has 567 timings
2021-03-06 10:46:38 :: 1582754 :: algorithm timing - grubbs - total: 1.742929 - median: 0.002438
2021-03-06 10:46:38 :: 1582754 :: algorithm run count - ks_test run 147 times
2021-03-06 10:46:38 :: 1582754 :: algorithm timings count - ks_test has 147 timings
2021-03-06 10:46:38 :: 1582754 :: algorithm timing - ks_test - total: 0.127648 - median: 0.000529
2021-03-06 10:46:38 :: 1582754 :: algorithm run count - mean_subtraction_cumulation run 40 times
2021-03-06 10:46:38 :: 1582754 :: algorithm timings count - mean_subtraction_cumulation has 40 timings
2021-03-06 10:46:38 :: 1582754 :: algorithm timing - mean_subtraction_cumulation - total: 0.152515 - median: 0.003152
2021-03-06 10:46:39 :: 1582754 :: algorithm run count - median_absolute_deviation run 35 times
2021-03-06 10:46:39 :: 1582754 :: algorithm timings count - median_absolute_deviation has 35 timings
2021-03-06 10:46:39 :: 1582754 :: algorithm timing - median_absolute_deviation - total: 0.143770 - median: 0.003248
2021-03-06 10:46:39 :: 1582754 :: algorithm run count - stddev_from_moving_average run 30 times
2021-03-06 10:46:39 :: 1582754 :: algorithm timings count - stddev_from_moving_average has 30 timings
2021-03-06 10:46:39 :: 1582754 :: algorithm timing - stddev_from_moving_average - total: 0.125173 - median: 0.003092
2021-03-06 10:46:39 :: 1582754 :: algorithm run count - least_squares run 16 times
2021-03-06 10:46:39 :: 1582754 :: algorithm timings count - least_squares has 16 timings
2021-03-06 10:46:39 :: 1582754 :: algorithm timing - least_squares - total: 0.089108 - median: 0.005538
2021-03-06 10:46:39 :: 1582754 :: algorithm run count - abs_stddev_from_median run 1 times
2021-03-06 10:46:39 :: 1582754 :: algorithm timings count - abs_stddev_from_median has 1 timings
2021-03-06 10:46:39 :: 1582754 :: algorithm timing - abs_stddev_from_median - total: 0.036797 - median: 0.036797
2021-03-06 10:46:39 :: 1582754 :: algorithm run count - adtk_level_shift run 271 times
2021-03-06 10:46:39 :: 1582754 :: algorithm timings count - adtk_level_shift has 271 timings
2021-03-06 10:46:39 :: 1582754 :: algorithm timing - adtk_level_shift - total: 13.729565 - median: 0.035791
...
...
2021-03-06 10:46:39 :: 1582754 :: seconds to run :: 27.93 # THE TOTAL ANALYZER RUNTIME
Therefore the analysis methodology implemented for the adtk_level_shift
custom_algorithm is as folows:
- When new metrics are added either to the configuration or by actual new
metrics coming online that match the ``algorithm_parameters['namespace']``,
Skyline implements sharding on new metrics into time slots to prevent a
thundering herd situation from developing. A newly added metrics will
eventually be assigned into a time shard and be added and the last analysed
timestamp will be added to the ``analyzer.last.adtk_level_shift`` Redis hash
key to determine the next scheduled run with
``algorithm_parameters['namespace']``
- A ``run_every`` parameter is implemented so that the algorithm can be
configured to run on a metric once every ``run_every`` minutes. The default
is to run it every 5 minutes using window 5 (rolling) and trigger as
anomalous if the algorithm labels any of the last 5 datapoints as anomalous.
This means that there could be up to a 5 minute delay on an alert on the
60 second, 168 SECOND_ORDER_RESOLUTION_HOURS metrics in the example, but a
``c=9.0`` level shift would be detected and would be alerted on (if both
analyzer and mirage triggered on it). This periodic running of the
algorithm is a tradeoff so that the adtk_level_shift load and runtime can be
spread over ``run_every`` minutes.
- The algorithm is not run against metrics that are sparsely populated.
When the algorithm is run on sparsely populated metrics it results in lots
of false positives and noise.
The Skyline CUSTOM_ALGORITHMS implementation of the adtk LevelShiftAD
algorithm is configured as the example shown below. However please note
that the algorithm_parameters shown in this example configuration are
suitiable for metrics that have a 60 second relation and have a
:mod:`settings.ALERTS` Mirage SECOND_ORDER_RESOLUTION_HOURS of 168 (7 days).
For metrics with a different resolution/frequency may require different
values appropriate for metric resolution.
:
Example CUSTOM_ALGORITHMS configuration:
'adtk_level_shift': {
'namespaces': [
'skyline.analyzer.run_time', 'skyline.analyzer.total_metrics',
'skyline.analyzer.exceptions'
],
'algorithm_source': '/opt/skyline/github/skyline/skyline/custom_algorithms/adtk_level_shift.py',
'algorithm_parameters': {'c': 9.0, 'run_every': 5, 'side': 'both', 'window': 5},
'max_execution_time': 0.5,
'consensus': 1,
'algorithms_allowed_in_consensus': ['adtk_level_shift'],
'run_3sigma_algorithms': True,
'run_before_3sigma': True,
'run_only_if_consensus': False,
'use_with': ["analyzer", "mirage"],
'debug_logging': False,
},
"""
# You MUST define the algorithm_name
algorithm_name = 'adtk_level_shift'
# Define the default state of None and None, anomalous does not default to
# False as that is not correct, False is only correct if the algorithm
# determines the data point is not anomalous. The same is true for the
# anomalyScore.
anomalous = None
anomalyScore = None
# @aded 20210308 - Feature #3978: luminosity - classify_metrics
# Feature #3642: Anomaly type classification
return_anomalies = False
anomalies = []
realtime_analysis = True
current_logger = None
# If you wanted to log, you can but this should only be done during
# testing and development
def get_log(current_skyline_app):
current_skyline_app_logger = current_skyline_app + 'Log'
current_logger = logging.getLogger(current_skyline_app_logger)
return current_logger
start = timer()
# Use the algorithm_parameters to determine the sample_period
debug_logging = None
try:
debug_logging = algorithm_parameters['debug_logging']
except:
debug_logging = False
if debug_logging:
try:
current_logger = get_log(current_skyline_app)
current_logger.debug('debug :: %s :: debug_logging enabled with algorithm_parameters - %s' % (
algorithm_name, str(algorithm_parameters)))
except:
# This except pattern MUST be used in ALL custom algortihms to
# facilitate the traceback from any errors. The algorithm we want to
# run super fast and without spamming the log with lots of errors.
# But we do not want the function returning and not reporting
# anything to the log, so the pythonic except is used to "sample" any
# algorithm errors to a tmp file and report once per run rather than
# spewing tons of errors into the log e.g. analyzer.log
record_algorithm_error(current_skyline_app, parent_pid, algorithm_name, traceback.format_exc())
# Return None and None as the algorithm could not determine True or False
return (False, None)
# Allow the LevelShiftAD window parameter to be passed in the
# algorithm_parameters
window = 5
try:
window = algorithm_parameters['window']
except:
pass
# Allow the LevelShiftAD c parameter to be passed in the
# algorithm_parameters
c = 9.0
try:
c = algorithm_parameters['c']
except:
pass
run_every = window
try:
run_every = algorithm_parameters['run_every']
except:
pass
side = 'both'
try:
side = algorithm_parameters['side']
except:
pass
if debug_logging:
current_logger.debug('debug :: algorithm_parameters :: %s' % (
str(algorithm_parameters)))
# @added 20210308 - Feature #3978: luminosity - classify_metrics
# Feature #3642: Anomaly type classification
try:
return_anomalies = algorithm_parameters['return_anomalies']
except:
return_anomalies = False
try:
realtime_analysis = algorithm_parameters['realtime_analysis']
except:
realtime_analysis = True
# @added 20210316 - Feature #3978: luminosity - classify_metrics
# Feature #3642: Anomaly type classification
save_plots_to = False
try:
save_plots_to = algorithm_parameters['save_plots_to']
except:
pass
# @added 20210323 - Feature #3978: luminosity - classify_metrics
# Feature #3642: Anomaly type classification
save_plots_to_absolute_dir = False
try:
save_plots_to_absolute_dir = algorithm_parameters['save_plots_to_absolute_dir']
except:
pass
filename_prefix = False
try:
filename_prefix = algorithm_parameters['filename_prefix']
except:
pass
# @added 20210318 - Feature #3978: luminosity - classify_metrics
# Feature #3642: Anomaly type classification
run_PersistAD = False
try:
run_PersistAD = algorithm_parameters['run_PersistAD']
except:
pass
if debug_logging:
current_logger.debug('debug :: algorithm_parameters :: %s' % (
str(algorithm_parameters)))
try:
base_name = algorithm_parameters['base_name']
except:
# This except pattern MUST be used in ALL custom algortihms to
# facilitate the traceback from any errors. The algorithm we want to
# run super fast and without spamming the log with lots of errors.
# But we do not want the function returning and not reporting
# anything to the log, so the pythonic except is used to "sample" any
# algorithm errors to a tmp file and report once per run rather than
# spewing tons of errors into the log e.g. analyzer.log
record_algorithm_error(current_skyline_app, parent_pid, algorithm_name, traceback.format_exc())
# Return None and None as the algorithm could not determine True or False
if return_anomalies:
return (False, None, anomalies)
else:
return (False, None)
if debug_logging:
current_logger.debug('debug :: %s :: base_name - %s' % (
algorithm_name, str(base_name)))
# Due to the load and runtime of LevelShiftAD it is only run in analyzer
# periodically
if current_skyline_app == 'analyzer':
redis_conn_decoded = get_redis_conn_decoded(current_skyline_app)
last_hash_key = 'analyzer.last.%s' % algorithm_name
last_check = None
try:
raw_last_check = redis_conn_decoded.hget(last_hash_key, base_name)
last_check = int(raw_last_check)
except:
last_check = None
last_window_timestamps = [int(item[0]) for item in timeseries[-run_every:]]
if last_check in last_window_timestamps:
if debug_logging:
current_logger.debug('debug :: %s :: run_every period is not over yet, skipping base_name - %s' % (
algorithm_name, str(base_name)))
if return_anomalies:
return (False, None, anomalies)
else:
return (False, None)
# If there is no last timestamp, shard the metric, it will eventually
# be added.
if not last_check:
now = datetime.datetime.now()
now_seconds = int(now.second)
if now_seconds == 0:
now_seconds = 1
period_seconds = int(60 / run_every)
shard = int(period_seconds)
last_shard = 60
shard = int(period_seconds)
shards = [shard]
while shard < last_shard:
shard = shard + period_seconds
shards.append((shard))
shard_value = round(now_seconds / shards[0]) * shards[0]
if shard_value <= shards[0]:
shard_value = shards[0]
metric_as_bytes = str(base_name).encode()
value = zlib.adler32(metric_as_bytes)
shard_index = [(index + 1) for index, s_value in enumerate(shards) if s_value == shard_value][0]
modulo_result = value % shard_index
if modulo_result == 0:
if debug_logging:
current_logger.debug('debug :: %s :: skipping as not sharded into this run - %s' % (
algorithm_name, str(base_name)))
if return_anomalies:
return (False, None, anomalies)
else:
return (False, None)
if debug_logging:
current_logger.debug('debug :: %s :: analysing %s' % (
algorithm_name, str(base_name)))
try:
int_metric_timestamp = int(timeseries[-1][0])
except:
int_metric_timestamp = 0
if int_metric_timestamp:
try:
redis_conn_decoded.hset(
last_hash_key, base_name,
int_metric_timestamp)
except:
pass
# ALWAYS WRAP YOUR ALGORITHM IN try and the BELOW except
try:
start_preprocessing = timer()
# INFO: Sorting time series of 10079 data points took 0.002215 seconds
timeseries = sorted(timeseries, key=lambda x: x[0])
if debug_logging:
current_logger.debug('debug :: %s :: time series of length - %s' % (
algorithm_name, str(len(timeseries))))
# Testing the data to ensure it meets minimum requirements, in the case
# of Skyline's use of the LevelShiftAD algorithm this means that:
# - the time series must have at least 75% of its full_duration
# - the time series must have at least 99% of the data points for the
# in the sample being analysed.
do_not_use_sparse_data = False
if current_skyline_app == 'analyzer':
do_not_use_sparse_data = True
# @added 20210305 - Feature #3970: custom_algorithm - adtk_level_shift
# Task #3664:: POC with adtk
# With mirage also do not run LevelShiftAD on sparsely populated data
if current_skyline_app == 'mirage':
do_not_use_sparse_data = True
# @aded 20210309 - Feature #3978: luminosity - classify_metrics
# Feature #3642: Anomaly type classification
if current_skyline_app == 'luminosity':
do_not_use_sparse_data = True
if do_not_use_sparse_data:
total_period = 0
total_datapoints = 0
try:
start_timestamp = int(timeseries[0][0])
end_timestamp = int(timeseries[-1][0])
total_period = end_timestamp - start_timestamp
total_datapoints = len(timeseries)
except SystemExit as e:
if debug_logging:
current_logger.debug('debug_logging :: %s :: SystemExit called, exiting - %s' % (
algorithm_name, e))
if return_anomalies:
return (anomalous, anomalyScore, anomalies)
else:
return (anomalous, anomalyScore)
except:
traceback_msg = traceback.format_exc()
record_algorithm_error(current_skyline_app, parent_pid, algorithm_name, traceback_msg)
if debug_logging:
current_logger.error(traceback_msg)
current_logger.error('error :: debug_logging :: %s :: failed to determine total_period and total_datapoints' % (
algorithm_name))
timeseries = []
if not timeseries:
if return_anomalies:
return (anomalous, anomalyScore, anomalies)
else:
return (anomalous, anomalyScore)
if current_skyline_app == 'analyzer':
# Default for analyzer at required period to 18 hours
period_required = int(FULL_DURATION * 0.75)
else:
# Determine from timeseries
if total_period < FULL_DURATION:
period_required = int(FULL_DURATION * 0.75)
else:
period_required = int(total_period * 0.75)
# If the time series does not have 75% of its full_duration it does not
# have sufficient data to sample
try:
if total_period < period_required:
if debug_logging:
current_logger.debug('debug :: %s :: time series does not have sufficient data' % (
algorithm_name))
if return_anomalies:
return (anomalous, anomalyScore, anomalies)
else:
return (anomalous, anomalyScore)
except SystemExit as e:
if debug_logging:
current_logger.debug('debug_logging :: %s :: SystemExit called, exiting - %s' % (
algorithm_name, e))
if return_anomalies:
return (anomalous, anomalyScore, anomalies)
else:
return (anomalous, anomalyScore)
except:
traceback_msg = traceback.format_exc()
record_algorithm_error(current_skyline_app, parent_pid, algorithm_name, traceback_msg)
if debug_logging:
current_logger.error(traceback_msg)
current_logger.error('error :: debug_logging :: %s :: falied to determine if time series has sufficient data' % (
algorithm_name))
if return_anomalies:
return (anomalous, anomalyScore, anomalies)
else:
return (anomalous, anomalyScore)
# If the time series does not have 75% of its full_duration data points
# it does not have sufficient data to sample
# Determine resolution from the last 30 data points
# INFO took 0.002060 seconds
resolution_timestamps = []
metric_resolution = False
for metric_datapoint in timeseries[-30:]:
timestamp = int(metric_datapoint[0])
resolution_timestamps.append(timestamp)
timestamp_resolutions = []
if resolution_timestamps:
last_timestamp = None
for timestamp in resolution_timestamps:
if last_timestamp:
resolution = timestamp - last_timestamp
timestamp_resolutions.append(resolution)
last_timestamp = timestamp
else:
last_timestamp = timestamp
try:
del resolution_timestamps
except:
pass
if timestamp_resolutions:
try:
timestamp_resolutions_count = Counter(timestamp_resolutions)
ordered_timestamp_resolutions_count = timestamp_resolutions_count.most_common()
metric_resolution = int(ordered_timestamp_resolutions_count[0][0])
except SystemExit as e:
if debug_logging:
current_logger.debug('debug_logging :: %s :: SystemExit called, exiting - %s' % (
algorithm_name, e))
if return_anomalies:
return (anomalous, anomalyScore, anomalies)
else:
return (anomalous, anomalyScore)
except:
traceback_msg = traceback.format_exc()
record_algorithm_error(current_skyline_app, parent_pid, algorithm_name, traceback_msg)
if debug_logging:
current_logger.error(traceback_msg)
current_logger.error('error :: debug_logging :: %s :: failed to determine if time series has sufficient data' % (
algorithm_name))
try:
del timestamp_resolutions
except:
pass
minimum_datapoints = None
if metric_resolution:
minimum_datapoints = int(period_required / metric_resolution)
if minimum_datapoints:
if total_datapoints < minimum_datapoints:
if debug_logging:
current_logger.debug('debug :: %s :: time series does not have sufficient data, minimum_datapoints required is %s and time series has %s' % (
algorithm_name, str(minimum_datapoints),
str(total_datapoints)))
if return_anomalies:
return (anomalous, anomalyScore, anomalies)
else:
return (anomalous, anomalyScore)
# Is the time series fully populated?
# full_duration_datapoints = int(full_duration / metric_resolution)
total_period_datapoints = int(total_period / metric_resolution)
# minimum_percentage_sparsity = 95
minimum_percentage_sparsity = 90
sparsity = int(total_datapoints / (total_period_datapoints / 100))
if sparsity < minimum_percentage_sparsity:
if debug_logging:
current_logger.debug('debug :: %s :: time series does not have sufficient data, minimum_percentage_sparsity required is %s and time series has %s' % (
algorithm_name, str(minimum_percentage_sparsity),
str(sparsity)))
if return_anomalies:
return (anomalous, anomalyScore, anomalies)
else:
return (anomalous, anomalyScore)
if len(set(item[1] for item in timeseries)) == 1:
if debug_logging:
current_logger.debug('debug :: %s :: time series does not have sufficient variability, all the values are the same' % algorithm_name)
anomalous = False
anomalyScore = 0.0
if return_anomalies:
return (anomalous, anomalyScore, anomalies)
else:
return (anomalous, anomalyScore)
end_preprocessing = timer()
preprocessing_runtime = end_preprocessing - start_preprocessing
if debug_logging:
current_logger.debug('debug :: %s :: preprocessing took %.6f seconds' % (
algorithm_name, preprocessing_runtime))
if not timeseries:
if debug_logging:
current_logger.debug('debug :: %s :: LevelShiftAD not run as no data' % (
algorithm_name))
anomalies = []
if return_anomalies:
return (anomalous, anomalyScore, anomalies)
else:
return (anomalous, anomalyScore)
else:
if debug_logging:
current_logger.debug('debug :: %s :: timeseries length: %s' % (
algorithm_name, str(len(timeseries))))
if len(timeseries) < 100:
if debug_logging:
current_logger.debug('debug :: %s :: time series does not have sufficient data' % (
algorithm_name))
if return_anomalies:
return (anomalous, anomalyScore, anomalies)
else:
return (anomalous, anomalyScore)
start_analysis = timer()
try:
df = pd.DataFrame(timeseries, columns=['date', 'value'])
df['date'] = pd.to_datetime(df['date'], unit='s')
datetime_index = pd.DatetimeIndex(df['date'].values)
df = df.set_index(datetime_index)
df.drop('date', axis=1, inplace=True)
s = validate_series(df)
level_shift_ad = LevelShiftAD(c=c, side=side, window=window)
anomaly_df = level_shift_ad.fit_detect(s)
anomalies = anomaly_df.loc[anomaly_df['value'] > 0]
anomalous = False
if len(anomalies) > 0:
anomaly_timestamps = list(anomalies.index.astype(np.int64) // 10**9)
if realtime_analysis:
last_window_timestamps = [int(item[0]) for item in timeseries[-window:]]
# if timeseries[-1][0] in anomaly_timestamps:
for timestamp in last_window_timestamps:
if timestamp in anomaly_timestamps:
anomalous = True
break
else:
anomalous = True
# Convert anomalies dataframe to anomalies_list
anomalies_list = []
# @added 20210316 - Feature #3978: luminosity - classify_metrics
# Feature #3642: Anomaly type classification
# Convert anomalies dataframe to anomalies_dict
anomalies_dict = {}
anomalies_dict['metric'] = base_name
anomalies_dict['timestamp'] = int(timeseries[-1][0])
anomalies_dict['from_timestamp'] = int(timeseries[0][0])
anomalies_dict['algorithm'] = algorithm_name
anomalies_dict['anomalies'] = {}
for ts, value in timeseries:
if int(ts) in anomaly_timestamps:
anomalies_list.append([int(ts), value])
anomalies_dict['anomalies'][int(ts)] = value
anomalies = list(anomalies_list)
# @added 20210316 - Feature #3978: luminosity - classify_metrics
# Feature #3642: Anomaly type classification
if save_plots_to:
try:
from adtk.visualization import plot
metric_dir = base_name.replace('.', '/')
timestamp_dir = str(int(timeseries[-1][0]))
save_path = '%s/%s/%s/%s' % (
save_plots_to, algorithm_name, metric_dir,
timestamp_dir)
if save_plots_to_absolute_dir:
save_path = '%s' % save_plots_to
anomalies_dict['file_path'] = save_path
save_to_file = '%s/%s.%s.png' % (
save_path, algorithm_name, base_name)
if filename_prefix:
save_to_file = '%s/%s.%s.%s.png' % (
save_path, filename_prefix, algorithm_name,
base_name)
save_to_path = os_path_dirname(save_to_file)
title = '%s\n%s' % (algorithm_name, base_name)
if not os_path_exists(save_to_path):
try:
mkdir_p(save_to_path)
except Exception as e:
current_logger.error('error :: %s :: failed to create dir - %s - %s' % (
algorithm_name, save_to_path, e))
if os_path_exists(save_to_path):
try:
plot(s, anomaly=anomaly_df, anomaly_color='red', title=title, save_to_file=save_to_file)
if debug_logging:
current_logger.debug('debug :: %s :: plot saved to - %s' % (
algorithm_name, save_to_file))
except Exception as e:
current_logger.error('error :: %s :: failed to plot - %s - %s' % (
algorithm_name, base_name, e))
anomalies_file = '%s/%s.%s.anomalies_list.txt' % (
save_path, algorithm_name, base_name)
with open(anomalies_file, 'w') as fh:
fh.write(str(anomalies_list))
# os.chmod(anomalies_file, mode=0o644)
data_file = '%s/data.txt' % (save_path)
with open(data_file, 'w') as fh:
fh.write(str(anomalies_dict))
except SystemExit as e:
if debug_logging:
current_logger.debug('debug_logging :: %s :: SystemExit called during save plot, exiting - %s' % (
algorithm_name, e))
if return_anomalies:
return (anomalous, anomalyScore, anomalies)
else:
return (anomalous, anomalyScore)
except Exception as e:
traceback_msg = traceback.format_exc()
record_algorithm_error(current_skyline_app, parent_pid, algorithm_name, traceback_msg)
if debug_logging:
current_logger.error(traceback_msg)
current_logger.error('error :: %s :: failed to plot or save anomalies file - %s - %s' % (
algorithm_name, base_name, e))
else:
anomalies = []
# @added 20210318 - Feature #3978: luminosity - classify_metrics
# Feature #3642: Anomaly type classification
if anomalies and run_PersistAD and not realtime_analysis:
persist_ad_algorithm_parameters = {}
try:
persist_ad_algorithm_parameters = algorithm_parameters['persist_ad_algorithm_parameters']
except:
pass
persist_ad_window = 20
try:
persist_ad_window = persist_ad_algorithm_parameters['window']
except:
pass
persist_ad_c = 9.9
try:
persist_ad_c = persist_ad_algorithm_parameters['c']
except:
pass
try:
from adtk.detector import PersistAD
persist_ad = PersistAD(c=persist_ad_c, side='both', window=persist_ad_window)
persist_ad_anomaly_df = persist_ad.fit_detect(s)
persist_ad_anomalies = persist_ad_anomaly_df.loc[persist_ad_anomaly_df['value'] > 0]
if len(persist_ad_anomalies) > 0:
current_logger.info('%s :: %s anomalies found with PersistAD on %s' % (
algorithm_name, str(len(persist_ad_anomalies)),
base_name))
persist_ad_anomaly_timestamps = list(persist_ad_anomalies.index.astype(np.int64) // 10**9)
# Convert persist_ad_anomalies dataframe to persist_ad_anomalies_list
persist_ad_anomalies_list = []
persist_ad_anomalies_dict = {}
persist_ad_anomalies_dict['metric'] = base_name
persist_ad_anomalies_dict['timestamp'] = int(timeseries[-1][0])
persist_ad_anomalies_dict['from_timestamp'] = int(timeseries[0][0])
persist_ad_anomalies_dict['algorithm'] = 'adtk_PersistAD'
persist_ad_anomalies_dict['anomalies'] = {}
for ts, value in timeseries:
if int(ts) in persist_ad_anomaly_timestamps:
persist_ad_anomalies_list.append([int(ts), value])
persist_ad_anomalies_dict['anomalies'][int(ts)] = value
persist_ad_anomalies = list(persist_ad_anomalies_list)
if save_plots_to:
try:
from adtk.visualization import plot
metric_dir = base_name.replace('.', '/')
timestamp_dir = str(int(timeseries[-1][0]))
save_path = '%s/%s/%s/%s' % (
save_plots_to, algorithm_name, metric_dir,
timestamp_dir)
if save_plots_to_absolute_dir:
save_path = '%s' % save_plots_to
persist_ad_anomalies_dict['file_path'] = save_path
save_to_file = '%s/%s.PersistAD.%s.png' % (
save_path, algorithm_name, base_name)
if filename_prefix:
save_to_file = '%s/%s.%s.%s.png' % (
save_path, filename_prefix, algorithm_name,
base_name)
save_to_path = os_path_dirname(save_to_file)
title = '%s - PersistAD verification\n%s' % (algorithm_name, base_name)
if not os_path_exists(save_to_path):
try:
mkdir_p(save_to_path)
except Exception as e:
current_logger.error('error :: %s :: failed to create dir - %s - %s' % (
algorithm_name, save_to_path, e))
if os_path_exists(save_to_path):
try:
plot(s, anomaly=persist_ad_anomaly_df, anomaly_color='red', title=title, save_to_file=save_to_file)
if debug_logging:
current_logger.debug('debug :: %s :: plot saved to - %s' % (
algorithm_name, save_to_file))
except Exception as e:
current_logger.error('error :: %s :: failed to plot - %s - %s' % (
algorithm_name, base_name, e))
anomalies_file = '%s/%s.%s.PersistAD.anomalies_list.txt' % (
save_path, algorithm_name, base_name)
with open(anomalies_file, 'w') as fh:
fh.write(str(persist_ad_anomalies))
# os.chmod(anomalies_file, mode=0o644)
data_file = '%s/PersistAD.data.txt' % (save_path)
with open(data_file, 'w') as fh:
fh.write(str(persist_ad_anomalies_dict))
except Exception as e:
traceback_msg = traceback.format_exc()
record_algorithm_error(current_skyline_app, parent_pid, algorithm_name, traceback_msg)
if debug_logging:
current_logger.error(traceback_msg)
current_logger.error('error :: %s :: failed to plot or save PersistAD anomalies file - %s - %s' % (
algorithm_name, base_name, e))
except Exception as e:
traceback_msg = traceback.format_exc()
record_algorithm_error(current_skyline_app, parent_pid, algorithm_name, traceback_msg)
if debug_logging:
current_logger.error(traceback_msg)
current_logger.error('error :: %s :: failed to analysis with PersistAD anomalies file - %s - %s' % (
algorithm_name, base_name, e))
try:
del df
except:
pass
except SystemExit as e:
if debug_logging:
current_logger.debug('debug_logging :: %s :: SystemExit called, during analysis, exiting - %s' % (
algorithm_name, e))
if return_anomalies:
return (anomalous, anomalyScore, anomalies)
else:
return (anomalous, anomalyScore)
except:
traceback_msg = traceback.format_exc()
record_algorithm_error(current_skyline_app, parent_pid, algorithm_name, traceback_msg)
if debug_logging:
current_logger.error(traceback_msg)
current_logger.error('error :: debug_logging :: %s :: failed to run on ts' % (
algorithm_name))
if return_anomalies:
return (anomalous, anomalyScore, anomalies)
else:
return (anomalous, anomalyScore)
end_analysis = timer()
analysis_runtime = end_analysis - start_analysis
if debug_logging:
current_logger.debug('debug :: %s :: LevelShiftAD took %.6f seconds' % (
algorithm_name, analysis_runtime))
if anomalous:
anomalyScore = 1.0
else:
anomalyScore = 0.0
if debug_logging:
current_logger.info('%s :: anomalous - %s, anomalyScore - %s' % (
algorithm_name, str(anomalous), str(anomalyScore)))
if debug_logging:
end = timer()
processing_runtime = end - start
current_logger.info('%s :: completed analysis in %.6f seconds' % (
algorithm_name, processing_runtime))
try:
del timeseries
except:
pass
if return_anomalies:
return (anomalous, anomalyScore, anomalies)
else:
return (anomalous, anomalyScore)
except SystemExit as e:
if debug_logging:
current_logger.debug('debug_logging :: %s :: SystemExit called (before StopIteration), exiting - %s' % (
algorithm_name, e))
if return_anomalies:
return (anomalous, anomalyScore, anomalies)
else:
return (anomalous, anomalyScore)
except StopIteration:
# This except pattern MUST be used in ALL custom algortihms to
# facilitate the traceback from any errors. The algorithm we want to
# run super fast and without spamming the log with lots of errors.
# But we do not want the function returning and not reporting
# anything to the log, so the pythonic except is used to "sample" any
# algorithm errors to a tmp file and report once per run rather than
# spewing tons of errors into the log e.g. analyzer.log
if return_anomalies:
return (False, None, anomalies)
else:
return (False, None)
except:
record_algorithm_error(current_skyline_app, parent_pid, algorithm_name, traceback.format_exc())
# Return None and None as the algorithm could not determine True or False
if return_anomalies:
return (False, None, anomalies)
else:
return (False, None)
if return_anomalies:
return (anomalous, anomalyScore, anomalies)
else:
return (anomalous, anomalyScore)
import json
import pandas as pd
from adtk.data import validate_series
from adtk.visualization import plot
from adtk.detector import SeasonalAD
with open('data.txt') as json_file:
data = json.load(json_file)
print(data)
s_train = pd.read_csv("./training.csv", index_col="Datetime", parse_dates=True, squeeze=True)
s_train = validate_series(s_train)
# print(s_train)
plot(s_train)
seasonal_ad = SeasonalAD()
anomalies = seasonal_ad.fit_detect(s_train)
print(anomalies)
plot(s_train, anomaly_pred=anomalies, ap_color='red', ap_marker_on_curve=True)
# from firebase import Firebase
#
# config = {
# "apiKey" : "AIzaSyDHWPY4NelJCF-UkuLjcH2WX4njgU5TDVI",
# "authDomain" : "fireguard-88888.firebaseapp.com",
# "databaseURL" : "https://fireguard-88888.firebaseio.com",
# "projectId" : "fireguard-88888",
# "storageBucket": "fireguard-88888.appspot.com",
# "messagingSenderId": "434458514176",
# "appId": "1:434458514176:web:60d16d55a6f382e7e899e5"
# }
#
# NOT NEEDED AS ADTK HANDLES DATETIME INDEXING
# # data vis
# chart_data = data[['date', 'sessions']]
# chart_data.head
# # Convert df date colum to pd.Datetime and swap out date for datetime index in df2
# datetime_series = pd.to_datetime(chart_data['date'])
# datetime_index = pd.DatetimeIndex(datetime_series.values)
# df2=data.set_index(datetime_index)
# df2.drop('date',axis=1,inplace=True)
# # validate and data vis
# chart_data = df2[['sessions']]
# print(chart_data)
data = pd.read_csv(csv_data, index_col=DATE_COL, parse_dates=True)
s = data['sessions']
s = validate_series(s)
# Threshhold analysis
threshold_ad = ThresholdAD(high=100000, low=60000)
anomalies = threshold_ad.detect(s)
# Visualise threshold AD
plot(s,
anomaly=anomalies,
ts_linewidth=1,
ts_markersize=3,
anomaly_markersize=5,
anomaly_color='red',
anomaly_tag="marker")
#!/usr/bin/env python3
import pandas as pd
from adtk.data import validate_series
from adtk.visualization import plot
from adtk.detector import LevelShiftAD
s_train = pd.read_csv("./Kohl_BB_Data.csv", index_col="time", parse_dates=True, squeeze=True)
s_train = validate_series(s_train)
#print(s_train)
plot(s_train)
level_shift_ad = LevelShiftAD(c=6.0, side='both', window=1) # This is almost matching to TSOutlier
anomalies_1 = level_shift_ad.fit_detect(s_train)
s_test_output = pd.concat([s_train,anomalies_1],axis=1)
print(s_test_output)
plot(s_train, anomaly=anomalies_1, anomaly_color='red');
