def test_algorithm_params(self):
"""
Test if optional parameter algorithm_params works as expected.
"""
self.assertRaises(exceptions.InvalidDataFormat, lambda: Correlator(self.s1, self.s2, algorithm_name='cross_correlator', algorithm_params=1))
correlator = Correlator(self.s1, self.s2, algorithm_name='cross_correlator', algorithm_params={'max_shift_seconds': 180})
self.assertEqual(self.correlator2.get_correlation_result().coefficient, correlator.get_correlation_result().coefficient)
def test_algorithm_params(self):
"""
Test if optional parameter algorithm_params works as expected.
"""
self.assertRaises(exceptions.InvalidDataFormat, lambda: Correlator(self.s1, self.s2, algorithm_name='cross_correlator', algorithm_params=1))
correlator = Correlator(self.s1, self.s2, algorithm_name='cross_correlator', algorithm_params={'max_shift_seconds': 180})
self.assertEqual(self.correlator2.get_correlation_result().coefficient, correlator.get_correlation_result().coefficient)
def test_algorithm(self): """ Test if optional parameter algorithm works as expected. """ self.assertRaises(exceptions.AlgorithmNotFound, lambda: Correlator(self.s1, self.s2, algorithm_name='NotValidAlgorithm')) correlator = Correlator(self.s1, self.s2, algorithm_name='cross_correlator') self.assertEqual(self.correlator2.get_correlation_result().coefficient, correlator.get_correlation_result().coefficient) self.assertEqual(self.correlator2.get_correlation_result().shift, correlator.get_correlation_result().shift)
def test_algorithm(self):
"""
Test if optional parameter algorithm works as expected.
"""
self.assertRaises(exceptions.AlgorithmNotFound, lambda: Correlator(self.s1, self.s2, algorithm_name='NotValidAlgorithm'))
correlator = Correlator(self.s1, self.s2, algorithm_name='cross_correlator')
self.assertEqual(self.correlator2.get_correlation_result().coefficient, correlator.get_correlation_result().coefficient)
self.assertEqual(self.correlator2.get_correlation_result().shift, correlator.get_correlation_result().shift)
def test_use_anomaly_score(self):
"""
Test if use_anomaly_score works as expected.
"""
correlator1 = Correlator(self.s1, self.s2, use_anomaly_score=True)
self.assertNotEqual(
self.correlator1.get_correlation_result().coefficient,
correlator1.get_correlation_result().coefficient)
class TestCorrelator(unittest.TestCase):
def setUp(self):
self.s1 = {0: 0, 1: 0, 2: 0, 3: 0, 4: 0.5, 5: 1, 6: 1, 7: 1, 8: 0}
self.s2 = {0: 0, 1: 0.5, 2: 1, 3: 1, 4: 1, 5: 0, 6: 0, 7: 0, 8: 0}
self.s3 = {0: 0, 1: 0.5, 2: 1, 3: 1, 4: 1, 5: 0}
self.correlator1 = Correlator(self.s1, self.s2)
self.correlator2 = Correlator(self.s1, self.s3)
def test_cross_correlation(self):
"""
Test if CrossCorrelation algorithm gives same results as expected.
"""
self.assertEqual(self.correlator1.get_correlation_result().coefficient, self.correlator2.get_correlation_result().coefficient)
self.assertEqual(self.correlator1.get_correlation_result().shift, self.correlator2.get_correlation_result().shift)
def test_if_correlate(self):
"""
Test if function is_correlated gives same result as function get_correlation_result
when there is a correlation.
"""
self.assertEqual(True, self.correlator2.is_correlated() is not None)
self.assertEqual(self.correlator2.get_correlation_result(), self.correlator2.is_correlated())
def test_algorithm(self):
"""
Test if optional parameter algorithm works as expected.
"""
self.assertRaises(exceptions.AlgorithmNotFound, lambda: Correlator(self.s1, self.s2, algorithm_name='NotValidAlgorithm'))
correlator = Correlator(self.s1, self.s2, algorithm_name='cross_correlator')
self.assertEqual(self.correlator2.get_correlation_result().coefficient, correlator.get_correlation_result().coefficient)
self.assertEqual(self.correlator2.get_correlation_result().shift, correlator.get_correlation_result().shift)
def test_algorithm_params(self):
"""
Test if optional parameter algorithm_params works as expected.
"""
self.assertRaises(exceptions.InvalidDataFormat, lambda: Correlator(self.s1, self.s2, algorithm_name='cross_correlator', algorithm_params=1))
correlator = Correlator(self.s1, self.s2, algorithm_name='cross_correlator', algorithm_params={'max_shift_seconds': 180})
self.assertEqual(self.correlator2.get_correlation_result().coefficient, correlator.get_correlation_result().coefficient)
def test_maximal_shift_seconds(self):
"""
Test if parameter max_shift_seconds works as expected.
"""
correlator = Correlator(self.s1, self.s2, algorithm_name='cross_correlator', algorithm_params={'max_shift_seconds': 0})
self.assertNotEqual(self.correlator2.get_correlation_result().coefficient, correlator.get_correlation_result().coefficient)
def test_sanity_check(self):
"""
Test if exception NotEnoughDataPoints is raised as expected.
"""
s4 = {0: 0}
self.assertRaises(exceptions.NotEnoughDataPoints, lambda: Correlator(s4, self.s1))
def test_time_series_format(self):
"""
Test if exception InvalidDataFormat is raised as expected.
"""
self.assertRaises(exceptions.InvalidDataFormat, lambda: Correlator(list(), 1))
def test_maximal_shift_seconds(self):
"""
Test if parameter max_shift_seconds works as expected.
"""
correlator = Correlator(self.s1,
self.s2,
algorithm_name='cross_correlator',
algorithm_params={'max_shift_seconds': 0})
self.assertNotEqual(
self.correlator2.get_correlation_result().coefficient,
correlator.get_correlation_result().coefficient)
def test_sanity_check(self):
"""
Test if exception NotEnoughDataPoints is raised as expected.
"""
s4 = {0: 0}
self.assertRaises(exceptions.NotEnoughDataPoints,
lambda: Correlator(s4, self.s1))
def pointsOfCorrelation(ts1, ts2, thresholdVal):
corrPoints = []
# Conduct AD on each of each of the time series.
# algorithm_params={'absolute_threshold_value_lower':lower,'absolute_threshold_value_upper':upper}
# detector = AnomalyDetector(ts2, score_threshold=thresholdVal, algorithm_name="derivative_detector")
detector = AnomalyDetector(ts2,
score_threshold=thresholdVal,
algorithm_name="exp_avg_detector")
# score = detector.get_all_scores()
anomalies = detector.get_anomalies()
# For anomalous points in ts1, return correlated points and correlation coefficient.
for a in anomalies:
time_period = a.get_time_window()
try:
my_correlator = Correlator(ts1, ts2, time_period)
if my_correlator.is_correlated(threshold=0.8):
correlatorResultObj = my_correlator.get_correlation_result()
# Change time period to human readable format
start = strftime('%Y-%m-%d %H:%M:%S',
localtime(a.start_timestamp))
end = strftime('%Y-%m-%d %H:%M:%S', localtime(a.end_timestamp))
time_period = (start, end)
# Return anomalous time period, correlation coefficient and anomaly score.
# Note: Anomaly score for absolute threshold will be diff between value and threshold.
result = [
time_period,
round(correlatorResultObj.coefficient, 2),
round(a.anomaly_score, 2)
]
corrPoints.append(result)
except:
continue
return corrPoints
def correlate_series(s1: pd.Series, s2: pd.Series, window_len=41):
ts1 = s1.to_dict()
ts2 = s2.to_dict()
results = np.zeros(len(s1))
for i in range(len(s1) - window_len + 1):
time_period = (i, i + window_len)
correlator = Correlator(ts1, ts2, time_period=time_period)
results[i] = correlator.correlation_result.coefficient
return pd.Series(results)
def _analyze(self):
"""
Analyzes if a matrix has anomalies.
If any anomaly is found, determine if the matrix correlates with any other matrixes.
To be implemented.
"""
output = defaultdict(list)
output_by_name = defaultdict(list)
scores = self.anomaly_detector.get_all_scores()
if self.anomalies:
for anomaly in self.anomalies:
metrix_scores = scores
start_t, end_t = anomaly.get_time_window()
t = anomaly.exact_timestamp
# Compute extended start timestamp and extended end timestamp.
room = (end_t - start_t) / 2
if not room:
room = 30
extended_start_t = start_t - room
extended_end_t = end_t + room
metrix_scores_cropped = metrix_scores.crop(
extended_start_t, extended_end_t)
# Adjust the two timestamps if not enough data points are included.
while len(metrix_scores_cropped) < 2:
extended_start_t = extended_start_t - room
extended_end_t = extended_end_t + room
metrix_scores_cropped = metrix_scores.crop(
extended_start_t, extended_end_t)
# Correlate with other metrics
for entry in self.related_metrices:
try:
entry_correlation_result = Correlator(
self.metrix,
entry,
time_period=(extended_start_t, extended_end_t),
use_anomaly_score=True).get_correlation_result()
record = extended_start_t, extended_end_t, entry_correlation_result.__dict__, entry
record_by_name = extended_start_t, extended_end_t, entry_correlation_result.__dict__
output[t].append(record)
output_by_name[entry].append(record_by_name)
except exceptions.NotEnoughDataPoints:
pass
self.output = output
self.output_by_name = output_by_name
def setUp(self):
self.s1 = {0: 0, 1: 0, 2: 0, 3: 0, 4: 0.5, 5: 1, 6: 1, 7: 1, 8: 0}
self.s2 = {0: 0, 1: 0.5, 2: 1, 3: 1, 4: 1, 5: 0, 6: 0, 7: 0, 8: 0}
self.s3 = {0: 0, 1: 0.5, 2: 1, 3: 1, 4: 1, 5: 0}
self.correlator1 = Correlator(self.s1, self.s2)
self.correlator2 = Correlator(self.s1, self.s3)
class TestCorrelator(unittest.TestCase):
def setUp(self):
self.s1 = {0: 0, 1: 0, 2: 0, 3: 0, 4: 0.5, 5: 1, 6: 1, 7: 1, 8: 0}
self.s2 = {0: 0, 1: 0.5, 2: 1, 3: 1, 4: 1, 5: 0, 6: 0, 7: 0, 8: 0}
self.s3 = {0: 0, 1: 0.5, 2: 1, 3: 1, 4: 1, 5: 0}
self.correlator1 = Correlator(self.s1, self.s2)
self.correlator2 = Correlator(self.s1, self.s3)
def test_use_anomaly_score(self):
"""
Test if use_anomaly_score works as expected.
"""
correlator1 = Correlator(self.s1, self.s2, use_anomaly_score=True)
self.assertNotEqual(
self.correlator1.get_correlation_result().coefficient,
correlator1.get_correlation_result().coefficient)
def test_cross_correlation(self):
"""
Test if CrossCorrelation algorithm gives same results as expected.
"""
self.assertEqual(self.correlator1.get_correlation_result().coefficient,
self.correlator2.get_correlation_result().coefficient)
self.assertEqual(self.correlator1.get_correlation_result().shift,
self.correlator2.get_correlation_result().shift)
def test_if_correlate(self):
"""
Test if function is_correlated gives same result as function get_correlation_result
when there is a correlation.
"""
self.assertEqual(True, self.correlator2.is_correlated() is not None)
self.assertEqual(self.correlator2.get_correlation_result(),
self.correlator2.is_correlated())
def test_algorithm(self):
"""
Test if optional parameter algorithm works as expected.
"""
self.assertRaises(
exceptions.AlgorithmNotFound, lambda: Correlator(
self.s1, self.s2, algorithm_name='NotValidAlgorithm'))
correlator = Correlator(self.s1,
self.s2,
algorithm_name='cross_correlator')
self.assertEqual(self.correlator2.get_correlation_result().coefficient,
correlator.get_correlation_result().coefficient)
self.assertEqual(self.correlator2.get_correlation_result().shift,
correlator.get_correlation_result().shift)
def test_algorithm_params(self):
"""
Test if optional parameter algorithm_params works as expected.
"""
self.assertRaises(
exceptions.InvalidDataFormat,
lambda: Correlator(self.s1,
self.s2,
algorithm_name='cross_correlator',
algorithm_params=1))
correlator = Correlator(self.s1,
self.s2,
algorithm_name='cross_correlator',
algorithm_params={'max_shift_seconds': 180})
self.assertEqual(self.correlator2.get_correlation_result().coefficient,
correlator.get_correlation_result().coefficient)
def test_maximal_shift_seconds(self):
"""
Test if parameter max_shift_seconds works as expected.
"""
correlator = Correlator(self.s1,
self.s2,
algorithm_name='cross_correlator',
algorithm_params={'max_shift_seconds': 0})
self.assertNotEqual(
self.correlator2.get_correlation_result().coefficient,
correlator.get_correlation_result().coefficient)
def test_sanity_check(self):
"""
Test if exception NotEnoughDataPoints is raised as expected.
"""
s4 = {0: 0}
self.assertRaises(exceptions.NotEnoughDataPoints,
lambda: Correlator(s4, self.s1))
def test_time_series_format(self):
"""
Test if exception InvalidDataFormat is raised as expected.
"""
self.assertRaises(exceptions.InvalidDataFormat,
lambda: Correlator(list(), 1))
def test_time_series_format(self):
"""
Test if exception InvalidDataFormat is raised as expected.
"""
self.assertRaises(exceptions.InvalidDataFormat,
lambda: Correlator(list(), 1))
def test_maximal_shift_seconds(self):
"""
Test if parameter max_shift_seconds works as expected.
"""
correlator = Correlator(self.s1, self.s2, algorithm_name='cross_correlator', algorithm_params={'max_shift_seconds': 0})
self.assertNotEqual(self.correlator2.get_correlation_result().coefficient, correlator.get_correlation_result().coefficient)
def main(argv):
try:
opts, args = getopt.getopt(sys.argv[1:], "hi:c:v",
["help", "input="
"correlate="])
except getopt.GetoptError as err:
# print help information and exit:
print str(err) # will print something like "option -a not recognized"
usage()
sys.exit(2)
input = None
correlate = None
verbose = False
for o, a in opts:
if o == "-v":
verbose = True
elif o in ("-h", "--help"):
usage()
sys.exit()
elif o in ("-i", "--input"):
input = a
elif o in ("-c", "--correlate"):
correlate = a
else:
assert False, "unhandled option"
regex = r"(\s+)([+-]?\d*\.\d+)(?![-+0-9\.])(\s+)(\d+)(\s+)(\w+)(\s+)(\w+)(\s+)([a-z])(\s+)(\d+)(?:(?!X).)*(Length = )(\d+)(\s+)(BitCount = )(\d+)(\s+)(ID = )(\d+)"
pattern = re.compile(regex, re.UNICODE)
file1 = open(input).read()
matches1 = re.finditer(regex, file1)
file2 = open(correlate).read()
matches2 = re.finditer(regex, file2)
mydict1 = {}
mydict2 = {}
for matchNum, match in enumerate(matches1):
matchNum = matchNum + 1
myTime = match.group(2)
myTime = float(myTime) * 1000000
mydict1[myTime] = match.group(20)
for matchNum, match in enumerate(matches2):
matchNum = matchNum + 1
myTime = match.group(2)
myTime = float(myTime) * 1000000
mydict2[myTime] = match.group(20)
#print mydict1
my_detector1 = AnomalyDetector(mydict1,
algorithm_name=("exp_avg_detector"))
score1 = my_detector1.get_all_scores()
anomalies = my_detector1.get_anomalies()
for a in anomalies:
time_period = a.get_time_window()
my_correlator = Correlator(mydict1, mydict2, time_period)
if my_correlator.is_correlated(treshold=0.8):
print "mydict2 correlate with mydict at time period (%d, %d)" % time_period
def test_use_anomaly_score(self): """ Test if use_anomaly_score works as expected. """ correlator1 = Correlator(self.s1, self.s2, use_anomaly_score=True) self.assertNotEqual(self.correlator1.get_correlation_result().coefficient, correlator1.get_correlation_result().coefficient)
def get_correlations(
base_name, anomaly_timestamp, anomalous_ts, assigned_metrics, raw_assigned,
remote_assigned, anomalies):
logger = logging.getLogger(skyline_app_logger)
# Distill timeseries strings into lists
start = timer()
count = 0
metrics_checked_for_correlation = 0
# Sample the time series
# @modified 20180720 - Feature #2464: luminosity_remote_data
# Added note here - if you modify the value of 600 here, it must be
# modified in the luminosity_remote_data function in
# skyline/webapp/backend.py as well
from_timestamp = anomaly_timestamp - 600
correlated_metrics = []
correlations = []
no_data = False
if not anomalous_ts:
no_data = True
if not assigned_metrics:
no_data = True
if not raw_assigned:
no_data = True
if not anomalies:
no_data = True
if no_data:
logger.error('error :: get_correlations :: no data')
return (correlated_metrics, correlations)
# @added 20200428 - Feature #3510: Enable Luminosity to handle correlating namespaces only
# Feature #3500: webapp - crucible_process_metrics
# Feature #1448: Crucible web UI
# Discard the check if the anomaly_timestamp is not in FULL_DURATION as it
# will have been added via the Crucible or webapp/crucible route
start_timestamp_of_full_duration_data = int(time() - settings.FULL_DURATION)
if anomaly_timestamp < (start_timestamp_of_full_duration_data + 2000):
logger.info('get_correlations :: the anomaly_timestamp is too old not correlating')
return (correlated_metrics, correlations)
start_local_correlations = timer()
local_redis_metrics_checked_count = 0
local_redis_metrics_correlations_count = 0
logger.info('get_correlations :: the local Redis metric count is %s' % str(len(assigned_metrics)))
# @added 20200428 - Feature #3510: Enable Luminosity to handle correlating namespaces only
# Removed here and handled in get_assigned_metrics
for i, metric_name in enumerate(assigned_metrics):
count += 1
# print(metric_name)
# @modified 20180719 - Branch #2270: luminosity
# Removed test limiting that was errorneously left in
# if count > 1000:
# break
correlated = None
metric_base_name = metric_name.replace(settings.FULL_NAMESPACE, '', 1)
if str(metric_base_name) == str(base_name):
continue
try:
raw_series = raw_assigned[i]
unpacker = Unpacker(use_list=False)
unpacker.feed(raw_series)
timeseries = list(unpacker)
except:
timeseries = []
if not timeseries:
# print('no time series data for %s' % base_name)
continue
# @added 20200507 - Feature #3532: Sort all time series
# To ensure that there are no unordered timestamps in the time
# series which are artefacts of the collector or carbon-relay, sort
# all time series by timestamp before analysis.
original_timeseries = timeseries
if original_timeseries:
timeseries = sort_timeseries(original_timeseries)
del original_timeseries
# Convert the time series if this is a known_derivative_metric
known_derivative_metric = is_derivative_metric(skyline_app, metric_base_name)
if known_derivative_metric:
try:
derivative_timeseries = nonNegativeDerivative(timeseries)
timeseries = derivative_timeseries
except:
logger.error(traceback.format_exc())
logger.error('error :: nonNegativeDerivative')
correlate_ts = []
for ts, value in timeseries:
if int(ts) < from_timestamp:
continue
if int(ts) <= anomaly_timestamp:
correlate_ts.append((int(ts), value))
# @modified 20180720 - Feature #2464: luminosity_remote_data
# Added note here - if you modify the value of 61 here, it must be
# modified in the luminosity_remote_data function in
# skyline/webapp/backend.py as well
if int(ts) > (anomaly_timestamp + 61):
break
if not correlate_ts:
continue
local_redis_metrics_checked_count += 1
anomaly_ts_dict = dict(anomalous_ts)
correlate_ts_dict = dict(correlate_ts)
for a in anomalies:
try:
# @modified 20180720 - Feature #2464: luminosity_remote_data
# Added note here - if you modify the value of 120 here, it must be
# modified in the luminosity_remote_data function in
# skyline/webapp/backend.py as well
if int(a.exact_timestamp) < int(anomaly_timestamp - 120):
continue
if int(a.exact_timestamp) > int(anomaly_timestamp + 120):
continue
except:
continue
try:
time_period = (int(anomaly_timestamp - 120), int(anomaly_timestamp + 120))
my_correlator = Correlator(anomaly_ts_dict, correlate_ts_dict, time_period)
# For better correlation use 0.9 instead of 0.8 for the threshold
# @modified 20180524 - Feature #2360: CORRELATE_ALERTS_ONLY
# Branch #2270: luminosity
# Feature #2378: Add redis auth to Skyline and rebrow
# Added this to setting.py
# if my_correlator.is_correlated(threshold=0.9):
try:
cross_correlation_threshold = settings.LUMINOL_CROSS_CORRELATION_THRESHOLD
metrics_checked_for_correlation += 1
except:
cross_correlation_threshold = 0.9
if my_correlator.is_correlated(threshold=cross_correlation_threshold):
correlation = my_correlator.get_correlation_result()
correlated = True
correlations.append([metric_base_name, correlation.coefficient, correlation.shift, correlation.shifted_coefficient])
local_redis_metrics_correlations_count += 1
except:
pass
if correlated:
correlated_metrics.append(metric_base_name)
# @added 20180720 - Feature #2464: luminosity_remote_data
# Added the correlation of preprocessed remote data
end_local_correlations = timer()
logger.info('get_correlations :: checked - local_redis_metrics_checked_count is %s' % str(local_redis_metrics_checked_count))
logger.info('get_correlations :: correlated - local_redis_metrics_correlations_count is %s' % str(local_redis_metrics_correlations_count))
logger.info('get_correlations :: processed %s correlations on local_redis_metrics_checked_count %s local metrics in %.6f seconds' % (
str(local_redis_metrics_correlations_count),
str(local_redis_metrics_checked_count),
(end_local_correlations - start_local_correlations)))
remote_metrics_count = 0
remote_correlations_check_count = 0
remote_correlations_count = 0
logger.info('get_correlations :: remote_assigned count %s' % str(len(remote_assigned)))
start_remote_correlations = timer()
for ts_data in remote_assigned:
remote_metrics_count += 1
correlated = None
metric_name = str(ts_data[0])
metric_base_name = metric_name.replace(settings.FULL_NAMESPACE, '', 1)
if str(metric_base_name) == str(base_name):
continue
timeseries = []
try:
timeseries = ts_data[1]
except:
timeseries = []
if not timeseries:
continue
correlate_ts = []
for ts, value in timeseries:
if int(ts) < from_timestamp:
continue
if int(ts) <= anomaly_timestamp:
correlate_ts.append((int(ts), value))
# @modified 20180720 - Feature #2464: luminosity_remote_data
# Added note here - if you modify the value of 61 here, it must be
# modified in the luminosity_remote_data function in
# skyline/webapp/backend.py as well
if int(ts) > (anomaly_timestamp + 61):
break
if not correlate_ts:
continue
anomaly_ts_dict = dict(anomalous_ts)
correlate_ts_dict = dict(correlate_ts)
for a in anomalies:
try:
# @modified 20180720 - Feature #2464: luminosity_remote_data
# Added note here - if you modify the value of 120 here, it must be
# modified in the luminosity_remote_data function in
# skyline/webapp/backend.py as well
if int(a.exact_timestamp) < int(anomaly_timestamp - 120):
continue
if int(a.exact_timestamp) > int(anomaly_timestamp + 120):
continue
except:
continue
try:
time_period = (int(anomaly_timestamp - 120), int(anomaly_timestamp + 120))
my_correlator = Correlator(anomaly_ts_dict, correlate_ts_dict, time_period)
metrics_checked_for_correlation += 1
remote_correlations_check_count += 1
try:
cross_correlation_threshold = settings.LUMINOL_CROSS_CORRELATION_THRESHOLD
except:
cross_correlation_threshold = 0.9
if my_correlator.is_correlated(threshold=cross_correlation_threshold):
correlation = my_correlator.get_correlation_result()
correlated = True
correlations.append([metric_base_name, correlation.coefficient, correlation.shift, correlation.shifted_coefficient])
remote_correlations_count += 1
except:
pass
if correlated:
correlated_metrics.append(metric_base_name)
end_remote_correlations = timer()
logger.info('get_correlations :: checked - remote_correlations_check_count is %s' % str(remote_correlations_check_count))
logger.info('get_correlations :: correlated - remote_correlations_count is %s' % str(remote_correlations_count))
logger.info('get_correlations :: processed remote correlations on remote_metrics_count %s local metric in %.6f seconds' % (
str(remote_metrics_count),
(end_remote_correlations - start_remote_correlations)))
end = timer()
logger.info('get_correlations :: checked a total of %s metrics and correlated %s metrics to %s anomaly, processed in %.6f seconds' % (
str(metrics_checked_for_correlation), str(len(correlated_metrics)),
base_name, (end - start)))
# @added 20170720 - Task #2462: Implement useful metrics for Luminosity
# Added runtime to calculate avg_runtime Graphite metric
runtime = '%.6f' % (end - start)
return (correlated_metrics, correlations, metrics_checked_for_correlation, runtime)
time_series_b='./SAR-device.sdb.await__消弧线圈-LINE3-高阻接地__U0.csv',
time_period=time_period)
print(correlator.get_correlation_result().coefficient)
if correlator.is_correlated(threshold=0.7):
print('Ture Correlator')
else:
print('False Correlator')
"""
#####################
time_period = asAnomal[0].get_time_window()
correlator = Correlator( #time_series_a='./SAR-device.sdb.await__20190122_1500__负荷开关__0001__BAY01_0514_20181210_023118_850__U0__568-588.csv',
#time_series_a='./SAR-device.sdb.await__DCU1923ZeroOrder(4_8)2017-11-10 11_51_02_831016__U0_657-677.csv',
#time_series_b='./SAR-device.sdb.await__DCU1923ZeroOrder(4_8)2017-11-10 11_51_02_831016__U0_401-421.csv')
time_series_a=
'./SAR-device.sdb.await__20190122_1500__负荷开关__0001__BAY01_0514_20181210_023118_850__U0.csv',
time_series_b=
'./SAR-device.sdb.await__故障回放__PDZ810_20190108__RD_IN_287__F_BAY01_0239_20181126_071233_183__U0.csv'
)
#time_series_a='./SAR-device.sdb.await__故障回放__PDZ810_20190108__RD_IN_287__F_BAY01_0239_20181126_071233_183__U0.csv',
#time_series_b='./SAR-device.sdb.await__故障回放__PDZ810_20190108__RD_IN_287__F_BAY01_0239_20181126_071233_183__U0.csv',
#time_series_b='./SAR-device.sdb.await__DCU1923ZeroOrder(4_8)2017-11-10 11_51_02_831016__U0_913-933.csv')
#time_series_b='./SAR-device.sdb.await__New__Index580__L3 区外 L5 1000欧 B相__BAY01_0046_20181001_044812_313__U0_314-334.csv')
#time_period=time_period)
print(correlator.get_correlation_result().coefficient)
if correlator.is_correlated(threshold=0.2):
print('Ture Correlator')
else:
print('False Correlator')
# 1. 找到前 3 CYCLE 中最大的 MAXscore； 这个思路中 【 2倍的比例关系】 仅是根据已有数据的试验分析。
def setUp(self):
self.s1 = {0: 0, 1: 0, 2: 0, 3: 0, 4: 0.5, 5: 1, 6: 1, 7: 1, 8: 0}
self.s2 = {0: 0, 1: 0.5, 2: 1, 3: 1, 4: 1, 5: 0, 6: 0, 7: 0, 8: 0}
self.s3 = {0: 0, 1: 0.5, 2: 1, 3: 1, 4: 1, 5: 0}
self.correlator1 = Correlator(self.s1, self.s2)
self.correlator2 = Correlator(self.s1, self.s3)
def get_correlations(base_name, anomaly_timestamp, anomalous_ts,
assigned_metrics, raw_assigned, anomalies):
logger = logging.getLogger(skyline_app_logger)
# Distill timeseries strings into lists
start = timer()
count = 0
# Sample the time series
from_timestamp = anomaly_timestamp - 600
correlated_metrics = []
correlations = []
no_data = False
if not anomalous_ts:
no_data = True
if not assigned_metrics:
no_data = True
if not raw_assigned:
no_data = True
if not anomalies:
no_data = True
if no_data:
logger.error('error :: get_correlations :: no data')
return (correlated_metrics, correlations)
for i, metric_name in enumerate(assigned_metrics):
count += 1
# print(metric_name)
if count > 1000:
break
correlated = None
metric_base_name = metric_name.replace(settings.FULL_NAMESPACE, '', 1)
if str(metric_base_name) == str(base_name):
continue
try:
raw_series = raw_assigned[i]
unpacker = Unpacker(use_list=False)
unpacker.feed(raw_series)
timeseries = list(unpacker)
except:
timeseries = []
if not timeseries:
# print('no time series data for %s' % base_name)
continue
# Convert the time series if this is a known_derivative_metric
known_derivative_metric = is_derivative_metric(skyline_app,
metric_base_name)
if known_derivative_metric:
try:
derivative_timeseries = nonNegativeDerivative(timeseries)
timeseries = derivative_timeseries
except:
logger.error(traceback.format_exc())
logger.error('error :: nonNegativeDerivative')
correlate_ts = []
for ts, value in timeseries:
if int(ts) < from_timestamp:
continue
if int(ts) <= anomaly_timestamp:
correlate_ts.append((int(ts), value))
if int(ts) > (anomaly_timestamp + 61):
break
if not correlate_ts:
continue
anomaly_ts_dict = dict(anomalous_ts)
correlate_ts_dict = dict(correlate_ts)
for a in anomalies:
try:
if int(a.exact_timestamp) < int(anomaly_timestamp - 120):
continue
if int(a.exact_timestamp) > int(anomaly_timestamp + 120):
continue
except:
continue
try:
time_period = (int(anomaly_timestamp - 120),
int(anomaly_timestamp + 120))
my_correlator = Correlator(anomaly_ts_dict, correlate_ts_dict,
time_period)
# For better correlation use 0.9 instead of 0.8 for the threshold
# @modified 20180524 - Feature #2360: CORRELATE_ALERTS_ONLY
# Branch #2270: luminosity
# Feature #2378: Add redis auth to Skyline and rebrow
# Added this to setting.py
# if my_correlator.is_correlated(threshold=0.9):
try:
cross_correlation_threshold = settings.LUMINOL_CROSS_CORRELATION_THRESHOLD
except:
cross_correlation_threshold = 0.9
if my_correlator.is_correlated(
threshold=cross_correlation_threshold):
correlation = my_correlator.get_correlation_result()
correlated = True
correlations.append([
metric_base_name, correlation.coefficient,
correlation.shift, correlation.shifted_coefficient
])
except:
pass
if correlated:
correlated_metrics.append(metric_base_name)
end = timer()
logger.info(
'correlated %s metrics to %s anomaly, processed in %.6f seconds' %
(str(len(correlated_metrics)), base_name, (end - start)))
return (correlated_metrics, correlations)
