def test_algorithm_params(self): """ Test if optional parameter algorithm_params works as expected. """ self.assertRaises(exceptions.InvalidDataFormat, lambda: Correlator(self.s1, self.s2, algorithm_name='cross_correlator', algorithm_params=1)) correlator = Correlator(self.s1, self.s2, algorithm_name='cross_correlator', algorithm_params={'max_shift_seconds': 180}) self.assertEqual(self.correlator2.get_correlation_result().coefficient, correlator.get_correlation_result().coefficient)
def test_algorithm(self): """ Test if optional parameter algorithm works as expected. """ self.assertRaises(exceptions.AlgorithmNotFound, lambda: Correlator(self.s1, self.s2, algorithm_name='NotValidAlgorithm')) correlator = Correlator(self.s1, self.s2, algorithm_name='cross_correlator') self.assertEqual(self.correlator2.get_correlation_result().coefficient, correlator.get_correlation_result().coefficient) self.assertEqual(self.correlator2.get_correlation_result().shift, correlator.get_correlation_result().shift)
def test_use_anomaly_score(self): """ Test if use_anomaly_score works as expected. """ correlator1 = Correlator(self.s1, self.s2, use_anomaly_score=True) self.assertNotEqual( self.correlator1.get_correlation_result().coefficient, correlator1.get_correlation_result().coefficient)
class TestCorrelator(unittest.TestCase): def setUp(self): self.s1 = {0: 0, 1: 0, 2: 0, 3: 0, 4: 0.5, 5: 1, 6: 1, 7: 1, 8: 0} self.s2 = {0: 0, 1: 0.5, 2: 1, 3: 1, 4: 1, 5: 0, 6: 0, 7: 0, 8: 0} self.s3 = {0: 0, 1: 0.5, 2: 1, 3: 1, 4: 1, 5: 0} self.correlator1 = Correlator(self.s1, self.s2) self.correlator2 = Correlator(self.s1, self.s3) def test_cross_correlation(self): """ Test if CrossCorrelation algorithm gives same results as expected. """ self.assertEqual(self.correlator1.get_correlation_result().coefficient, self.correlator2.get_correlation_result().coefficient) self.assertEqual(self.correlator1.get_correlation_result().shift, self.correlator2.get_correlation_result().shift) def test_if_correlate(self): """ Test if function is_correlated gives same result as function get_correlation_result when there is a correlation. """ self.assertEqual(True, self.correlator2.is_correlated() is not None) self.assertEqual(self.correlator2.get_correlation_result(), self.correlator2.is_correlated()) def test_algorithm(self): """ Test if optional parameter algorithm works as expected. """ self.assertRaises(exceptions.AlgorithmNotFound, lambda: Correlator(self.s1, self.s2, algorithm_name='NotValidAlgorithm')) correlator = Correlator(self.s1, self.s2, algorithm_name='cross_correlator') self.assertEqual(self.correlator2.get_correlation_result().coefficient, correlator.get_correlation_result().coefficient) self.assertEqual(self.correlator2.get_correlation_result().shift, correlator.get_correlation_result().shift) def test_algorithm_params(self): """ Test if optional parameter algorithm_params works as expected. """ self.assertRaises(exceptions.InvalidDataFormat, lambda: Correlator(self.s1, self.s2, algorithm_name='cross_correlator', algorithm_params=1)) correlator = Correlator(self.s1, self.s2, algorithm_name='cross_correlator', algorithm_params={'max_shift_seconds': 180}) self.assertEqual(self.correlator2.get_correlation_result().coefficient, correlator.get_correlation_result().coefficient) def test_maximal_shift_seconds(self): """ Test if parameter max_shift_seconds works as expected. """ correlator = Correlator(self.s1, self.s2, algorithm_name='cross_correlator', algorithm_params={'max_shift_seconds': 0}) self.assertNotEqual(self.correlator2.get_correlation_result().coefficient, correlator.get_correlation_result().coefficient) def test_sanity_check(self): """ Test if exception NotEnoughDataPoints is raised as expected. """ s4 = {0: 0} self.assertRaises(exceptions.NotEnoughDataPoints, lambda: Correlator(s4, self.s1)) def test_time_series_format(self): """ Test if exception InvalidDataFormat is raised as expected. """ self.assertRaises(exceptions.InvalidDataFormat, lambda: Correlator(list(), 1))
def test_maximal_shift_seconds(self): """ Test if parameter max_shift_seconds works as expected. """ correlator = Correlator(self.s1, self.s2, algorithm_name='cross_correlator', algorithm_params={'max_shift_seconds': 0}) self.assertNotEqual( self.correlator2.get_correlation_result().coefficient, correlator.get_correlation_result().coefficient)
def test_sanity_check(self): """ Test if exception NotEnoughDataPoints is raised as expected. """ s4 = {0: 0} self.assertRaises(exceptions.NotEnoughDataPoints, lambda: Correlator(s4, self.s1))
def pointsOfCorrelation(ts1, ts2, thresholdVal): corrPoints = [] # Conduct AD on each of each of the time series. # algorithm_params={'absolute_threshold_value_lower':lower,'absolute_threshold_value_upper':upper} # detector = AnomalyDetector(ts2, score_threshold=thresholdVal, algorithm_name="derivative_detector") detector = AnomalyDetector(ts2, score_threshold=thresholdVal, algorithm_name="exp_avg_detector") # score = detector.get_all_scores() anomalies = detector.get_anomalies() # For anomalous points in ts1, return correlated points and correlation coefficient. for a in anomalies: time_period = a.get_time_window() try: my_correlator = Correlator(ts1, ts2, time_period) if my_correlator.is_correlated(threshold=0.8): correlatorResultObj = my_correlator.get_correlation_result() # Change time period to human readable format start = strftime('%Y-%m-%d %H:%M:%S', localtime(a.start_timestamp)) end = strftime('%Y-%m-%d %H:%M:%S', localtime(a.end_timestamp)) time_period = (start, end) # Return anomalous time period, correlation coefficient and anomaly score. # Note: Anomaly score for absolute threshold will be diff between value and threshold. result = [ time_period, round(correlatorResultObj.coefficient, 2), round(a.anomaly_score, 2) ] corrPoints.append(result) except: continue return corrPoints
def correlate_series(s1: pd.Series, s2: pd.Series, window_len=41): ts1 = s1.to_dict() ts2 = s2.to_dict() results = np.zeros(len(s1)) for i in range(len(s1) - window_len + 1): time_period = (i, i + window_len) correlator = Correlator(ts1, ts2, time_period=time_period) results[i] = correlator.correlation_result.coefficient return pd.Series(results)
def _analyze(self): """ Analyzes if a matrix has anomalies. If any anomaly is found, determine if the matrix correlates with any other matrixes. To be implemented. """ output = defaultdict(list) output_by_name = defaultdict(list) scores = self.anomaly_detector.get_all_scores() if self.anomalies: for anomaly in self.anomalies: metrix_scores = scores start_t, end_t = anomaly.get_time_window() t = anomaly.exact_timestamp # Compute extended start timestamp and extended end timestamp. room = (end_t - start_t) / 2 if not room: room = 30 extended_start_t = start_t - room extended_end_t = end_t + room metrix_scores_cropped = metrix_scores.crop( extended_start_t, extended_end_t) # Adjust the two timestamps if not enough data points are included. while len(metrix_scores_cropped) < 2: extended_start_t = extended_start_t - room extended_end_t = extended_end_t + room metrix_scores_cropped = metrix_scores.crop( extended_start_t, extended_end_t) # Correlate with other metrics for entry in self.related_metrices: try: entry_correlation_result = Correlator( self.metrix, entry, time_period=(extended_start_t, extended_end_t), use_anomaly_score=True).get_correlation_result() record = extended_start_t, extended_end_t, entry_correlation_result.__dict__, entry record_by_name = extended_start_t, extended_end_t, entry_correlation_result.__dict__ output[t].append(record) output_by_name[entry].append(record_by_name) except exceptions.NotEnoughDataPoints: pass self.output = output self.output_by_name = output_by_name
def setUp(self): self.s1 = {0: 0, 1: 0, 2: 0, 3: 0, 4: 0.5, 5: 1, 6: 1, 7: 1, 8: 0} self.s2 = {0: 0, 1: 0.5, 2: 1, 3: 1, 4: 1, 5: 0, 6: 0, 7: 0, 8: 0} self.s3 = {0: 0, 1: 0.5, 2: 1, 3: 1, 4: 1, 5: 0} self.correlator1 = Correlator(self.s1, self.s2) self.correlator2 = Correlator(self.s1, self.s3)
class TestCorrelator(unittest.TestCase): def setUp(self): self.s1 = {0: 0, 1: 0, 2: 0, 3: 0, 4: 0.5, 5: 1, 6: 1, 7: 1, 8: 0} self.s2 = {0: 0, 1: 0.5, 2: 1, 3: 1, 4: 1, 5: 0, 6: 0, 7: 0, 8: 0} self.s3 = {0: 0, 1: 0.5, 2: 1, 3: 1, 4: 1, 5: 0} self.correlator1 = Correlator(self.s1, self.s2) self.correlator2 = Correlator(self.s1, self.s3) def test_use_anomaly_score(self): """ Test if use_anomaly_score works as expected. """ correlator1 = Correlator(self.s1, self.s2, use_anomaly_score=True) self.assertNotEqual( self.correlator1.get_correlation_result().coefficient, correlator1.get_correlation_result().coefficient) def test_cross_correlation(self): """ Test if CrossCorrelation algorithm gives same results as expected. """ self.assertEqual(self.correlator1.get_correlation_result().coefficient, self.correlator2.get_correlation_result().coefficient) self.assertEqual(self.correlator1.get_correlation_result().shift, self.correlator2.get_correlation_result().shift) def test_if_correlate(self): """ Test if function is_correlated gives same result as function get_correlation_result when there is a correlation. """ self.assertEqual(True, self.correlator2.is_correlated() is not None) self.assertEqual(self.correlator2.get_correlation_result(), self.correlator2.is_correlated()) def test_algorithm(self): """ Test if optional parameter algorithm works as expected. """ self.assertRaises( exceptions.AlgorithmNotFound, lambda: Correlator( self.s1, self.s2, algorithm_name='NotValidAlgorithm')) correlator = Correlator(self.s1, self.s2, algorithm_name='cross_correlator') self.assertEqual(self.correlator2.get_correlation_result().coefficient, correlator.get_correlation_result().coefficient) self.assertEqual(self.correlator2.get_correlation_result().shift, correlator.get_correlation_result().shift) def test_algorithm_params(self): """ Test if optional parameter algorithm_params works as expected. """ self.assertRaises( exceptions.InvalidDataFormat, lambda: Correlator(self.s1, self.s2, algorithm_name='cross_correlator', algorithm_params=1)) correlator = Correlator(self.s1, self.s2, algorithm_name='cross_correlator', algorithm_params={'max_shift_seconds': 180}) self.assertEqual(self.correlator2.get_correlation_result().coefficient, correlator.get_correlation_result().coefficient) def test_maximal_shift_seconds(self): """ Test if parameter max_shift_seconds works as expected. """ correlator = Correlator(self.s1, self.s2, algorithm_name='cross_correlator', algorithm_params={'max_shift_seconds': 0}) self.assertNotEqual( self.correlator2.get_correlation_result().coefficient, correlator.get_correlation_result().coefficient) def test_sanity_check(self): """ Test if exception NotEnoughDataPoints is raised as expected. """ s4 = {0: 0} self.assertRaises(exceptions.NotEnoughDataPoints, lambda: Correlator(s4, self.s1)) def test_time_series_format(self): """ Test if exception InvalidDataFormat is raised as expected. """ self.assertRaises(exceptions.InvalidDataFormat, lambda: Correlator(list(), 1))
def test_time_series_format(self): """ Test if exception InvalidDataFormat is raised as expected. """ self.assertRaises(exceptions.InvalidDataFormat, lambda: Correlator(list(), 1))
def test_maximal_shift_seconds(self): """ Test if parameter max_shift_seconds works as expected. """ correlator = Correlator(self.s1, self.s2, algorithm_name='cross_correlator', algorithm_params={'max_shift_seconds': 0}) self.assertNotEqual(self.correlator2.get_correlation_result().coefficient, correlator.get_correlation_result().coefficient)
def main(argv): try: opts, args = getopt.getopt(sys.argv[1:], "hi:c:v", ["help", "input=" "correlate="]) except getopt.GetoptError as err: # print help information and exit: print str(err) # will print something like "option -a not recognized" usage() sys.exit(2) input = None correlate = None verbose = False for o, a in opts: if o == "-v": verbose = True elif o in ("-h", "--help"): usage() sys.exit() elif o in ("-i", "--input"): input = a elif o in ("-c", "--correlate"): correlate = a else: assert False, "unhandled option" regex = r"(\s+)([+-]?\d*\.\d+)(?![-+0-9\.])(\s+)(\d+)(\s+)(\w+)(\s+)(\w+)(\s+)([a-z])(\s+)(\d+)(?:(?!X).)*(Length = )(\d+)(\s+)(BitCount = )(\d+)(\s+)(ID = )(\d+)" pattern = re.compile(regex, re.UNICODE) file1 = open(input).read() matches1 = re.finditer(regex, file1) file2 = open(correlate).read() matches2 = re.finditer(regex, file2) mydict1 = {} mydict2 = {} for matchNum, match in enumerate(matches1): matchNum = matchNum + 1 myTime = match.group(2) myTime = float(myTime) * 1000000 mydict1[myTime] = match.group(20) for matchNum, match in enumerate(matches2): matchNum = matchNum + 1 myTime = match.group(2) myTime = float(myTime) * 1000000 mydict2[myTime] = match.group(20) #print mydict1 my_detector1 = AnomalyDetector(mydict1, algorithm_name=("exp_avg_detector")) score1 = my_detector1.get_all_scores() anomalies = my_detector1.get_anomalies() for a in anomalies: time_period = a.get_time_window() my_correlator = Correlator(mydict1, mydict2, time_period) if my_correlator.is_correlated(treshold=0.8): print "mydict2 correlate with mydict at time period (%d, %d)" % time_period
def test_use_anomaly_score(self): """ Test if use_anomaly_score works as expected. """ correlator1 = Correlator(self.s1, self.s2, use_anomaly_score=True) self.assertNotEqual(self.correlator1.get_correlation_result().coefficient, correlator1.get_correlation_result().coefficient)
def get_correlations( base_name, anomaly_timestamp, anomalous_ts, assigned_metrics, raw_assigned, remote_assigned, anomalies): logger = logging.getLogger(skyline_app_logger) # Distill timeseries strings into lists start = timer() count = 0 metrics_checked_for_correlation = 0 # Sample the time series # @modified 20180720 - Feature #2464: luminosity_remote_data # Added note here - if you modify the value of 600 here, it must be # modified in the luminosity_remote_data function in # skyline/webapp/backend.py as well from_timestamp = anomaly_timestamp - 600 correlated_metrics = [] correlations = [] no_data = False if not anomalous_ts: no_data = True if not assigned_metrics: no_data = True if not raw_assigned: no_data = True if not anomalies: no_data = True if no_data: logger.error('error :: get_correlations :: no data') return (correlated_metrics, correlations) # @added 20200428 - Feature #3510: Enable Luminosity to handle correlating namespaces only # Feature #3500: webapp - crucible_process_metrics # Feature #1448: Crucible web UI # Discard the check if the anomaly_timestamp is not in FULL_DURATION as it # will have been added via the Crucible or webapp/crucible route start_timestamp_of_full_duration_data = int(time() - settings.FULL_DURATION) if anomaly_timestamp < (start_timestamp_of_full_duration_data + 2000): logger.info('get_correlations :: the anomaly_timestamp is too old not correlating') return (correlated_metrics, correlations) start_local_correlations = timer() local_redis_metrics_checked_count = 0 local_redis_metrics_correlations_count = 0 logger.info('get_correlations :: the local Redis metric count is %s' % str(len(assigned_metrics))) # @added 20200428 - Feature #3510: Enable Luminosity to handle correlating namespaces only # Removed here and handled in get_assigned_metrics for i, metric_name in enumerate(assigned_metrics): count += 1 # print(metric_name) # @modified 20180719 - Branch #2270: luminosity # Removed test limiting that was errorneously left in # if count > 1000: # break correlated = None metric_base_name = metric_name.replace(settings.FULL_NAMESPACE, '', 1) if str(metric_base_name) == str(base_name): continue try: raw_series = raw_assigned[i] unpacker = Unpacker(use_list=False) unpacker.feed(raw_series) timeseries = list(unpacker) except: timeseries = [] if not timeseries: # print('no time series data for %s' % base_name) continue # @added 20200507 - Feature #3532: Sort all time series # To ensure that there are no unordered timestamps in the time # series which are artefacts of the collector or carbon-relay, sort # all time series by timestamp before analysis. original_timeseries = timeseries if original_timeseries: timeseries = sort_timeseries(original_timeseries) del original_timeseries # Convert the time series if this is a known_derivative_metric known_derivative_metric = is_derivative_metric(skyline_app, metric_base_name) if known_derivative_metric: try: derivative_timeseries = nonNegativeDerivative(timeseries) timeseries = derivative_timeseries except: logger.error(traceback.format_exc()) logger.error('error :: nonNegativeDerivative') correlate_ts = [] for ts, value in timeseries: if int(ts) < from_timestamp: continue if int(ts) <= anomaly_timestamp: correlate_ts.append((int(ts), value)) # @modified 20180720 - Feature #2464: luminosity_remote_data # Added note here - if you modify the value of 61 here, it must be # modified in the luminosity_remote_data function in # skyline/webapp/backend.py as well if int(ts) > (anomaly_timestamp + 61): break if not correlate_ts: continue local_redis_metrics_checked_count += 1 anomaly_ts_dict = dict(anomalous_ts) correlate_ts_dict = dict(correlate_ts) for a in anomalies: try: # @modified 20180720 - Feature #2464: luminosity_remote_data # Added note here - if you modify the value of 120 here, it must be # modified in the luminosity_remote_data function in # skyline/webapp/backend.py as well if int(a.exact_timestamp) < int(anomaly_timestamp - 120): continue if int(a.exact_timestamp) > int(anomaly_timestamp + 120): continue except: continue try: time_period = (int(anomaly_timestamp - 120), int(anomaly_timestamp + 120)) my_correlator = Correlator(anomaly_ts_dict, correlate_ts_dict, time_period) # For better correlation use 0.9 instead of 0.8 for the threshold # @modified 20180524 - Feature #2360: CORRELATE_ALERTS_ONLY # Branch #2270: luminosity # Feature #2378: Add redis auth to Skyline and rebrow # Added this to setting.py # if my_correlator.is_correlated(threshold=0.9): try: cross_correlation_threshold = settings.LUMINOL_CROSS_CORRELATION_THRESHOLD metrics_checked_for_correlation += 1 except: cross_correlation_threshold = 0.9 if my_correlator.is_correlated(threshold=cross_correlation_threshold): correlation = my_correlator.get_correlation_result() correlated = True correlations.append([metric_base_name, correlation.coefficient, correlation.shift, correlation.shifted_coefficient]) local_redis_metrics_correlations_count += 1 except: pass if correlated: correlated_metrics.append(metric_base_name) # @added 20180720 - Feature #2464: luminosity_remote_data # Added the correlation of preprocessed remote data end_local_correlations = timer() logger.info('get_correlations :: checked - local_redis_metrics_checked_count is %s' % str(local_redis_metrics_checked_count)) logger.info('get_correlations :: correlated - local_redis_metrics_correlations_count is %s' % str(local_redis_metrics_correlations_count)) logger.info('get_correlations :: processed %s correlations on local_redis_metrics_checked_count %s local metrics in %.6f seconds' % ( str(local_redis_metrics_correlations_count), str(local_redis_metrics_checked_count), (end_local_correlations - start_local_correlations))) remote_metrics_count = 0 remote_correlations_check_count = 0 remote_correlations_count = 0 logger.info('get_correlations :: remote_assigned count %s' % str(len(remote_assigned))) start_remote_correlations = timer() for ts_data in remote_assigned: remote_metrics_count += 1 correlated = None metric_name = str(ts_data[0]) metric_base_name = metric_name.replace(settings.FULL_NAMESPACE, '', 1) if str(metric_base_name) == str(base_name): continue timeseries = [] try: timeseries = ts_data[1] except: timeseries = [] if not timeseries: continue correlate_ts = [] for ts, value in timeseries: if int(ts) < from_timestamp: continue if int(ts) <= anomaly_timestamp: correlate_ts.append((int(ts), value)) # @modified 20180720 - Feature #2464: luminosity_remote_data # Added note here - if you modify the value of 61 here, it must be # modified in the luminosity_remote_data function in # skyline/webapp/backend.py as well if int(ts) > (anomaly_timestamp + 61): break if not correlate_ts: continue anomaly_ts_dict = dict(anomalous_ts) correlate_ts_dict = dict(correlate_ts) for a in anomalies: try: # @modified 20180720 - Feature #2464: luminosity_remote_data # Added note here - if you modify the value of 120 here, it must be # modified in the luminosity_remote_data function in # skyline/webapp/backend.py as well if int(a.exact_timestamp) < int(anomaly_timestamp - 120): continue if int(a.exact_timestamp) > int(anomaly_timestamp + 120): continue except: continue try: time_period = (int(anomaly_timestamp - 120), int(anomaly_timestamp + 120)) my_correlator = Correlator(anomaly_ts_dict, correlate_ts_dict, time_period) metrics_checked_for_correlation += 1 remote_correlations_check_count += 1 try: cross_correlation_threshold = settings.LUMINOL_CROSS_CORRELATION_THRESHOLD except: cross_correlation_threshold = 0.9 if my_correlator.is_correlated(threshold=cross_correlation_threshold): correlation = my_correlator.get_correlation_result() correlated = True correlations.append([metric_base_name, correlation.coefficient, correlation.shift, correlation.shifted_coefficient]) remote_correlations_count += 1 except: pass if correlated: correlated_metrics.append(metric_base_name) end_remote_correlations = timer() logger.info('get_correlations :: checked - remote_correlations_check_count is %s' % str(remote_correlations_check_count)) logger.info('get_correlations :: correlated - remote_correlations_count is %s' % str(remote_correlations_count)) logger.info('get_correlations :: processed remote correlations on remote_metrics_count %s local metric in %.6f seconds' % ( str(remote_metrics_count), (end_remote_correlations - start_remote_correlations))) end = timer() logger.info('get_correlations :: checked a total of %s metrics and correlated %s metrics to %s anomaly, processed in %.6f seconds' % ( str(metrics_checked_for_correlation), str(len(correlated_metrics)), base_name, (end - start))) # @added 20170720 - Task #2462: Implement useful metrics for Luminosity # Added runtime to calculate avg_runtime Graphite metric runtime = '%.6f' % (end - start) return (correlated_metrics, correlations, metrics_checked_for_correlation, runtime)
time_series_b='./SAR-device.sdb.await__消弧线圈-LINE3-高阻接地__U0.csv', time_period=time_period) print(correlator.get_correlation_result().coefficient) if correlator.is_correlated(threshold=0.7): print('Ture Correlator') else: print('False Correlator') """ ##################### time_period = asAnomal[0].get_time_window() correlator = Correlator( #time_series_a='./SAR-device.sdb.await__20190122_1500__负荷开关__0001__BAY01_0514_20181210_023118_850__U0__568-588.csv', #time_series_a='./SAR-device.sdb.await__DCU1923ZeroOrder(4_8)2017-11-10 11_51_02_831016__U0_657-677.csv', #time_series_b='./SAR-device.sdb.await__DCU1923ZeroOrder(4_8)2017-11-10 11_51_02_831016__U0_401-421.csv') time_series_a= './SAR-device.sdb.await__20190122_1500__负荷开关__0001__BAY01_0514_20181210_023118_850__U0.csv', time_series_b= './SAR-device.sdb.await__故障回放__PDZ810_20190108__RD_IN_287__F_BAY01_0239_20181126_071233_183__U0.csv' ) #time_series_a='./SAR-device.sdb.await__故障回放__PDZ810_20190108__RD_IN_287__F_BAY01_0239_20181126_071233_183__U0.csv', #time_series_b='./SAR-device.sdb.await__故障回放__PDZ810_20190108__RD_IN_287__F_BAY01_0239_20181126_071233_183__U0.csv', #time_series_b='./SAR-device.sdb.await__DCU1923ZeroOrder(4_8)2017-11-10 11_51_02_831016__U0_913-933.csv') #time_series_b='./SAR-device.sdb.await__New__Index580__L3 区外 L5 1000欧 B相__BAY01_0046_20181001_044812_313__U0_314-334.csv') #time_period=time_period) print(correlator.get_correlation_result().coefficient) if correlator.is_correlated(threshold=0.2): print('Ture Correlator') else: print('False Correlator') # 1. 找到前 3 CYCLE 中最大的 MAXscore; 这个思路中 【 2倍的比例关系】 仅是根据已有数据的试验分析。
def get_correlations(base_name, anomaly_timestamp, anomalous_ts, assigned_metrics, raw_assigned, anomalies): logger = logging.getLogger(skyline_app_logger) # Distill timeseries strings into lists start = timer() count = 0 # Sample the time series from_timestamp = anomaly_timestamp - 600 correlated_metrics = [] correlations = [] no_data = False if not anomalous_ts: no_data = True if not assigned_metrics: no_data = True if not raw_assigned: no_data = True if not anomalies: no_data = True if no_data: logger.error('error :: get_correlations :: no data') return (correlated_metrics, correlations) for i, metric_name in enumerate(assigned_metrics): count += 1 # print(metric_name) if count > 1000: break correlated = None metric_base_name = metric_name.replace(settings.FULL_NAMESPACE, '', 1) if str(metric_base_name) == str(base_name): continue try: raw_series = raw_assigned[i] unpacker = Unpacker(use_list=False) unpacker.feed(raw_series) timeseries = list(unpacker) except: timeseries = [] if not timeseries: # print('no time series data for %s' % base_name) continue # Convert the time series if this is a known_derivative_metric known_derivative_metric = is_derivative_metric(skyline_app, metric_base_name) if known_derivative_metric: try: derivative_timeseries = nonNegativeDerivative(timeseries) timeseries = derivative_timeseries except: logger.error(traceback.format_exc()) logger.error('error :: nonNegativeDerivative') correlate_ts = [] for ts, value in timeseries: if int(ts) < from_timestamp: continue if int(ts) <= anomaly_timestamp: correlate_ts.append((int(ts), value)) if int(ts) > (anomaly_timestamp + 61): break if not correlate_ts: continue anomaly_ts_dict = dict(anomalous_ts) correlate_ts_dict = dict(correlate_ts) for a in anomalies: try: if int(a.exact_timestamp) < int(anomaly_timestamp - 120): continue if int(a.exact_timestamp) > int(anomaly_timestamp + 120): continue except: continue try: time_period = (int(anomaly_timestamp - 120), int(anomaly_timestamp + 120)) my_correlator = Correlator(anomaly_ts_dict, correlate_ts_dict, time_period) # For better correlation use 0.9 instead of 0.8 for the threshold # @modified 20180524 - Feature #2360: CORRELATE_ALERTS_ONLY # Branch #2270: luminosity # Feature #2378: Add redis auth to Skyline and rebrow # Added this to setting.py # if my_correlator.is_correlated(threshold=0.9): try: cross_correlation_threshold = settings.LUMINOL_CROSS_CORRELATION_THRESHOLD except: cross_correlation_threshold = 0.9 if my_correlator.is_correlated( threshold=cross_correlation_threshold): correlation = my_correlator.get_correlation_result() correlated = True correlations.append([ metric_base_name, correlation.coefficient, correlation.shift, correlation.shifted_coefficient ]) except: pass if correlated: correlated_metrics.append(metric_base_name) end = timer() logger.info( 'correlated %s metrics to %s anomaly, processed in %.6f seconds' % (str(len(correlated_metrics)), base_name, (end - start))) return (correlated_metrics, correlations)