def fit(self, x): tmp_dict = {} for i in range(len(x)): tmp_dict[i] = x[i] self.cls = anomaly_detector.AnomalyDetector( tmp_dict, algorithm_name=ALGO_NAME, algorithm_params=self.param_dict)
def calculate_anomalies(ts): """Takes in a df column of concentrations and returns a list of anomaly scores""" tsd = ts.to_dict() detector = anomaly_detector.AnomalyDetector(tsd) score = detector.get_all_scores() dfs = pd.DataFrame(score.iteritems()) return dfs[1]
def find_anomalies(data): """Run luminol and find anomalies""" points = {} i = 0 for point in data: points[i] = point i += 1 detector = ad.AnomalyDetector(points, algorithm_name='default_detector') anomalies = detector.get_anomalies() return anomalies
def fit(self, X): """Fit detector. Parameters ---------- X : dataframe of shape (n_samples, n_features) The input samples. """ # a=str(ts[:,0]) X = X.to_numpy() timestamp = np.asarray(X[:, 0].astype(np.datetime64)) pca = IncrementalPCA(n_components=1) value = np.reshape(pca.fit_transform(X[:, 1:]), -1) X = pd.Series(value, index=timestamp) X.index = X.index.map(lambda d: to_epoch(str(d))) lts = TimeSeries(X.to_dict()) self.ts = timestamp self.ts_value = value self.detector = anomaly_detector.AnomalyDetector(lts) return self
def detect_anomaly(self): """ Detect anomalies in the timeseries data for the submetrics specified in the config file. Identified anomalies are stored in self.anomalies as well as written to .anomalies.csv file to be used by the client charting page. Anomaly detection uses the luminol library (http://pypi.python.org/pypi/luminol) """ if not self.anomaly_detection_metrics or len(self.anomaly_detection_metrics) <= 0: return for submetric in self.anomaly_detection_metrics: csv_file = self.get_csv(submetric) if naarad.utils.is_valid_file(csv_file): detector = anomaly_detector.AnomalyDetector(csv_file) anomalies = detector.get_anomalies() if len(anomalies) <= 0: return self.anomalies[submetric] = anomalies anomaly_csv_file = os.path.join(self.resource_directory, self.label + '.' + submetric + '.anomalies.csv') with open(anomaly_csv_file, 'w') as FH: for anomaly in anomalies: FH.write(",".join([str(anomaly.anomaly_score), str(anomaly.start_timestamp), str(anomaly.end_timestamp), str(anomaly.exact_timestamp)])) FH.write('\n')
def luminoldetect(): ts = urllib.unquote(request.args.get('ts_path')[1:]) my_detector = anomaly_detector.AnomalyDetector(ts) score = my_detector.get_all_scores().values # Create a new csv file that contains both values and scores. anom_scores = list() for i, (timestamp, value) in enumerate(my_detector.time_series.iteritems()): t_str = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(timestamp / 1000)) anom_scores.append([t_str, value, score[i]]) write_csv(anom_scores, ts.split('/')[-1]) anoms = my_detector.get_anomalies() result = list() for anom in anoms: entry = list() anom_dict = anom.__dict__ for key in anom_dict: entry.append(anom_dict[key]) result.append(entry) return jsonify(anom=result, anom_score=anom_scores)
def real_anomaly_detection(samples, smoothing_factor): ts = TimeSeries(samples) detector = anomaly_detector.AnomalyDetector( ts, algorithm_name="derivative_detector", algorithm_params={ 'smoothing_factor': smoothing_factor # smoothing factor used to compute exponential moving averages }) anomalies = detector.get_anomalies() anomaly_times = [] for a in anomalies: start = a.get_time_window()[0] end = a.get_time_window()[1] while (start <= end): anomaly_times.append(start) start += 1 print anomaly_times return anomaly_times
def run_anomaly(metric_name, start_time, stop_time, push_ready=True): # metric_name="devbeefydocker01.aunalytics.com.vmstat.cpu.user" # from_time="-24h" # to_time="now" temp = get_graphite_data(metric_name, start_time, stop_time) detector = anomaly_detector.AnomalyDetector(temp) anomalies = detector.get_anomalies() # print (anomalies) anomalies_windows = [] for ac in anomalies: # print(ac.get_time_window()) anomalies_windows.append(ac.get_time_window()) if push_ready is True: ra = [] for c in anomalies_windows: if int(c[0]) > (time.time() - 3600): for r in range(c[0], c[1], 60): ra.append(metric_name + "_anomaly 1 " + str(r) + "\n") else: ra = anomalies_windows return temp, ra
from datetime import datetime from typing import List import pandas as pd from luminol import anomaly_detector from luminol.modules.anomaly import Anomaly sold_items = 'data/sold-items-de-summer-2018.csv' pd.read_csv(sold_items) detector = anomaly_detector.AnomalyDetector(time_series=sold_items) anomalies: List[Anomaly] = detector.get_anomalies() for a in anomalies: dd = datetime.utcfromtimestamp( a.exact_timestamp).strftime('%Y-%m-%d %H:%M:%S') print(f'{dd}: {a.anomaly_score}')
# In[40]: # output = output.set_index(0) outputDF['tickepoch'] = outputDF['tickdate'].apply( lambda x: time.mktime(time.strptime(str(x), '%Y-%m-%d %H:%M:%S'))) outputDF = outputDF.set_index('tickepoch') # In[41]: outputDF.head() # In[42]: # to_dict adds another wrapper dict for each column ts_dict = outputDF.to_dict()['tickvalue'] detector = anomaly_detector.AnomalyDetector(ts_dict) anomalies = detector.get_anomalies() # In[43]: anomaly = anomalies[0] print anomaly print str(datetime.datetime.fromtimestamp(anomaly.start_timestamp)) # In[44]: import matplotlib.pyplot as plt # In[47]: anomaly_list = []
model.fit_predict(X) lof = model.negative_outlier_factor_ plt.plot(lof / min(lof), color="orange") plt.title("LocalOutlierFactor On Feature 1") plt.plot(Y_f) plt.show() #LocalOutlierFactor başarılı #%% LUMINOL from luminol import anomaly_detector df = df.reset_index() df["timestamp"] = (df["timestamp"].astype('uint64') / 1e6).astype('uint32') df.set_index("timestamp") a = df.drop(columns=["Feature2"]).to_dict("series")["Feature1"].to_dict() detector = anomaly_detector.AnomalyDetector(a, algorithm_name="exp_avg_detector", score_threshold=0.2) anomalies = detector.get_anomalies() time_periods = [] values = [] for key in (a): time_periods.append(key) values.append(a[key]) plt.plot(time_periods, values) anom_times = [] anom_values = [] for anom in anomalies: anom_times.append(np.arange(anom.start_timestamp, anom.end_timestamp)) anom_values.append(anom.anomaly_score)
# In[58]: # output = output.set_index(0) outputDF['tickepoch'] = outputDF['tickdate'].apply(lambda x: time.mktime(time.strptime(str(x), '%Y-%m-%d %H:%M:%S'))) outputDF = outputDF.set_index('tickepoch') outputDF.shape # In[64]: # to_dict adds another wrapper dict for each column ts_dict = outputDF.to_dict()['tickvalue'] bit_params = {'precision': 1, 'lag_window_size':1, 'future_window_size':1, 'chunk_size':10} detector = anomaly_detector.AnomalyDetector(ts_dict, algorithm_name='bitmap_detector', algorithm_params=bit_params) anomalies = detector.get_anomalies() print len(anomalies) # In[67]: # anomaly = anomalies[0] # print anomaly # print str(datetime.datetime.fromtimestamp(anomaly.start_timestamp)) # In[8]: