def fit(self, x):
tmp_dict = {}
for i in range(len(x)):
tmp_dict[i] = x[i]
self.cls = anomaly_detector.AnomalyDetector(
tmp_dict,
algorithm_name=ALGO_NAME,
algorithm_params=self.param_dict)
def calculate_anomalies(ts):
"""Takes in a df column of concentrations and returns a list of anomaly scores"""
tsd = ts.to_dict()
detector = anomaly_detector.AnomalyDetector(tsd)
score = detector.get_all_scores()
dfs = pd.DataFrame(score.iteritems())
return dfs[1]
def find_anomalies(data):
"""Run luminol and find anomalies"""
points = {}
i = 0
for point in data:
points[i] = point
i += 1
detector = ad.AnomalyDetector(points, algorithm_name='default_detector')
anomalies = detector.get_anomalies()
return anomalies
def fit(self, X):
"""Fit detector.
Parameters
----------
X : dataframe of shape (n_samples, n_features)
The input samples.
"""
# a=str(ts[:,0])
X = X.to_numpy()
timestamp = np.asarray(X[:, 0].astype(np.datetime64))
pca = IncrementalPCA(n_components=1)
value = np.reshape(pca.fit_transform(X[:, 1:]), -1)
X = pd.Series(value, index=timestamp)
X.index = X.index.map(lambda d: to_epoch(str(d)))
lts = TimeSeries(X.to_dict())
self.ts = timestamp
self.ts_value = value
self.detector = anomaly_detector.AnomalyDetector(lts)
return self
def detect_anomaly(self):
"""
Detect anomalies in the timeseries data for the submetrics specified in the config file. Identified anomalies are
stored in self.anomalies as well as written to .anomalies.csv file to be used by the client charting page. Anomaly
detection uses the luminol library (http://pypi.python.org/pypi/luminol)
"""
if not self.anomaly_detection_metrics or len(self.anomaly_detection_metrics) <= 0:
return
for submetric in self.anomaly_detection_metrics:
csv_file = self.get_csv(submetric)
if naarad.utils.is_valid_file(csv_file):
detector = anomaly_detector.AnomalyDetector(csv_file)
anomalies = detector.get_anomalies()
if len(anomalies) <= 0:
return
self.anomalies[submetric] = anomalies
anomaly_csv_file = os.path.join(self.resource_directory, self.label + '.' + submetric + '.anomalies.csv')
with open(anomaly_csv_file, 'w') as FH:
for anomaly in anomalies:
FH.write(",".join([str(anomaly.anomaly_score), str(anomaly.start_timestamp), str(anomaly.end_timestamp), str(anomaly.exact_timestamp)]))
FH.write('\n')
def luminoldetect():
ts = urllib.unquote(request.args.get('ts_path')[1:])
my_detector = anomaly_detector.AnomalyDetector(ts)
score = my_detector.get_all_scores().values
# Create a new csv file that contains both values and scores.
anom_scores = list()
for i, (timestamp, value) in enumerate(my_detector.time_series.iteritems()):
t_str = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(timestamp / 1000))
anom_scores.append([t_str, value, score[i]])
write_csv(anom_scores, ts.split('/')[-1])
anoms = my_detector.get_anomalies()
result = list()
for anom in anoms:
entry = list()
anom_dict = anom.__dict__
for key in anom_dict:
entry.append(anom_dict[key])
result.append(entry)
return jsonify(anom=result, anom_score=anom_scores)
def real_anomaly_detection(samples, smoothing_factor):
ts = TimeSeries(samples)
detector = anomaly_detector.AnomalyDetector(
ts,
algorithm_name="derivative_detector",
algorithm_params={
'smoothing_factor':
smoothing_factor # smoothing factor used to compute exponential moving averages
})
anomalies = detector.get_anomalies()
anomaly_times = []
for a in anomalies:
start = a.get_time_window()[0]
end = a.get_time_window()[1]
while (start <= end):
anomaly_times.append(start)
start += 1
print anomaly_times
return anomaly_times
def run_anomaly(metric_name, start_time, stop_time, push_ready=True):
# metric_name="devbeefydocker01.aunalytics.com.vmstat.cpu.user"
# from_time="-24h"
# to_time="now"
temp = get_graphite_data(metric_name, start_time, stop_time)
detector = anomaly_detector.AnomalyDetector(temp)
anomalies = detector.get_anomalies()
# print (anomalies)
anomalies_windows = []
for ac in anomalies:
# print(ac.get_time_window())
anomalies_windows.append(ac.get_time_window())
if push_ready is True:
ra = []
for c in anomalies_windows:
if int(c[0]) > (time.time() - 3600):
for r in range(c[0], c[1], 60):
ra.append(metric_name + "_anomaly 1 " + str(r) + "\n")
else:
ra = anomalies_windows
return temp, ra
from datetime import datetime
from typing import List
import pandas as pd
from luminol import anomaly_detector
from luminol.modules.anomaly import Anomaly
sold_items = 'data/sold-items-de-summer-2018.csv'
pd.read_csv(sold_items)
detector = anomaly_detector.AnomalyDetector(time_series=sold_items)
anomalies: List[Anomaly] = detector.get_anomalies()
for a in anomalies:
dd = datetime.utcfromtimestamp(
a.exact_timestamp).strftime('%Y-%m-%d %H:%M:%S')
print(f'{dd}: {a.anomaly_score}')
# In[40]:
# output = output.set_index(0)
outputDF['tickepoch'] = outputDF['tickdate'].apply(
lambda x: time.mktime(time.strptime(str(x), '%Y-%m-%d %H:%M:%S')))
outputDF = outputDF.set_index('tickepoch')
# In[41]:
outputDF.head()
# In[42]:
# to_dict adds another wrapper dict for each column
ts_dict = outputDF.to_dict()['tickvalue']
detector = anomaly_detector.AnomalyDetector(ts_dict)
anomalies = detector.get_anomalies()
# In[43]:
anomaly = anomalies[0]
print anomaly
print str(datetime.datetime.fromtimestamp(anomaly.start_timestamp))
# In[44]:
import matplotlib.pyplot as plt
# In[47]:
anomaly_list = []
model.fit_predict(X)
lof = model.negative_outlier_factor_
plt.plot(lof / min(lof), color="orange")
plt.title("LocalOutlierFactor On Feature 1")
plt.plot(Y_f)
plt.show()
#LocalOutlierFactor başarılı
#%% LUMINOL
from luminol import anomaly_detector
df = df.reset_index()
df["timestamp"] = (df["timestamp"].astype('uint64') / 1e6).astype('uint32')
df.set_index("timestamp")
a = df.drop(columns=["Feature2"]).to_dict("series")["Feature1"].to_dict()
detector = anomaly_detector.AnomalyDetector(a,
algorithm_name="exp_avg_detector",
score_threshold=0.2)
anomalies = detector.get_anomalies()
time_periods = []
values = []
for key in (a):
time_periods.append(key)
values.append(a[key])
plt.plot(time_periods, values)
anom_times = []
anom_values = []
for anom in anomalies:
anom_times.append(np.arange(anom.start_timestamp, anom.end_timestamp))
anom_values.append(anom.anomaly_score)
# In[58]:
# output = output.set_index(0)
outputDF['tickepoch'] = outputDF['tickdate'].apply(lambda x: time.mktime(time.strptime(str(x), '%Y-%m-%d %H:%M:%S')))
outputDF = outputDF.set_index('tickepoch')
outputDF.shape
# In[64]:
# to_dict adds another wrapper dict for each column
ts_dict = outputDF.to_dict()['tickvalue']
bit_params = {'precision': 1, 'lag_window_size':1, 'future_window_size':1, 'chunk_size':10}
detector = anomaly_detector.AnomalyDetector(ts_dict, algorithm_name='bitmap_detector', algorithm_params=bit_params)
anomalies = detector.get_anomalies()
print len(anomalies)
# In[67]:
# anomaly = anomalies[0]
# print anomaly
# print str(datetime.datetime.fromtimestamp(anomaly.start_timestamp))
# In[8]:
