def start(_socketio):
global co
global socketio
global model_illum
global model_humid
global model_temp
global data_illum
global data_humid
global data_temp
socketio = _socketio
model_illum = banpei.SST(w=30)
model_humid = banpei.SST(w=30)
model_temp = banpei.SST(w=30)
data_illum = []
data_humid = []
data_temp = []
configfile = open('./config.json', 'r')
global config
config = json.load(configfile)
# setup mongodb connection
client = MongoClient(config['mongo']['ip'], config['mongo']['port'])
db = client.sensordb
co = db.sensordata
for i in co.find({
'topic': 'sensor/04016777/Illumination'
}).sort('timestamp').limit(1):
data_illum.insert(0, i['value'])
print(i['value'])
for i in co.find({
'topic': 'sensor/04016897/Humidity'
}).sort('timestamp').limit(1):
data_humid.insert(0, i['value'])
print(i['value'])
for i in co.find({
'topic': 'sensor/04016897/Temperature'
}).sort('timestamp').limit(1):
data_temp.insert(0, i['value'])
print(i['value'])
# setup mqtt connection
th = threading.Thread(target=mqtt_connect, name='th', args=())
th.setDaemon(True)
th.start()
def start(_socketio):
global socketio
global p
global stream
global model
global rmsArray
configfile = open('./config.json', 'r')
config = json.load(configfile)
global config_audio
config_audio = config['audio']
# setup mongodb connection
global co
client = MongoClient(config['mongo']['ip'], config['mongo']['port'])
db = client.sensordb
co = db.sensordata
socketio = _socketio
p = pyaudio.PyAudio()
model = banpei.SST(w=30)
rmsArray = []
th = threading.Thread(target=getstream, name='th', args=())
th.setDaemon(True)
th.start()
def anomaly_detection(raw_data,fname,n_feat=feat):
vanilla_peaks={} # Using SST getting just index
for ind in range(n_feat):
temp_data=raw_data[:,ind] # Going per feature/column
model=banpei.SST(w=50)
anomaly=model.detect(temp_data,is_lanczos=True) # Changepoint scores using SST
peak=sig.find_peaks(anomaly) # Data peaks- naive
vanilla_peaks[ind]=[peak[0],anomaly] # As dict --> Index values, actual scores
print("\t\t\t Anomaly detection Index {} done.".format(ind))
vanilla_fname=fname+'_vanillasst.csv'
with open(vanilla_fname,'w') as f:
writer=csv.writer(f)
for k,v in vanilla_peaks.items():
writer.writerow([k,v])
f.close()
print("\t\t Saved anomaly detection")
def analyst(args):
ALL_CPU_ID = -2
ALL_SYSTEM_NAME = 'all'
data = pd.read_csv(args.path)
if args.cpu_id != ALL_CPU_ID:
data = data[data['cpu_id'] == args.cpu_id]
if args.system_name != ALL_SYSTEM_NAME:
data = data[data['system_name'] == args.system_name]
y = data[args.target]
model = banpei.SST(w=50)
results = model.detect(y, is_lanczos=True)
start = args.start
length = args.length
total_length = len(y)
while True:
end = start + length
end = end if end < total_length else total_length
origin = y.iloc[start:end]
result = results[start:end]
x = range(start, end)
plt.subplot(211)
plt.plot(x, origin)
plt.subplot(212)
plt.plot(x, result)
name = args.system_name + '_' + str(
args.cpu_id) + '_' + str(start) + '_' + str(end) + '.png'
plt.savefig('analyst/' + name)
plt.close()
start = end
if end == total_length:
break
-83, -82, -81, -78, -77, -75, -73, -70, -68, -65, -61, -59, -55, -52, -48,
-45, -41, -37, -33, -29, -25, -21, -16, -12, -8, -3, 0, 4, 8, 13, 17, 21,
25, 30, 33, 38, 41, 45, 50, 53, 56, 60, 63, 66, 69, 71, 74, 76, 78, 80, 81,
84, 85, 86, 87, 87, 89, 89, 88, 89, 89, 88, 87, 86, 85, 84, 82, 81, 78, 76,
74, 71, 69, 66, 63, 60, 57, 53, 51, 46, 43, 39, 35, 30, 26, 23, 18, 14, 10,
6, 0, -2, -6, -11, -15, -19, -24, -28, -32, -36, -40, -43, -47, -51, -54,
-58, -61, -64, -67, -69, -72, -74, -76, -79, -81, -82, -83, -84, -86, -86,
-87, -87, -87, -87, -87, -86, -85, -84, -84, -82, -81, -79, -76, -75, -72,
-70, -67, -64, -61, -59, -55, -52, -48, -45, -41, -37, -33, -29, -24, -21,
-16, -12, -7, -3, 0, 4, 8, 12, 17, 22, 26, 30, 33, 38, 42, 45, 49, 53, 56,
59, 62, 65, 68, 71, 73, 76, 79, 80, 81, 84, 84, 86, 87, 88, 88, 89, 89, 89,
89, 88, 87, 86, 85, 84, 82, 80, 78, 76, 74, 71, 69, 67, 63, 60, 58, 53, 50,
46, 43, 38, 35, 30, 26, 22, 18, 14, 10, 5, 1, -2, -6, -11, -15, -19, -24,
-28, -32, -36, -40, -44, -47, -51, -54, -58, -61, -64, -67, -69, -72, -75,
-77, -78, -80, -82, -84, -84, -85, -86, -86, -87, -87, -87, -86, -86, -85,
-84, -84, -82, -80, -79, -77, -75, -73, -70, -67, -65, -61, -59, -55, -52,
-48, -45, -41, -38, -33, -29, -25, -21, -17, -12, -8, -4, 0, 4, 8, 13, 18,
22, 25, 30, 34, 38, 42, 45, 49, 53, 55, 60, 62, 66, 68, 71, 74, 76, 78, 81,
82
])
pylab.subplot(211)
fig = pylab.plot(data, color="green", lw=2)
model = banpei.SST(w=128)
results = model.detect(data)
print('Find the Change Point Index is %d' %
(results.tolist().index(max(results))))
pylab.subplot(212)
fig = pylab.plot(results, color="red", lw=2)
def use_sst(time_series):
#
# OFFSET = 0
# WIN = 2
# model = banpei.SST(w=10)
#
# # #timeseries是性能指标序列
# # time_series = []
# # for row in time_series_set:
# # time_series.append(row[1])
# # # time是性能指标对应的全时间序列
# # time = []
# # for row in time_series_set:
# # time.append(row[0])
# # print("time", time_series)
# score = model.detect(time_series)
# #print(score)
# lst = score.tolist()
#
# #print("score", lst)
#
# iter = len(lst) - 1
# while iter > (OFFSET - 1):
# lst[iter] = lst[iter - OFFSET]
# iter -= 1
# for k in range(0, 32):
# lst[k] = 0.0
#
#
#
#
# # 用t-score给sst结果判断正负
# # for i in range(2, (len(lst) - 2)):
# # front = time_series[(i - WIN): i: 1]
# # rear = time_series[i: (i + WIN): 1]
# # t_score = t_test(front, rear)
# # if t_score < 0:
# # lst[i] = -lst[i]
#
#
#
# # 极值
# # x = np.array(lst)
# # avgLst = average_num(lst)
# # npMaxList = argrelextrema(x, np.greater)[0]
# # maxlst = npMaxList.tolist()
# # maxList = []
# # for max in maxlst:
# # if lst[max] > avgLst*5:
# # maxList.append(max)
# # print(maxList)
# # tScoreSeries = []
# # for item in maxList:
# # print(time[item])
# # front = time_series[(item - WIN):item:1]
# # rear = time_series[item:(item + WIN):1]
# # t_score = t_test(front, rear, WIN)
# # tScoreSeries.append([item, t_score])
# # print(tScoreSeries)
# #
# # temp = tScoreSeries[0]
# # tempFront = 0
# # tempRear = 0
# # for item in tScoreSeries[1:]:
# # if temp[-1] * item[-1] < 0:
# # if (temp[-1] > 0) and (item[-1] < 0):
# # continue
# # else:
# # if (temp[-1] < 0) and (item[-1] > 0):
# # tempRear = (temp[0] + item[0]) / 2
# # for i in range(tempFront, tempRear):
# # sst_list[i][-1] = - sst_list[i][-1]
# # tempFront = tempRear
# # temp = item
# # else:
# # continue
# #
# # lstSeries = []
# # for item in sst_list:
# # lstSeries.append(item[-1])
# # print(lstSeries)
#
# # sst_list = []
# # for i in range(len(time)):
# # row = [time[i], lst[i]]
# # sst_list.append(row)
model = banpei.SST(w=50)
results = model.detect(time_series)
return results
import pandas as pd
import banpei
import numpy as np
pathToProcess = "/opt/sonarg/sonarw/agg_out/"
fileToProcess = "TS_Query_10.1.csv"
raw_data = pd.read_csv(pathToProcess + 'DailyExtract_' + fileToProcess)
data = raw_data['Records Affected']
print('Nbr of recs to Process : ', len(data))
model = banpei.SST(w=30)
results = model.detect(data, is_lanczos=True)
anomalies = np.array([data, results])
print(anomalies.shape)
rAnomalies = np.reshape(anomalies, (len(results), 2))
print(rAnomalies.shape)
print(rAnomalies)
# print(type(results),len(results))
# print(results)
# numpy.savetxt(pathToProcess + 'Banpei_' + fileToProcess, results)
def load_sst_model(w, m=2, k=None, L=None):
return banpei.SST(w, m, k, L)
def use_sst(time_series_set):
OFFSET = 28
WIN = 2
model = banpei.SST(w=50)
#timeseries是性能指标序列
time_series = []
for row in time_series_set:
time_series.append(row[1])
# time是性能指标对应的全时间序列
time = []
for row in time_series_set:
time.append(row[0])
# print("time", time_series)
score = model.detect(time_series)
#print(score)
lst = score.tolist()
#print("score", lst)
iter = len(lst) - 1
while iter > (OFFSET - 1):
lst[iter] = lst[iter - OFFSET]
iter -= 1
for k in range(0, 32):
lst[k] = 0.0
# 用t-score给sst结果判断正负
for i in range(2, (len(lst) - 2)):
front = time_series[(i - WIN):i:1]
rear = time_series[i:(i + WIN):1]
t_score = t_test(front, rear)
if t_score < 0:
lst[i] = -lst[i]
# 极值
# x = np.array(lst)
# avgLst = average_num(lst)
# npMaxList = argrelextrema(x, np.greater)[0]
# maxlst = npMaxList.tolist()
# maxList = []
# for max in maxlst:
# if lst[max] > avgLst*5:
# maxList.append(max)
# print(maxList)
# tScoreSeries = []
# for item in maxList:
# print(time[item])
# front = time_series[(item - WIN):item:1]
# rear = time_series[item:(item + WIN):1]
# t_score = t_test(front, rear, WIN)
# tScoreSeries.append([item, t_score])
# print(tScoreSeries)
#
# temp = tScoreSeries[0]
# tempFront = 0
# tempRear = 0
# for item in tScoreSeries[1:]:
# if temp[-1] * item[-1] < 0:
# if (temp[-1] > 0) and (item[-1] < 0):
# continue
# else:
# if (temp[-1] < 0) and (item[-1] > 0):
# tempRear = (temp[0] + item[0]) / 2
# for i in range(tempFront, tempRear):
# sst_list[i][-1] = - sst_list[i][-1]
# tempFront = tempRear
# temp = item
# else:
# continue
#
# lstSeries = []
# for item in sst_list:
# lstSeries.append(item[-1])
# print(lstSeries)
sst_list = []
for i in range(len(time)):
row = [time[i], lst[i]]
sst_list.append(row)
return sst_list