def getTFfromTopK(file_name, height, duration, Thresholds):
result_txt = [
float(i) for i in getResult(
file_name=file_name, height=height, duration=duration)
]
lc_data = ut_lc.getDataFromFile(fileName=file_name,
height=height,
duration=duration)
return_result = []
ans_list = lc_data["ansList"]
starting_point = lc_data["ansList"][0]
for Threshold in Thresholds:
TP = set()
removeTran = set()
# FP = 0
alarm_index = set()
for index, value in enumerate(result_txt):
if value > Threshold:
if checkTP(indexAlarm=index, starting_point=starting_point):
TP.add(index)
elif len(list(set(index_list[index]) & set(ans_list))):
removeTran.add(index)
alarm_index.add(index)
# TP = len(list(result_index & set(ans_list)))
FP = alarm_index.difference(TP.copy().union(removeTran))
return_result.append({"TP": len(TP), "FP": len(FP)})
return return_result
def getTFfromTopK(file_name, height, duration, Thresholds):
result_txt = [
float(i) for i in getResult(
file_name=file_name, height=height, duration=duration)
]
lc_data = ut_lc.getDataFromFile(fileName=file_name,
height=height,
duration=duration)
return_result = []
ans_list = lc_data["ansList"]
for Threshold in Thresholds:
# TP = 0
# FP = 0
result_index = set()
for index, value in enumerate(result_txt):
if value > Threshold:
alarm_list = index_list[index]
for i in alarm_list:
result_index.add(i)
TP = len(list(result_index & set(ans_list)))
if (TP > duration):
print("I{} D{} file{}".format(height, duration, file_name))
FP = len(set(result_index)) - TP
return_result.append({"TP": TP, "FP": FP})
return return_result
def getTF(file_name, height, duration, Thresholds):
rows_txt = getResult(file_name=file_name, height=height, duration=duration)
lc_data = ut_lc.getDataFromFile(fileName=file_name,
height=height,
duration=duration)
return_result = []
ans_list = [
*range(lc_data["ansList"][0] - 3, lc_data["ansList"][0] + 3)
] + [*range(lc_data["ansList"][-1] - 3, lc_data["ansList"][-1] + 3)]
trans_list = lc_data["ansList"] + ans_list
for Threshold in Thresholds:
# TP = 0
# FP = 0
result_index = set()
for index, row in enumerate(rows_txt):
if row['kstar'] > Threshold:
result_index.add(row['cur_index'] - 3)
result_index.add(row['cur_index'] - 2)
result_index.add(row['cur_index'] - 1)
result_index.add(row['cur_index'])
result_index.add(row['cur_index'] + 1)
result_index.add(row['cur_index'] + 2)
# alarm_list = index_list[index]
# for i in alarm_list:
# result_index.add(i)
TP = len(list(result_index & set(ans_list)))
# del index in trans
FP = len(list(result_index)) - len(
list(result_index & set(trans_list)))
return_result.append({"TP": TP, "FP": FP})
return return_result
def showGraph(file_name, height, duration):
result_list = [float(i) for i in getResult(file_name=file_name, height=height, duration=duration)]
lc_data = ut_lc.getDataFromFile(fileName=file_name, height=height, duration=duration)
fig, axes = plt.subplots(2, 1, sharex=True, figsize=(20, 7))
axes[0].plot(lc_data["timestamp"], lc_data["instances"])
axes[0].set_title('Raw Data', size=22)
axes[1].plot(lc_data["timestamp"], result_list)
axes[1].set_title('Matrix Profile', size=22)
plt.show()
plt.clf()
def getTFfromThreshold(file_name, height, duration, thresholds):
result_txt = [float(i) for i in getResult(file_name=file_name, height=height, duration=duration)]
# result_txt = np.array(result_txt)
# index_array = list(numpy.argsort(-result_txt)[:len(result_txt)])
lc_data = ut_lc.getDataFromFile(fileName=file_name, height=height, duration=duration)
return_result = []
ans_list = lc_data["ansList"]
for threshold in thresholds:
# TP = 0
# FP = 0
result_index = []
for index, value in enumerate(result_txt):
if value > threshold:
result_index.append(index)
TP = len(list(set(result_index) & set(ans_list)))
FP = len(set(result_index)) - TP
return_result.append({"TP": TP, "FP": FP})
return return_result
def getTFfromTopK(file_name,height,duration,Thresholds):
result_txt= [float(i) for i in getResult(file_name=file_name, height=height, duration=duration)]
lc_data = ut_lc.getDataFromFile(fileName=file_name, height=height, duration=duration)
return_result = []
ans_list = lc_data["ansList"]
for Threshold in Thresholds:
# TP = 0
# FP = 0
TP = set()
result_index = set()
for index, value in enumerate(result_txt):
if value > Threshold:
if checkTP(alarm_index=index,ans_list=ans_list):
TP.add(index)
# alarm_list = index_list[index]
# for i in alarm_list:
result_index.add(index)
FP = len(result_index) - len(TP)
return_result.append({"TP": len(TP), "FP": FP})
return return_result
def getTFfromTopK(
file_name,
height,
duration,
Ks,
range_size=20,
):
result_txt = [
float(i) for i in getResult(
file_name=file_name, height=height, duration=duration)
]
result_txt = np.array(result_txt)
index_array = list(numpy.argsort(-result_txt)[:len(result_txt)])
lc_data = ut_lc.getDataFromFile(fileName=file_name,
height=height,
duration=duration)
# result_list = []
# while len(index_array)>0:
# value = index_array[0]
# result_list.append(value)
# remove_list = [*range(value, value + range_size)]
# for remove_instance in remove_list:
# if (remove_instance in index_array):
# index_array.remove(remove_instance)
return_result = []
ans_list = lc_data["ansList"]
for kth in Ks:
# TP = 0
# FP = 0
result_kth = index_array[:kth]
TP = len(list(set(result_kth) & set(ans_list)))
FP = len(result_kth) - TP
return_result.append({"TP": TP, "FP": FP})
if TP > duration:
print(file_name)
return return_result
import pywt
import numpy as np
import matplotlib.pyplot as plt
import utility.utility_light_curve as ut_lc
heights = [1,3]
durations = [60,100,200,500]
i = 38
if __name__ == '__main__':
for height in heights:
for duration in durations:
listFiles = ut_lc.getListLight(height=height, duration=duration)
for lc_file in listFiles[i:i+1]:
data = ut_lc.getDataFromFile(fileName=lc_file,
height=height, duration=duration)
fig, axs = plt.subplots(2, 1)
axs[0].plot(data["timestamp"], data["instances"])
coef, freqs=pywt.cwt(data["instances"],np.arange(1,129),'gaus1')
# plt.subplot(2, 1, 2)
axs[1].matshow(coef) # doctest: +SKIP
axs[1].autoscale()
plt.show() # doctest: +SKIP
plt.clf()
avg_codisp[index] = 0
avg_codisp[index] += tree.codisp(index) / num_trees
return avg_codisp
if __name__ == '__main__':
for height in heights:
for duration in durations:
# folder_name = "I{}_L{}".format(height,duration)
folder_name = "temp"
ut.checkFolderandCreate(folder_name)
listFile = ut_lc.getListLight(height=height, duration=duration)
# for file_name in listFile[340:]:
for file_name in listFile[0:1]:
lc_data = ut_lc.getDataFromFile(fileName=file_name,
height=height,
duration=duration)
# corePlot = sketchDyBinService(windowSize=windowSize, initialBin=initialBin, isOnline=True)
# sketchInstances = corePlot.sketchMode(instances=lc_data['instances'])
# windowList = corePlot.getWindow()
# binInstances = []
# for bin in windowList:
# binInstances.append(bin.get_representation())
avg_codisp = get_avgcodisp(lc_data['instances'])
# codisp_normalization = []
# for index in range(shingle_size-1):
# bin = windowList[index]
# n = bin.get_number_instance()
def computeCompressgraph(instances, windowSize):
core = com_service(inputList=instances)
dictResult = core.TWINcomputeCompression(windowSize)
return dictResult["varaince_ratio"], dictResult["compression_ratio"]
if __name__ == '__main__':
windowSizeList = [2000, 1000]
heightList = ['1', '3']
durationList = ['60', '100', '200', '500']
for windowSize in windowSizeList:
for height in heightList:
for duration in durationList:
fileList = ut_light.getListFile(height=height,
duration=duration)
for fileName in fileList:
lightData = ut_light.getDataFromFile(fileName=fileName,
height=height,
duration=duration)
var_ratio, com_ratio = computeCompressgraph(
lightData["instances"], windowSize)
row = [
fileName, height, duration, windowSize, var_ratio,
com_ratio
]
print(row)
ut.add_to_csv(row=row, csv_name="comratio.csv")
import rrcf
import utility.utility_light_curve as ut_lc
from webService.backend.coreSketchDyBinService import sketchDyBinService
import utility.utility_genData as ut_gen
# Generate data
height = 5
duration = 100
num_trees = 40
shingle_size = 48
tree_size = 20
listFile = ut_lc.getListLight(height=height, duration=duration)
lc_data = ut_lc.getDataFromFile(fileName=listFile[101],
height=height,
duration=duration)
# sketchInstances = ut_gen.genFixBin(binSize=binSize,instances=lc_data['instances'])
# notEx = ut_gen.genFixBin(binSize=binSize,instances=lc_data['instances'],isExtract=False)
#
# Create a forest of empty trees
forest = []
for _ in range(num_trees):
tree = rrcf.RCTree()
forest.append(tree)
# Use the "shingle" generator to create rolling window
points = rrcf.shingle(lc_data['instances'], size=shingle_size)