def getTFfromTopK(file_name, height, duration, Thresholds): result_txt = [ float(i) for i in getResult( file_name=file_name, height=height, duration=duration) ] lc_data = ut_lc.getDataFromFile(fileName=file_name, height=height, duration=duration) return_result = [] ans_list = lc_data["ansList"] starting_point = lc_data["ansList"][0] for Threshold in Thresholds: TP = set() removeTran = set() # FP = 0 alarm_index = set() for index, value in enumerate(result_txt): if value > Threshold: if checkTP(indexAlarm=index, starting_point=starting_point): TP.add(index) elif len(list(set(index_list[index]) & set(ans_list))): removeTran.add(index) alarm_index.add(index) # TP = len(list(result_index & set(ans_list))) FP = alarm_index.difference(TP.copy().union(removeTran)) return_result.append({"TP": len(TP), "FP": len(FP)}) return return_result
def getTFfromTopK(file_name, height, duration, Thresholds): result_txt = [ float(i) for i in getResult( file_name=file_name, height=height, duration=duration) ] lc_data = ut_lc.getDataFromFile(fileName=file_name, height=height, duration=duration) return_result = [] ans_list = lc_data["ansList"] for Threshold in Thresholds: # TP = 0 # FP = 0 result_index = set() for index, value in enumerate(result_txt): if value > Threshold: alarm_list = index_list[index] for i in alarm_list: result_index.add(i) TP = len(list(result_index & set(ans_list))) if (TP > duration): print("I{} D{} file{}".format(height, duration, file_name)) FP = len(set(result_index)) - TP return_result.append({"TP": TP, "FP": FP}) return return_result
def getTF(file_name, height, duration, Thresholds): rows_txt = getResult(file_name=file_name, height=height, duration=duration) lc_data = ut_lc.getDataFromFile(fileName=file_name, height=height, duration=duration) return_result = [] ans_list = [ *range(lc_data["ansList"][0] - 3, lc_data["ansList"][0] + 3) ] + [*range(lc_data["ansList"][-1] - 3, lc_data["ansList"][-1] + 3)] trans_list = lc_data["ansList"] + ans_list for Threshold in Thresholds: # TP = 0 # FP = 0 result_index = set() for index, row in enumerate(rows_txt): if row['kstar'] > Threshold: result_index.add(row['cur_index'] - 3) result_index.add(row['cur_index'] - 2) result_index.add(row['cur_index'] - 1) result_index.add(row['cur_index']) result_index.add(row['cur_index'] + 1) result_index.add(row['cur_index'] + 2) # alarm_list = index_list[index] # for i in alarm_list: # result_index.add(i) TP = len(list(result_index & set(ans_list))) # del index in trans FP = len(list(result_index)) - len( list(result_index & set(trans_list))) return_result.append({"TP": TP, "FP": FP}) return return_result
def showGraph(file_name, height, duration): result_list = [float(i) for i in getResult(file_name=file_name, height=height, duration=duration)] lc_data = ut_lc.getDataFromFile(fileName=file_name, height=height, duration=duration) fig, axes = plt.subplots(2, 1, sharex=True, figsize=(20, 7)) axes[0].plot(lc_data["timestamp"], lc_data["instances"]) axes[0].set_title('Raw Data', size=22) axes[1].plot(lc_data["timestamp"], result_list) axes[1].set_title('Matrix Profile', size=22) plt.show() plt.clf()
def getTFfromThreshold(file_name, height, duration, thresholds): result_txt = [float(i) for i in getResult(file_name=file_name, height=height, duration=duration)] # result_txt = np.array(result_txt) # index_array = list(numpy.argsort(-result_txt)[:len(result_txt)]) lc_data = ut_lc.getDataFromFile(fileName=file_name, height=height, duration=duration) return_result = [] ans_list = lc_data["ansList"] for threshold in thresholds: # TP = 0 # FP = 0 result_index = [] for index, value in enumerate(result_txt): if value > threshold: result_index.append(index) TP = len(list(set(result_index) & set(ans_list))) FP = len(set(result_index)) - TP return_result.append({"TP": TP, "FP": FP}) return return_result
def getTFfromTopK(file_name,height,duration,Thresholds): result_txt= [float(i) for i in getResult(file_name=file_name, height=height, duration=duration)] lc_data = ut_lc.getDataFromFile(fileName=file_name, height=height, duration=duration) return_result = [] ans_list = lc_data["ansList"] for Threshold in Thresholds: # TP = 0 # FP = 0 TP = set() result_index = set() for index, value in enumerate(result_txt): if value > Threshold: if checkTP(alarm_index=index,ans_list=ans_list): TP.add(index) # alarm_list = index_list[index] # for i in alarm_list: result_index.add(index) FP = len(result_index) - len(TP) return_result.append({"TP": len(TP), "FP": FP}) return return_result
def getTFfromTopK( file_name, height, duration, Ks, range_size=20, ): result_txt = [ float(i) for i in getResult( file_name=file_name, height=height, duration=duration) ] result_txt = np.array(result_txt) index_array = list(numpy.argsort(-result_txt)[:len(result_txt)]) lc_data = ut_lc.getDataFromFile(fileName=file_name, height=height, duration=duration) # result_list = [] # while len(index_array)>0: # value = index_array[0] # result_list.append(value) # remove_list = [*range(value, value + range_size)] # for remove_instance in remove_list: # if (remove_instance in index_array): # index_array.remove(remove_instance) return_result = [] ans_list = lc_data["ansList"] for kth in Ks: # TP = 0 # FP = 0 result_kth = index_array[:kth] TP = len(list(set(result_kth) & set(ans_list))) FP = len(result_kth) - TP return_result.append({"TP": TP, "FP": FP}) if TP > duration: print(file_name) return return_result
import pywt import numpy as np import matplotlib.pyplot as plt import utility.utility_light_curve as ut_lc heights = [1,3] durations = [60,100,200,500] i = 38 if __name__ == '__main__': for height in heights: for duration in durations: listFiles = ut_lc.getListLight(height=height, duration=duration) for lc_file in listFiles[i:i+1]: data = ut_lc.getDataFromFile(fileName=lc_file, height=height, duration=duration) fig, axs = plt.subplots(2, 1) axs[0].plot(data["timestamp"], data["instances"]) coef, freqs=pywt.cwt(data["instances"],np.arange(1,129),'gaus1') # plt.subplot(2, 1, 2) axs[1].matshow(coef) # doctest: +SKIP axs[1].autoscale() plt.show() # doctest: +SKIP plt.clf()
avg_codisp[index] = 0 avg_codisp[index] += tree.codisp(index) / num_trees return avg_codisp if __name__ == '__main__': for height in heights: for duration in durations: # folder_name = "I{}_L{}".format(height,duration) folder_name = "temp" ut.checkFolderandCreate(folder_name) listFile = ut_lc.getListLight(height=height, duration=duration) # for file_name in listFile[340:]: for file_name in listFile[0:1]: lc_data = ut_lc.getDataFromFile(fileName=file_name, height=height, duration=duration) # corePlot = sketchDyBinService(windowSize=windowSize, initialBin=initialBin, isOnline=True) # sketchInstances = corePlot.sketchMode(instances=lc_data['instances']) # windowList = corePlot.getWindow() # binInstances = [] # for bin in windowList: # binInstances.append(bin.get_representation()) avg_codisp = get_avgcodisp(lc_data['instances']) # codisp_normalization = [] # for index in range(shingle_size-1): # bin = windowList[index] # n = bin.get_number_instance()
def computeCompressgraph(instances, windowSize): core = com_service(inputList=instances) dictResult = core.TWINcomputeCompression(windowSize) return dictResult["varaince_ratio"], dictResult["compression_ratio"] if __name__ == '__main__': windowSizeList = [2000, 1000] heightList = ['1', '3'] durationList = ['60', '100', '200', '500'] for windowSize in windowSizeList: for height in heightList: for duration in durationList: fileList = ut_light.getListFile(height=height, duration=duration) for fileName in fileList: lightData = ut_light.getDataFromFile(fileName=fileName, height=height, duration=duration) var_ratio, com_ratio = computeCompressgraph( lightData["instances"], windowSize) row = [ fileName, height, duration, windowSize, var_ratio, com_ratio ] print(row) ut.add_to_csv(row=row, csv_name="comratio.csv")
import rrcf import utility.utility_light_curve as ut_lc from webService.backend.coreSketchDyBinService import sketchDyBinService import utility.utility_genData as ut_gen # Generate data height = 5 duration = 100 num_trees = 40 shingle_size = 48 tree_size = 20 listFile = ut_lc.getListLight(height=height, duration=duration) lc_data = ut_lc.getDataFromFile(fileName=listFile[101], height=height, duration=duration) # sketchInstances = ut_gen.genFixBin(binSize=binSize,instances=lc_data['instances']) # notEx = ut_gen.genFixBin(binSize=binSize,instances=lc_data['instances'],isExtract=False) # # Create a forest of empty trees forest = [] for _ in range(num_trees): tree = rrcf.RCTree() forest.append(tree) # Use the "shingle" generator to create rolling window points = rrcf.shingle(lc_data['instances'], size=shingle_size)