def cal_sk_bin(data, input_k,input_bin, file_format, L,I): # input_k = 2 data.clear_change_points() detector = light_detector() main_path = 'ks_bin' result_text = [] ut.checkFolderandCreate(main_path) pickle_file = '{}\\cheb{}\\{}_L{}_I{}.pkl'.format(main_path,Cheb_win ,file_format, L,I) if not os.path.exists(pickle_file): detector.set_parameter(input_k=input_k, input_min_winsize=Cheb_win, input_max_winsize=Cheb_win, input_bin=input_bin) for i, file_name in enumerate(data.get_files_test()): instances = data.get_dataset_test(i) change_points = detector.get_changepoint_skmethod(instances) data.set_change_points(change_points) with open(pickle_file, 'wb') as output: pickle.dump(data, output) else: with open(pickle_file, 'rb') as pickle_input: data = pickle.load(pickle_input) # true_alerts, false_alerts, tran_found = data.count_TF_tranfound_withBin(Bin=input_bin) # result_text.append("SK Bin = {} and K = {}".format(input_bin, input_k)) # result_text.append("True alert = {} false_alert = {} tran_found = {}".format(true_alerts, false_alerts, tran_found)) return data,result_text
def cal_CANDAR_bin(data, input_k, input_bin, file_format, L,I,is_k = True): algorithm = "" data.clear_change_points() detector = light_detector() if is_k: main_path = 'CANDAR_bin' else: main_path = 'CANDAR_bin_E' result_text = [] ut.checkFolderandCreate(main_path) pickle_file = '{}\\cheb{}\\{}_L{}_I{}.pkl'.format(main_path,Cheb_win, file_format, L,I) if not os.path.exists(pickle_file): detector.set_parameter(input_k=input_k, input_min_winsize=Cheb_win, input_max_winsize=Cheb_win, input_bin=input_bin,is_k=is_k) for i, file_name in enumerate(data.get_files_test()): instances = data.get_dataset_test(i) change_points, mean_list = detector.get_changepoint_BinCheb(instances) data.set_change_points(change_points) with open(pickle_file, 'wb') as output: pickle.dump(data, output) else: with open(pickle_file, 'rb') as pickle_input: data = pickle.load(pickle_input) true_alerts, false_alerts, tran_found = data.count_TF_tranfound_withBin(Bin=input_bin) # result_text.append("BIN CANDAR Bin = {} and K = {} is_k = {}".format(input_bin, k,str(is_k))) # result_text.append("True alert = {} false_alert = {} tran_found = {}".format(true_alerts, false_alerts, tran_found)) return data ,result_text
def save_tran_overTime_dy(data_list, name_list, tran=False): periodSize = 200 for index, data in enumerate(data_list[0:200]): for index_tran in range(data.get_len_tran()): # chebE_core = chebyshev_Dy(e=e) skCore = skDetection(k=3) title = data.get_file_test_name(index=index_tran) instances = data.get_dataset_test(int(index_tran)) tran_st = "" tran_end = "" if tran: st_end_list = data.get_dataset_answer_st_ed(index_tran)[0] # for st_end in st_end_list: tran_st = st_end_list[0] tran_end = st_end_list[1] changePoints = [] full_pic_path = "{}{}_{}_bin{}_win{}".format\ (pic_path, name_list[index],typeSketc,bin_size,WINDOW_SIZE) ut.checkFolderandCreate(full_pic_path) coreSketch = typeOfSketch(typeName=typeSketc) StopTimeStampList = ut.genTimeStampStopList(instances=instances, periodSize=periodSize) count = 0 # hi-net for i, instance in enumerate(instances): if i in StopTimeStampList: target_file = "{}\\{}_{}.png".format( full_pic_path, "{:02d}".format(index_tran), i) window = coreSketch.getWindow() ############ Hinet if i < 20000: min_xlim = 0 else: min_xlim = count * periodSize - 10000 max_xlim = (count + 1) * periodSize # ut.plot_window_Dybin(window, target_file, title=title, # changePointList=changePoints, tran_st=tran_st, # tran_end=tran_end, min_xlim =min_xlim ,max_xlim=max_xlim) count = count + 1 ############ Hinet # ut.plot_window_Dybin(window, target_file, title=title, # changePointList=changePoints, tran_st=tran_st, # tran_end=tran_end) if i == len(instances) - 1: coreSketch.cutOff_element() target_file = "{}\\{}_{}.png".format( full_pic_path, "{:02d}".format(index_tran), i) window = coreSketch.getWindow() # ut.plot_window_Dybin(window, target_file, title=title, # changePointList=changePoints,tran_st=tran_st, # tran_end=tran_end) ut.plot_window_Dybin(window, target_file, title=title, changePointList=changePoints) canMergeLast, isFullBin = coreSketch.add_element(instance) window = coreSketch.getWindow()
def save_all_tran(data_list,name_list): for index, data in enumerate(data_list): ut.checkFolderandCreate("{}{}".format(pic_path, name_list[index])) randomlist = random.sample(range(0, data.get_len_tran()), 100) for index_tran in randomlist: fig = data.plot_tran_sten(index_tran=index_tran, start_index=0 , end_index=8520, plot_change=False, plot_tran=True) # plt.show() target_file = "{}{}\\{}.png".format(pic_path, name_list[index],"{:02d}".format(index_tran)) plt.savefig(target_file) plt.clf()
def genROCFile(name, data, threshold=0.05): resultRow = [] resultRow.append(name) resultRow.append(threshold) true_alerts = 0.0 false_alerts = 0.0 total_alert = 0.0 pos = 0.0 neg = 0.0 found = [] for i, file_name in enumerate(data.get_files_test()): ut.checkFolderandCreate("{}\\Changepoint\\{}".format( main_path, algo_name)) pickle_file = '{}\\Changepoint\\{}\\_{}_threshold{}_index_{:03d}.pkl'.format \ (main_path, algo_name, name, threshold, i) instances = data.get_dataset_test(i) if not os.path.exists(pickle_file): change_points = dybin(instances=instances, threshold=threshold) with open(pickle_file, 'wb') as output: pickle.dump(change_points, output) else: with open(pickle_file, 'rb') as pickle_input: change_points = pickle.load(pickle_input) # start_end = data.get_dataset_answer_st_ed(index=i)[0] # row = ut_light.calRowTF_NoBin(start=start_end[0], end=start_end[1], change_points=change_points, # size_instances=len(instances), # bin_size=bin_size) row = ut_light.calRowTF_NoBin_MixTran( st_ed_list=data.get_dataset_answer_st_ed(index=i), change_points=change_points, size_instances=len(instances), bin_size=bin_size) true_alerts = true_alerts + row[0] false_alerts = false_alerts + row[1] total_alert = total_alert + row[2] pos = pos + row[3] neg = neg + row[4] found.append(row[9]) resultRow = [] resultRow.append(name) resultRow.append(threshold) resultRow.append(true_alerts) resultRow.append(false_alerts) resultRow.append(total_alert) resultRow.append(pos) resultRow.append(neg) tp_rate = float(true_alerts) / pos resultRow.append(tp_rate) fp_rate = float(false_alerts) / neg resultRow.append(fp_rate) resultRow.append(sum(found)) return resultRow
def save_all_tran_algoritm(data_list, name_list): for index, data in enumerate(data_list): full_pic_path = "{}{}_alphamean{}alphavar{}".format( pic_path, name_list[index], alpha_mean, alpha_var) full_bitvalue_path = "{}{}_bitvalue_dy_IS_BIN_{}".format( pic_path, name_list[index], str(IS_BIN)) full_bittrend_path = "{}{}_bittread_dy_IS_BIN_{}".format( pic_path, name_list[index], str(IS_BIN)) full_quantine_path = "{}{}_quantine".format(pic_path, name_list[index]) ut.checkFolderandCreate(full_pic_path) ut.checkFolderandCreate(full_bitvalue_path) ut.checkFolderandCreate(full_bittrend_path) ut.checkFolderandCreate(full_quantine_path) for index_tran in range(data.get_len_tran()): instances = data.get_dataset_test(int(index_tran)) changePoint, window = dyBin_cal(instances) target_file = "{}\\{}.png".format(full_pic_path, "{:02d}".format(index_tran)) instances = plot_window(window, target_file) # bitValue,bitTrend = clipperCal(instances) # target_bitValueFile = "{}\\{}.png".format(full_bitvalue_path, "{:02d}".format(index_tran)) # plot_bit(bitValue,target_bitValueFile) # # target_bitTrendFile = "{}\\{}.png".format(full_bittrend_path, "{:02d}".format(index_tran)) # plot_bit(bitTrend, target_bitTrendFile) # if index_tran == 89: # print("a") quantileWindow = quantile_cal(instances) target_quantineFile = "{}\\{}.png".format( full_quantine_path, "{:02d}".format(index_tran)) plot_quantile(quantileWindow, target_quantineFile)
def save_tran_overTime_dy(data_list, name_list): periodSize = 500 for index, data in enumerate(data_list): for index_tran in range(data.get_len_tran()): title = data.get_file_test_name(index=index_tran) instances = data.get_dataset_test(int(index_tran)) full_pic_path = "{}{}_alphamean{}alphavar{}".format( pic_path, name_list[index], alpha_mean, alpha_var) ut.checkFolderandCreate(full_pic_path) periodTime = int(len(instances) / periodSize) for i in range(periodTime - 1): ins = instances[0:periodSize * (1 + i)] changePoint, window = dyBin_cal(ins) target_file = "{}\\{}_{}.png".format( full_pic_path, "{:02d}".format(index_tran), i) plot_window(window, target_file, title=title)
def save_tran_overTime(self, data, namePattern, StopTimeStampList, threshold, tran=False): for index_tran in range(data.get_len_tran()): tempPeriod = [] + StopTimeStampList title = data.get_file_test_name(index=index_tran) instances = data.get_dataset_test(int(index_tran)) if tran: st_end_list = data.get_dataset_answer_st_ed(index_tran) for st_end in st_end_list: for temp in st_end: if temp != 0: tempPeriod.append(temp) tempPeriod.append(temp + 5) else: st_end_list = [] full_pic_path = "{}{}_{}_bin{}_win{}".format( self.pic_path, namePattern, self.typeSketc, self.initialBin, self.windowSize) ut.checkFolderandCreate(full_pic_path) for i, instance in enumerate(instances): self.coreSketch.add_element(instance) if i in tempPeriod: target_file = "{}\\{}_{}.png".format( full_pic_path, "{:02d}".format(index_tran), i) self.coreSketch.plot_window(target_file=target_file, title=title, tran_list=st_end_list, isCutOff=False, plotChange=True, threshold=threshold) if i == len(instances) - 1: self.coreSketch.plot_window(target_file=target_file, title=title, tran_list=st_end_list, isCutOff=True, plotChange=True, threshold=threshold) self.coreSketch.reset()
def genFolderTransientPattern(pattern, height, duration): targetPath = '{}\\{}\\{}'.format(main_path, dataset_type, filterType) folderTran = '{}_L{}_I{}'.format(pattern, height, duration) ut.checkFolderandCreate("{}\\{}".format(targetPath, folderTran)) testPath = '{}\\{}\\test'.format(targetPath, folderTran) answerPath = '{}\\{}\\answer'.format(targetPath, folderTran) ut.checkFolderandCreate(testPath) ut.checkFolderandCreate(answerPath) picPath = '{}\\{}\\pic'.format(targetPath, folderTran) ut.checkFolderandCreate(picPath) return testPath, answerPath, picPath
resultRow.append(fp_rate) resultRow.append(sum(found)) return resultRow if __name__ == '__main__': ut.list_to_csv(head=head) result = [] df_list = [] name_list = ut_light.gen_list_mix(main_path=dataset_path,data_type='bgs',mix_size="*sq_L3_I5_*") for index, name in enumerate(name_list): for threshold in thresholds: algo_name = "SEA_{}_{}".format(bin_sizes,block_type) ut.checkFolderandCreate("{}\\ROC\\{}".format(main_path, algo_name)) pickle_ROC_file = '{}\\ROC\\{}\\_{}_threshold{}.pkl'.format(main_path, algo_name, name, threshold) if not os.path.exists(pickle_ROC_file): result = [] data = ut_light.load_light_select_dataset(input_path=dataset_path, file_name=name, is_mix=is_mix) print("##### cal name : {} , threshold = {} #####".format(name, threshold)) ROC_row = genROCFile(name=name, data=data, threshold=threshold) result.append(ROC_row) df = pd.DataFrame(result, columns=ROC_head) with open(pickle_ROC_file, 'wb') as output: pickle.dump(df, output) else: print("##### Load name : {} , threshold = {} #####".format(name, threshold)) with open(pickle_ROC_file, 'rb') as pickle_input: df = pickle.load(pickle_input) df_list.append(df)
tree.insert_point(point, index=index) # Compute codisp on the new point... # new_codisp = tree.codisp(index) # And take the average over all trees if not index in avg_codisp: avg_codisp[index] = 0 avg_codisp[index] += tree.codisp(index) / num_trees return avg_codisp if __name__ == '__main__': for height in heights: for duration in durations: # folder_name = "I{}_L{}".format(height,duration) folder_name = "temp" ut.checkFolderandCreate(folder_name) listFile = ut_lc.getListLight(height=height, duration=duration) # for file_name in listFile[340:]: for file_name in listFile[0:1]: lc_data = ut_lc.getDataFromFile(fileName=file_name, height=height, duration=duration) # corePlot = sketchDyBinService(windowSize=windowSize, initialBin=initialBin, isOnline=True) # sketchInstances = corePlot.sketchMode(instances=lc_data['instances']) # windowList = corePlot.getWindow() # binInstances = [] # for bin in windowList: # binInstances.append(bin.get_representation()) avg_codisp = get_avgcodisp(lc_data['instances'])
if __name__ == '__main__': listFile = ut_mdf.getListMDF() for indexFile, fileName in enumerate(listFile): print(fileName) LCFile = "{}{}.txt".format(path_to_lc_file, fileName) filesize = os.path.getsize(LCFile) if filesize != 0: fileDate = fileName.split("_date")[1] # Save LC to Common LC file = open(LCFile, 'rb') FdataMDF = [float(item) for item in ut.txt_to_list(LCFile)] save_lc = "{}{}\\{}\\".format(main_save_path, "lc_flux_catalog_aperture_r7", fileDate) ut.checkFolderandCreate(save_lc) save_lc_fileName = "{}{}.txt".format(save_lc, fileName) if not (ut.isFileNameInFolder(save_lc, "{}.txt".format(fileName))): ut.list_to_txt(rows=FdataMDF, csv_name=save_lc_fileName, is_sort=False) # Save LC to Common MJD path_lc_timestamp_path = "{}{}{}.txt".format( dataset_path, lc_timestamp_path, fileName) mjd_list = [ float(item) for item in ut.txt_to_list(path_lc_timestamp_path) ] save_mjd = "{}{}\\{}\\".format(main_save_path, "timestamp_MJD", fileDate)
def getListTop(): fileName = "{}top418.csv".format(dataset_path) listFile = ut.txt_to_list(fileName) return listFile if __name__ == '__main__': # listFile = getListTop() # temp = 'light_curve_Gaia-DR2_652961717644929664_date20200201' # listFile = ut_web.getListMDF(pattern = pattern) csvFile = ut.getListAllFileName(path="{}csv\\".format(dataset_path)) for pattern in csvFile: listFile = ut.txt_to_list(csv_name="{}csv\\{}.csv".format(dataset_path,pattern)) pathPngOutput = "{}{}\\{}\\".format(dataset_path, png_path, pattern) # listFile = ["light_curve_Gaia-DR2_608215408323505280_date20200201"] ut.checkFolderandCreate(pathPngOutput) if ut.isEmpty(pathPngOutput): for indexFile, fileName in enumerate(listFile): print(fileName) isFoundHtml = ut.isFileNameInFolder(path=pathHtmlOutput, fileName="{}.html".format(fileName)) isFoundPng = ut.isFileNameInFolder(path=pathPngOutput, fileName="{}.png".format(fileName)) if not (isFoundHtml): pathFile = "{}{}.txt".format(path_to_lc_file, fileName) filesize = os.path.getsize(pathFile) if filesize != 0: file = open(pathFile, 'rb') FdataMDF = [float(item) for item in ut.txt_to_list(pathFile)] path_lc_timestamp_path = "{}{}{}.txt".format(dataset_path, lc_timestamp_path, fileName) x_axis = [float(item) for item in ut.txt_to_list(path_lc_timestamp_path)] TOOLTIPS = [ ("index", "$index"),
is_sort = False) ut.list_to_txt(rows=new_answer_st_ed_list, csv_name='{}\\{}'.format(answer_list_path, new_file_name), is_sort=False) if __name__ == '__main__': name_list = ut_light.gen_list_title(data_type = data_type) new_list = gen_new_list_mix(path) for index, target_list in enumerate(new_list): main_tran = target_list[0] target_tran = target_list[1] folder = "{}_{}".format(main_tran.replace("{}_".format(data_type), ""), target_tran.replace("{}_".format(data_type), "")) print(folder) ut.checkFolderandCreate("{}\\{}\\{}".format(path, data_type ,folder)) test_list_path = "{}{}\\{}\\test".format(path, data_type, folder) ut.checkFolderandCreate(test_list_path) answer_list_path = "{}{}\\{}\\answer".format(path, data_type, folder) ut.checkFolderandCreate(answer_list_path) data_1 = ut_light.load_light_select_dataset(input_path=path, file_name=main_tran, is_mix=True) data_2 = ut_light.load_light_select_dataset(input_path=path, file_name=target_tran) merge_file(data_1, data_2,test_list_path,answer_list_path) # for i in range(len(name_list)): # data_1 = ut_light.load_light_select_dataset(input_path=path, # file_name=name_list[i])