def cal_sk_bin(data, input_k,input_bin, file_format, L,I):
# input_k = 2
data.clear_change_points()
detector = light_detector()
main_path = 'ks_bin'
result_text = []
ut.checkFolderandCreate(main_path)
pickle_file = '{}\\cheb{}\\{}_L{}_I{}.pkl'.format(main_path,Cheb_win ,file_format, L,I)
if not os.path.exists(pickle_file):
detector.set_parameter(input_k=input_k, input_min_winsize=Cheb_win, input_max_winsize=Cheb_win,
input_bin=input_bin)
for i, file_name in enumerate(data.get_files_test()):
instances = data.get_dataset_test(i)
change_points = detector.get_changepoint_skmethod(instances)
data.set_change_points(change_points)
with open(pickle_file, 'wb') as output:
pickle.dump(data, output)
else:
with open(pickle_file, 'rb') as pickle_input:
data = pickle.load(pickle_input)
# true_alerts, false_alerts, tran_found = data.count_TF_tranfound_withBin(Bin=input_bin)
# result_text.append("SK Bin = {} and K = {}".format(input_bin, input_k))
# result_text.append("True alert = {} false_alert = {} tran_found = {}".format(true_alerts, false_alerts, tran_found))
return data,result_text
def cal_CANDAR_bin(data, input_k, input_bin, file_format, L,I,is_k = True):
algorithm = ""
data.clear_change_points()
detector = light_detector()
if is_k:
main_path = 'CANDAR_bin'
else:
main_path = 'CANDAR_bin_E'
result_text = []
ut.checkFolderandCreate(main_path)
pickle_file = '{}\\cheb{}\\{}_L{}_I{}.pkl'.format(main_path,Cheb_win, file_format, L,I)
if not os.path.exists(pickle_file):
detector.set_parameter(input_k=input_k, input_min_winsize=Cheb_win, input_max_winsize=Cheb_win,
input_bin=input_bin,is_k=is_k)
for i, file_name in enumerate(data.get_files_test()):
instances = data.get_dataset_test(i)
change_points, mean_list = detector.get_changepoint_BinCheb(instances)
data.set_change_points(change_points)
with open(pickle_file, 'wb') as output:
pickle.dump(data, output)
else:
with open(pickle_file, 'rb') as pickle_input:
data = pickle.load(pickle_input)
true_alerts, false_alerts, tran_found = data.count_TF_tranfound_withBin(Bin=input_bin)
# result_text.append("BIN CANDAR Bin = {} and K = {} is_k = {}".format(input_bin, k,str(is_k)))
# result_text.append("True alert = {} false_alert = {} tran_found = {}".format(true_alerts, false_alerts, tran_found))
return data ,result_text
def save_tran_overTime_dy(data_list, name_list, tran=False):
periodSize = 200
for index, data in enumerate(data_list[0:200]):
for index_tran in range(data.get_len_tran()):
# chebE_core = chebyshev_Dy(e=e)
skCore = skDetection(k=3)
title = data.get_file_test_name(index=index_tran)
instances = data.get_dataset_test(int(index_tran))
tran_st = ""
tran_end = ""
if tran:
st_end_list = data.get_dataset_answer_st_ed(index_tran)[0]
# for st_end in st_end_list:
tran_st = st_end_list[0]
tran_end = st_end_list[1]
changePoints = []
full_pic_path = "{}{}_{}_bin{}_win{}".format\
(pic_path, name_list[index],typeSketc,bin_size,WINDOW_SIZE)
ut.checkFolderandCreate(full_pic_path)
coreSketch = typeOfSketch(typeName=typeSketc)
StopTimeStampList = ut.genTimeStampStopList(instances=instances,
periodSize=periodSize)
count = 0 # hi-net
for i, instance in enumerate(instances):
if i in StopTimeStampList:
target_file = "{}\\{}_{}.png".format(
full_pic_path, "{:02d}".format(index_tran), i)
window = coreSketch.getWindow()
############ Hinet
if i < 20000:
min_xlim = 0
else:
min_xlim = count * periodSize - 10000
max_xlim = (count + 1) * periodSize
# ut.plot_window_Dybin(window, target_file, title=title,
# changePointList=changePoints, tran_st=tran_st,
# tran_end=tran_end, min_xlim =min_xlim ,max_xlim=max_xlim)
count = count + 1
############ Hinet
# ut.plot_window_Dybin(window, target_file, title=title,
# changePointList=changePoints, tran_st=tran_st,
# tran_end=tran_end)
if i == len(instances) - 1:
coreSketch.cutOff_element()
target_file = "{}\\{}_{}.png".format(
full_pic_path, "{:02d}".format(index_tran), i)
window = coreSketch.getWindow()
# ut.plot_window_Dybin(window, target_file, title=title,
# changePointList=changePoints,tran_st=tran_st,
# tran_end=tran_end)
ut.plot_window_Dybin(window,
target_file,
title=title,
changePointList=changePoints)
canMergeLast, isFullBin = coreSketch.add_element(instance)
window = coreSketch.getWindow()
def save_all_tran(data_list,name_list):
for index, data in enumerate(data_list):
ut.checkFolderandCreate("{}{}".format(pic_path, name_list[index]))
randomlist = random.sample(range(0, data.get_len_tran()), 100)
for index_tran in randomlist:
fig = data.plot_tran_sten(index_tran=index_tran, start_index=0
, end_index=8520, plot_change=False, plot_tran=True)
# plt.show()
target_file = "{}{}\\{}.png".format(pic_path, name_list[index],"{:02d}".format(index_tran))
plt.savefig(target_file)
plt.clf()
def genROCFile(name, data, threshold=0.05):
resultRow = []
resultRow.append(name)
resultRow.append(threshold)
true_alerts = 0.0
false_alerts = 0.0
total_alert = 0.0
pos = 0.0
neg = 0.0
found = []
for i, file_name in enumerate(data.get_files_test()):
ut.checkFolderandCreate("{}\\Changepoint\\{}".format(
main_path, algo_name))
pickle_file = '{}\\Changepoint\\{}\\_{}_threshold{}_index_{:03d}.pkl'.format \
(main_path, algo_name, name, threshold, i)
instances = data.get_dataset_test(i)
if not os.path.exists(pickle_file):
change_points = dybin(instances=instances, threshold=threshold)
with open(pickle_file, 'wb') as output:
pickle.dump(change_points, output)
else:
with open(pickle_file, 'rb') as pickle_input:
change_points = pickle.load(pickle_input)
# start_end = data.get_dataset_answer_st_ed(index=i)[0]
# row = ut_light.calRowTF_NoBin(start=start_end[0], end=start_end[1], change_points=change_points,
# size_instances=len(instances),
# bin_size=bin_size)
row = ut_light.calRowTF_NoBin_MixTran(
st_ed_list=data.get_dataset_answer_st_ed(index=i),
change_points=change_points,
size_instances=len(instances),
bin_size=bin_size)
true_alerts = true_alerts + row[0]
false_alerts = false_alerts + row[1]
total_alert = total_alert + row[2]
pos = pos + row[3]
neg = neg + row[4]
found.append(row[9])
resultRow = []
resultRow.append(name)
resultRow.append(threshold)
resultRow.append(true_alerts)
resultRow.append(false_alerts)
resultRow.append(total_alert)
resultRow.append(pos)
resultRow.append(neg)
tp_rate = float(true_alerts) / pos
resultRow.append(tp_rate)
fp_rate = float(false_alerts) / neg
resultRow.append(fp_rate)
resultRow.append(sum(found))
return resultRow
def save_all_tran_algoritm(data_list, name_list):
for index, data in enumerate(data_list):
full_pic_path = "{}{}_alphamean{}alphavar{}".format(
pic_path, name_list[index], alpha_mean, alpha_var)
full_bitvalue_path = "{}{}_bitvalue_dy_IS_BIN_{}".format(
pic_path, name_list[index], str(IS_BIN))
full_bittrend_path = "{}{}_bittread_dy_IS_BIN_{}".format(
pic_path, name_list[index], str(IS_BIN))
full_quantine_path = "{}{}_quantine".format(pic_path, name_list[index])
ut.checkFolderandCreate(full_pic_path)
ut.checkFolderandCreate(full_bitvalue_path)
ut.checkFolderandCreate(full_bittrend_path)
ut.checkFolderandCreate(full_quantine_path)
for index_tran in range(data.get_len_tran()):
instances = data.get_dataset_test(int(index_tran))
changePoint, window = dyBin_cal(instances)
target_file = "{}\\{}.png".format(full_pic_path,
"{:02d}".format(index_tran))
instances = plot_window(window, target_file)
# bitValue,bitTrend = clipperCal(instances)
# target_bitValueFile = "{}\\{}.png".format(full_bitvalue_path, "{:02d}".format(index_tran))
# plot_bit(bitValue,target_bitValueFile)
#
# target_bitTrendFile = "{}\\{}.png".format(full_bittrend_path, "{:02d}".format(index_tran))
# plot_bit(bitTrend, target_bitTrendFile)
# if index_tran == 89:
# print("a")
quantileWindow = quantile_cal(instances)
target_quantineFile = "{}\\{}.png".format(
full_quantine_path, "{:02d}".format(index_tran))
plot_quantile(quantileWindow, target_quantineFile)
def save_tran_overTime_dy(data_list, name_list):
periodSize = 500
for index, data in enumerate(data_list):
for index_tran in range(data.get_len_tran()):
title = data.get_file_test_name(index=index_tran)
instances = data.get_dataset_test(int(index_tran))
full_pic_path = "{}{}_alphamean{}alphavar{}".format(
pic_path, name_list[index], alpha_mean, alpha_var)
ut.checkFolderandCreate(full_pic_path)
periodTime = int(len(instances) / periodSize)
for i in range(periodTime - 1):
ins = instances[0:periodSize * (1 + i)]
changePoint, window = dyBin_cal(ins)
target_file = "{}\\{}_{}.png".format(
full_pic_path, "{:02d}".format(index_tran), i)
plot_window(window, target_file, title=title)
def save_tran_overTime(self,
data,
namePattern,
StopTimeStampList,
threshold,
tran=False):
for index_tran in range(data.get_len_tran()):
tempPeriod = [] + StopTimeStampList
title = data.get_file_test_name(index=index_tran)
instances = data.get_dataset_test(int(index_tran))
if tran:
st_end_list = data.get_dataset_answer_st_ed(index_tran)
for st_end in st_end_list:
for temp in st_end:
if temp != 0:
tempPeriod.append(temp)
tempPeriod.append(temp + 5)
else:
st_end_list = []
full_pic_path = "{}{}_{}_bin{}_win{}".format(
self.pic_path, namePattern, self.typeSketc, self.initialBin,
self.windowSize)
ut.checkFolderandCreate(full_pic_path)
for i, instance in enumerate(instances):
self.coreSketch.add_element(instance)
if i in tempPeriod:
target_file = "{}\\{}_{}.png".format(
full_pic_path, "{:02d}".format(index_tran), i)
self.coreSketch.plot_window(target_file=target_file,
title=title,
tran_list=st_end_list,
isCutOff=False,
plotChange=True,
threshold=threshold)
if i == len(instances) - 1:
self.coreSketch.plot_window(target_file=target_file,
title=title,
tran_list=st_end_list,
isCutOff=True,
plotChange=True,
threshold=threshold)
self.coreSketch.reset()
def genFolderTransientPattern(pattern, height, duration):
targetPath = '{}\\{}\\{}'.format(main_path, dataset_type, filterType)
folderTran = '{}_L{}_I{}'.format(pattern, height, duration)
ut.checkFolderandCreate("{}\\{}".format(targetPath, folderTran))
testPath = '{}\\{}\\test'.format(targetPath, folderTran)
answerPath = '{}\\{}\\answer'.format(targetPath, folderTran)
ut.checkFolderandCreate(testPath)
ut.checkFolderandCreate(answerPath)
picPath = '{}\\{}\\pic'.format(targetPath, folderTran)
ut.checkFolderandCreate(picPath)
return testPath, answerPath, picPath
resultRow.append(fp_rate)
resultRow.append(sum(found))
return resultRow
if __name__ == '__main__':
ut.list_to_csv(head=head)
result = []
df_list = []
name_list = ut_light.gen_list_mix(main_path=dataset_path,data_type='bgs',mix_size="*sq_L3_I5_*")
for index, name in enumerate(name_list):
for threshold in thresholds:
algo_name = "SEA_{}_{}".format(bin_sizes,block_type)
ut.checkFolderandCreate("{}\\ROC\\{}".format(main_path, algo_name))
pickle_ROC_file = '{}\\ROC\\{}\\_{}_threshold{}.pkl'.format(main_path, algo_name, name, threshold)
if not os.path.exists(pickle_ROC_file):
result = []
data = ut_light.load_light_select_dataset(input_path=dataset_path, file_name=name, is_mix=is_mix)
print("##### cal name : {} , threshold = {} #####".format(name, threshold))
ROC_row = genROCFile(name=name, data=data, threshold=threshold)
result.append(ROC_row)
df = pd.DataFrame(result, columns=ROC_head)
with open(pickle_ROC_file, 'wb') as output:
pickle.dump(df, output)
else:
print("##### Load name : {} , threshold = {} #####".format(name, threshold))
with open(pickle_ROC_file, 'rb') as pickle_input:
df = pickle.load(pickle_input)
df_list.append(df)
tree.insert_point(point, index=index)
# Compute codisp on the new point...
# new_codisp = tree.codisp(index)
# And take the average over all trees
if not index in avg_codisp:
avg_codisp[index] = 0
avg_codisp[index] += tree.codisp(index) / num_trees
return avg_codisp
if __name__ == '__main__':
for height in heights:
for duration in durations:
# folder_name = "I{}_L{}".format(height,duration)
folder_name = "temp"
ut.checkFolderandCreate(folder_name)
listFile = ut_lc.getListLight(height=height, duration=duration)
# for file_name in listFile[340:]:
for file_name in listFile[0:1]:
lc_data = ut_lc.getDataFromFile(fileName=file_name,
height=height,
duration=duration)
# corePlot = sketchDyBinService(windowSize=windowSize, initialBin=initialBin, isOnline=True)
# sketchInstances = corePlot.sketchMode(instances=lc_data['instances'])
# windowList = corePlot.getWindow()
# binInstances = []
# for bin in windowList:
# binInstances.append(bin.get_representation())
avg_codisp = get_avgcodisp(lc_data['instances'])
if __name__ == '__main__':
listFile = ut_mdf.getListMDF()
for indexFile, fileName in enumerate(listFile):
print(fileName)
LCFile = "{}{}.txt".format(path_to_lc_file, fileName)
filesize = os.path.getsize(LCFile)
if filesize != 0:
fileDate = fileName.split("_date")[1]
# Save LC to Common LC
file = open(LCFile, 'rb')
FdataMDF = [float(item) for item in ut.txt_to_list(LCFile)]
save_lc = "{}{}\\{}\\".format(main_save_path,
"lc_flux_catalog_aperture_r7",
fileDate)
ut.checkFolderandCreate(save_lc)
save_lc_fileName = "{}{}.txt".format(save_lc, fileName)
if not (ut.isFileNameInFolder(save_lc, "{}.txt".format(fileName))):
ut.list_to_txt(rows=FdataMDF,
csv_name=save_lc_fileName,
is_sort=False)
# Save LC to Common MJD
path_lc_timestamp_path = "{}{}{}.txt".format(
dataset_path, lc_timestamp_path, fileName)
mjd_list = [
float(item)
for item in ut.txt_to_list(path_lc_timestamp_path)
]
save_mjd = "{}{}\\{}\\".format(main_save_path, "timestamp_MJD",
fileDate)
def getListTop():
fileName = "{}top418.csv".format(dataset_path)
listFile = ut.txt_to_list(fileName)
return listFile
if __name__ == '__main__':
# listFile = getListTop()
# temp = 'light_curve_Gaia-DR2_652961717644929664_date20200201'
# listFile = ut_web.getListMDF(pattern = pattern)
csvFile = ut.getListAllFileName(path="{}csv\\".format(dataset_path))
for pattern in csvFile:
listFile = ut.txt_to_list(csv_name="{}csv\\{}.csv".format(dataset_path,pattern))
pathPngOutput = "{}{}\\{}\\".format(dataset_path, png_path, pattern)
# listFile = ["light_curve_Gaia-DR2_608215408323505280_date20200201"]
ut.checkFolderandCreate(pathPngOutput)
if ut.isEmpty(pathPngOutput):
for indexFile, fileName in enumerate(listFile):
print(fileName)
isFoundHtml = ut.isFileNameInFolder(path=pathHtmlOutput, fileName="{}.html".format(fileName))
isFoundPng = ut.isFileNameInFolder(path=pathPngOutput, fileName="{}.png".format(fileName))
if not (isFoundHtml):
pathFile = "{}{}.txt".format(path_to_lc_file, fileName)
filesize = os.path.getsize(pathFile)
if filesize != 0:
file = open(pathFile, 'rb')
FdataMDF = [float(item) for item in ut.txt_to_list(pathFile)]
path_lc_timestamp_path = "{}{}{}.txt".format(dataset_path, lc_timestamp_path, fileName)
x_axis = [float(item) for item in ut.txt_to_list(path_lc_timestamp_path)]
TOOLTIPS = [
("index", "$index"),
is_sort = False)
ut.list_to_txt(rows=new_answer_st_ed_list,
csv_name='{}\\{}'.format(answer_list_path, new_file_name),
is_sort=False)
if __name__ == '__main__':
name_list = ut_light.gen_list_title(data_type = data_type)
new_list = gen_new_list_mix(path)
for index, target_list in enumerate(new_list):
main_tran = target_list[0]
target_tran = target_list[1]
folder = "{}_{}".format(main_tran.replace("{}_".format(data_type), ""),
target_tran.replace("{}_".format(data_type), ""))
print(folder)
ut.checkFolderandCreate("{}\\{}\\{}".format(path, data_type ,folder))
test_list_path = "{}{}\\{}\\test".format(path, data_type, folder)
ut.checkFolderandCreate(test_list_path)
answer_list_path = "{}{}\\{}\\answer".format(path, data_type, folder)
ut.checkFolderandCreate(answer_list_path)
data_1 = ut_light.load_light_select_dataset(input_path=path,
file_name=main_tran, is_mix=True)
data_2 = ut_light.load_light_select_dataset(input_path=path,
file_name=target_tran)
merge_file(data_1, data_2,test_list_path,answer_list_path)
# for i in range(len(name_list)):
# data_1 = ut_light.load_light_select_dataset(input_path=path,
# file_name=name_list[i])
