def ratioPlot():
listFiles = ut_mdf.getListMDF()
if request.method == 'POST':
fileName = str(request.form.get('selectFile'))
FdataMDF, FtimestampMDF = ut_service.getMDFData(fileName)
# sketch
windowSize = int(request.form.get('inputWindowSize'))
initialBin = int(request.form.get('inputInitialBin'))
x_axis = FtimestampMDF
# corePlot = sketchService(windowSize=windowSize, initialBin=3)
# sketch3Instances = corePlot.sketchMode(instances=FdataMDF)
corePlot = sketchDyBinService(windowSize=windowSize, initialBin=3)
sketchInstances = corePlot.sketchMode(instances=FdataMDF)
print("sketch {}".format(initialBin))
bin3Instnace = ut_gen.genFixBin(binSize=200, instances=FdataMDF)
print("fix 3")
fixBinInstnace = ut_gen.genFixBin(binSize=initialBin,
instances=FdataMDF)
print("fix {}".format(initialBin))
# jsMDF, divMDF = ut_bokeh.JSandDivMutliplePlot(x_axis=x_axis,
# y_axisList = [FdataMDF,sketch3Instances,sketchInstances,bin3Instnace,fixBinInstnace],
# circleList = [True,False,False,False,False],
# legend_labelList = ['Raw','DyBin3','DyBin{}'.format(initialBin),'Bin size = 3','Bin size = {}'.format(initialBin)],
# line_alphaList = [0.4,1,1,1,1],
# title=fileName)
jsMDF, divMDF = ut_bokeh.JSandDivMutliplePlot(
x_axis=x_axis,
y_axisList=[FdataMDF, sketchInstances, bin3Instnace],
circleList=[True, False, False],
legend_labelList=['Raw', 'DyBin', 'Bin size = 200'],
line_alphaList=[0.4, 1, 1],
title=fileName)
core = com_service(inputList=FdataMDF)
# dictResult = core.TWINcurveBinSize(maxBinSize=len(x_axis))
# jsCom, divCom = ut_bokeh.JSandDivComPlot(binSizeList=dictResult["binSizeList"], comList=dictResult["comList"],
# varList=dictResult["varList"])
# return render_template("html/ratio/ratio_plot.html", jsMDF=jsMDF, divMDF=divMDF
# , jsCom=jsCom, divCom=divCom, listFiles=listFiles)
dictResult = core.TWINcurveWindowSize()
# dictResult = core.TWINcurveWindowSizeFixBin(timestamps=FtimestampMDF)
jsCom, divCom = ut_bokeh.JSandDivComPlotTemp(
binSizeList=dictResult["binSizeList"],
comList=dictResult["comList"],
varList=dictResult["varList"])
return render_template("html/ratio/ratio_plot.html",
jsMDF=jsMDF,
divMDF=divMDF,
jsCom=jsCom,
divCom=divCom,
listFiles=listFiles)
def MDFPlot():
listFiles = ut_mdf.getListMDF()
if request.method == 'POST':
fileName = str(request.form.get('selectFile'))
FdataMDF, FtimestampMDF = ut_service.getMDFData(fileName)
# sketch
windowSize = int(request.form.get('inputWindowSize'))
initialBin = int(request.form.get('inputInitialBin'))
inputOnline = request.form.get('inputOnline')
# if inputOnline == "True":
# isOnline = True
# else:
# isOnline = False
isOnline = False
# print("**************{}***********".format(isOnline))
corePlot = sketchDyBinService(windowSize=windowSize,
initialBin=initialBin,
isOnline=isOnline)
threshold_tTest = float(request.form.get('threshold_tTest'))
threshold_fTest = float(request.form.get('threshold_fTest'))
sketchInstances = corePlot.sketchMode(instances=FdataMDF)
indexBinList = corePlot.getBinNameInstances()
getListMeanBin = corePlot.getListMeanBin()
print(getListMeanBin)
dictResult = corePlot.getListResultTest()
js, div = ut_bokeh.JSandDivRawAndBin(x_axis=FtimestampMDF,
y_raw=FdataMDF,
y_axisList=[sketchInstances],
legend_labelList=["dyBin"],
namesList=[indexBinList],
title="{} window size :{}".format(
fileName, windowSize))
return render_template("html/mdf/mdf_plot.html",
jsMDF=js,
divMDF=div,
listFiles=listFiles)
import json
import utility.utility_mdf as ut_mdf
from astropy.time import Time
import utility.utility_db as ut_db
dataset_path = 'D:\\mdwarf_data\\'
lc_path = "lc_flux_catalog_aperture_r7_txt\\"
lc_timestamp_path = "lc_timestamp_txt\\"
path_to_lc_file = "{}{}".format(dataset_path, lc_path)
if __name__ == '__main__':
fileList = ut_mdf.getListMDF()
rows = []
for index, fileName in enumerate(fileList):
data = ut_mdf.getDataFromFile(fileName=fileName)
fileDate = fileName.split('_date')[1]
startTimeMJD = data["timestamp"][0]
startTimeUTC = Time(startTimeMJD, format='mjd').iso
endTimeMJD = data["timestamp"][-1]
endTimeUTC = Time(endTimeMJD, format='mjd').iso
rows.append((fileName, fileDate, startTimeMJD, endTimeMJD,
startTimeUTC, endTimeUTC))
ut_db.insertMJDRow(rows=rows)
print("a")
import utility.utility_astronomy as ut_astro
import utility.utility_exportBokeh as ut_ex
dataset_path = 'D:\\mdwarf_data\\'
lc_path = "lc_flux_catalog_aperture_r7_txt\\"
lc_timestamp_path = "lc_timestamp_txt\\"
path_to_lc_file = "{}{}".format(dataset_path, lc_path)
html_path = "lc_flux_catalog_aperture_r7_html\\"
png_path = "lc_flux_catalog_aperture_r7_png\\"
# save file
main_save_path = '{}\\mdwarf_data_common\\'.format(dataset_path)
# LC
if __name__ == '__main__':
listFile = ut_mdf.getListMDF()
for indexFile, fileName in enumerate(listFile):
print(fileName)
LCFile = "{}{}.txt".format(path_to_lc_file, fileName)
filesize = os.path.getsize(LCFile)
if filesize != 0:
fileDate = fileName.split("_date")[1]
# Save LC to Common LC
file = open(LCFile, 'rb')
FdataMDF = [float(item) for item in ut.txt_to_list(LCFile)]
save_lc = "{}{}\\{}\\".format(main_save_path,
"lc_flux_catalog_aperture_r7",
fileDate)
ut.checkFolderandCreate(save_lc)
save_lc_fileName = "{}{}.txt".format(save_lc, fileName)
def ratioInput():
listFiles = ut_mdf.getListMDF()
return render_template("html/ratio/ratio_input.html", listFiles=listFiles)
def ratioMain():
listFiles = ut_mdf.getListMDF()
return render_template("html/ratio/ratio_main.html", listFiles=listFiles)
def xmeanMain():
listFiles = ut_mdf.getListMDF()
return render_template("html/xmean/xmean_main.html", listFiles=listFiles)
def MDFMain():
listFiles = ut_mdf.getListMDF()
return render_template("html/mdf/mdf_main.html", listFiles=listFiles)
def clusterPlot():
listFiles = ut_mdf.getListMDF()
if request.method == 'POST':
windowSize = int(request.form.get('inputWindowSize'))
fileName = str(request.form.get('selectFile'))
isDybin = request.form.get('inputDybin')
initialBin = int(request.form.get('inputInitialBin'))
FdataMDF, FtimestampMDF = ut_service.getMDFData(fileName)
corePlot = sketchDyBinService(windowSize=windowSize,
initialBin=5,
isOnline=False)
sketchInstances = corePlot.sketchMode(instances=FdataMDF)
indexBinList = corePlot.getBinNameInstances()
window = corePlot.getWindow()
indexTimestamp = 0
segment3D = []
for index, bin in enumerate(window):
raw = bin.get_raw()
start = copy.copy(indexTimestamp)
indexTimestamp = indexTimestamp + len(raw)
tempTimestamp = FtimestampMDF[start:indexTimestamp]
slope = ut_cal.calSlope(timestamps=tempTimestamp, instances=raw)
result = {
"raw": raw,
"index": index,
"variance": bin.get_variance(),
"mean": bin.get_representation(),
"slope": slope
}
segment3D.append(result)
cluster = ut_cluster.cluster_xMean_Dy(binsList=window,
kmax=windowSize,
hist_bin=20)
ut_plotly.plot3DClusterDYTemp(samples=cluster["samples"],
clusters=cluster["clusters"],
centers=cluster["centers"])
resultCluster = ut_gen.genHistrogram_Dict(binsList=segment3D,
hist_bin=20)
jsMDF, divMDF = ut_bokeh.JSandDivRawAndBin(
x_axis=FtimestampMDF,
y_raw=FdataMDF,
y_axisList=[sketchInstances],
legend_labelList=["dyBin"],
namesList=[indexBinList],
title="{} window size :{}".format(fileName, windowSize))
jsCl, divCl = ut_bokeh.JSandDivHistrogramCluster(
meansClass=resultCluster['meansClass'],
varsClass=resultCluster['varsClass'],
slopesClass=resultCluster['slopesClass'])
return render_template("html/cluster/cluster_plot.html",
listFiles=listFiles,
jsMDF=jsMDF,
divMDF=divMDF,
jsCl=jsCl,
divCl=divCl)
def clusterMain():
listFiles = ut_mdf.getListMDF()
return render_template("html/cluster/cluster_main.html",
listFiles=listFiles)
def clusterInput():
listFiles = ut_mdf.getListMDF()
return render_template("html/cluster/cluster_input.html",
listFiles=listFiles)
if __name__ == '__main__':
# windowSizeList = [500,250,100,50,20,10,5]
# datePatterns = ['20191130', '20191203', '20191205', '20191224', '20200130','20200203']
# datePatterns = ['20190812', '20191004', '20191005',
# '20191006', '20191106', '20191120',
# '20191121', '20191123', '20191121',
# '20191123', '20191124', '20191129',
# '20191130', '20191202', '20191203',
# '20191204', '20191205', '20191224',
# '20191226', '20200129', '20200130',
# '20200201', '20200202', '20200203']
datePatterns = [
'20190812', '20191004', '20191005', '20191006', '20191106', '20191120',
'20191121', '20191123', '20191121', '20191123', '20191124', '20191129',
'20191202', '20191204', '20191226', '20200129', '20200201', '20200202'
]
for datePattern in datePatterns:
fileList = ut_mdf.getListMDF(pattern=datePattern)
for fileName in fileList[1:]:
lightData = ut_mdf.getDataFromFile(fileName=fileName)
windowSizeList = genWindowList(len(lightData["instances"]))
for windowSize in windowSizeList:
var_ratio, com_ratio = computeCompressgraph(
lightData["instances"], windowSize)
row = [fileName, datePattern, windowSize, var_ratio, com_ratio]
print(row)
ut.add_to_csv(row=row, csv_name="comratio_mdf.csv")
if __name__ == '__main__':
windowSize = 40
upper_bound = 1.87519737
lower_bound = 0.53327720
listFile = ut.txt_to_list(csv_name="start_file.csv")
# for windowSize in listWindow:
for index, fileName1 in enumerate(listFile):
print("File Name {}".format(fileName1))
pattern = fileName1.split("_")[4]
lightData1 = ut_mdf.getDataFromFile(fileName=fileName1)
dyResult1 = ut_data.genListDyBin(instances=lightData1["instances"],
timestamp=lightData1["timestamp"],
windowSize=windowSize)
targetList = ut_mdf.getListMDF(pattern=pattern)
for fileName2 in targetList:
try:
lightData2 = ut_mdf.getDataFromFile(fileName=fileName2)
isOverlap = ut_mdf.isOverlapTimestamp(lightData1["timestamp"],
lightData2["timestamp"])
if isOverlap:
dyResult2 = ut_data.genListDyBin(
instances=lightData2["instances"],
timestamp=lightData2["timestamp"],
windowSize=windowSize)
# computeLocalOutlierFactor(dyResult1,windowSize=windowSize)
F_test = dyResult1["variance"] / dyResult2["variance"]
if lower_bound <= F_test <= upper_bound:
row = [fileName1, fileName2, windowSize, F_test]
print(row)
import utility.utility_mdf as ut_mdf
import utility.utility as ut
import utility.utility_genData as ut_gen
import os
from method.change_de.grabbs_test_detection import grabbs_test_detection as gb_test
dataset_path = 'D:\\mdwarf_data\\'
lc_path = "lc_flux_catalog_aperture_r7_txt\\"
lc_timestamp_path = "lc_timestamp_txt\\"
path_to_lc_file = "{}{}".format(dataset_path, lc_path)
initialBin = 3
theshold_compression = 0.03
alpha = 0.05
if __name__ == '__main__':
listFile = ut_mdf.getListMDF(
startFile='light_curve_Gaia-DR2_610350453746204032_date20200201')
for indexFile, fileName in enumerate(listFile):
try:
pathFile = "{}{}.txt".format(path_to_lc_file, fileName)
filesize = os.path.getsize(pathFile)
if filesize != 0:
result_found = 0
file = open(pathFile, 'rb')
FdataMDF = [float(item) for item in ut.txt_to_list(pathFile)]
path_lc_timestamp_path = "{}{}{}.txt".format(
dataset_path, lc_timestamp_path, fileName)
x_axis = [
float(item)
for item in ut.txt_to_list(path_lc_timestamp_path)
]
# window_size = int((len(FdataMDF) / initialBin) * theshold_compression)
