def ratioPlot():
listFiles = utility.utility_mdf.getListMDF()
if request.method == 'POST':
fileName = str(request.form.get('selectFile'))
FdataMDF, FtimestampMDF = ut_service.getMDFData(fileName)
# sketch
windowSize = int(request.form.get('inputWindowSize'))
initialBin = int(request.form.get('inputInitialBin'))
x_axis = FtimestampMDF
core = coreMean(inputList=FdataMDF)
dictResult = core.getCompressFixWindow(windowSize=windowSize)
a = core.getAccuracy(windowSize=windowSize)
print("com w_{} = {} ".format(windowSize,
core.getAccuracy(windowSize=windowSize)))
js, div = ut_bokeh.JSandDivDoublePlotStandard(
x_axis=x_axis,
y_axisRaw=FdataMDF,
y_axisBin=dictResult["compressInstances"])
return render_template("html/ratio/ratio_plot.html",
jsMDF=js,
divMDF=div,
jsCom="",
divCom="",
listFiles=listFiles)
def xmeanPlot():
listFiles = ut_service.getListLC_test()
if request.method == 'POST':
fileName = str(request.form.get('selectFile'))
amount_initial_centers = int(
request.form.get('amount_initial_centers'))
kmax = int(request.form.get('kmax'))
FdataMDF, FtimestampMDF = ut_service.getMDFData(fileName)
resultDic = ut_cluster.cluster_xMean(
listInput=FdataMDF,
amount_initial_centers=amount_initial_centers,
kmax=kmax)
TOOLTIPS = [
("index", "$index"),
("(x,y)", "@x{0,0.000000}, $y"),
]
js, div = ut_bokeh.JSandDivXMeanPlot(x_axis=FtimestampMDF,
y_axisRaw=FdataMDF,
resultDic=resultDic,
title=fileName,
TOOLTIPS=TOOLTIPS)
# js, div = ut_bokeh.JSandDivDoublePlot(x_axis=FtimestampMDF, y_axisRaw=FdataMDF, y_axisBin=sketchInstances,
# title=fileName, transList=transList,alarmList=alarmList)
#
# rows = corePlot.getDataTableMeanVarianceBin()
return render_template("html/LC_test/lc_test_plot.html",
jsResult=js,
divResult=div,
listFiles=listFiles)
def dy_tran():
if request.method == 'POST':
listKeys = [
'inputWindowSize', 'inputInitialBin', 'isOnline', 'fileName'
]
inputKeys = request.form.keys()
if ut_service.isCompleteInputKey(listKeys=listKeys,
inputKeys=inputKeys):
FdataMDF, FtimestampMDF = ut_service.getMDFData(
request.form.get('fileName'))
windowSize = int(request.form.get('inputWindowSize'))
initialBin = int(request.form.get('inputInitialBin'))
inputOnline = bool(request.form.get('isOnline'))
corePlot = sketchDyBinService(windowSize=windowSize,
initialBin=initialBin,
isOnline=inputOnline)
sketchInstances = corePlot.sketchMode(instances=FdataMDF)
indexBinList = corePlot.getBinNameInstances()
window = corePlot.getWindow()
else:
return jsonify(
status="404",
message="Keys are incorrect",
)
return render_template("html/mdf/mdf_main.html", listFiles=[])
def ratioPlot():
listFiles = ut_mdf.getListMDF()
if request.method == 'POST':
fileName = str(request.form.get('selectFile'))
FdataMDF, FtimestampMDF = ut_service.getMDFData(fileName)
# sketch
windowSize = int(request.form.get('inputWindowSize'))
initialBin = int(request.form.get('inputInitialBin'))
x_axis = FtimestampMDF
# corePlot = sketchService(windowSize=windowSize, initialBin=3)
# sketch3Instances = corePlot.sketchMode(instances=FdataMDF)
corePlot = sketchDyBinService(windowSize=windowSize, initialBin=3)
sketchInstances = corePlot.sketchMode(instances=FdataMDF)
print("sketch {}".format(initialBin))
bin3Instnace = ut_gen.genFixBin(binSize=200, instances=FdataMDF)
print("fix 3")
fixBinInstnace = ut_gen.genFixBin(binSize=initialBin,
instances=FdataMDF)
print("fix {}".format(initialBin))
# jsMDF, divMDF = ut_bokeh.JSandDivMutliplePlot(x_axis=x_axis,
# y_axisList = [FdataMDF,sketch3Instances,sketchInstances,bin3Instnace,fixBinInstnace],
# circleList = [True,False,False,False,False],
# legend_labelList = ['Raw','DyBin3','DyBin{}'.format(initialBin),'Bin size = 3','Bin size = {}'.format(initialBin)],
# line_alphaList = [0.4,1,1,1,1],
# title=fileName)
jsMDF, divMDF = ut_bokeh.JSandDivMutliplePlot(
x_axis=x_axis,
y_axisList=[FdataMDF, sketchInstances, bin3Instnace],
circleList=[True, False, False],
legend_labelList=['Raw', 'DyBin', 'Bin size = 200'],
line_alphaList=[0.4, 1, 1],
title=fileName)
core = com_service(inputList=FdataMDF)
# dictResult = core.TWINcurveBinSize(maxBinSize=len(x_axis))
# jsCom, divCom = ut_bokeh.JSandDivComPlot(binSizeList=dictResult["binSizeList"], comList=dictResult["comList"],
# varList=dictResult["varList"])
# return render_template("html/ratio/ratio_plot.html", jsMDF=jsMDF, divMDF=divMDF
# , jsCom=jsCom, divCom=divCom, listFiles=listFiles)
dictResult = core.TWINcurveWindowSize()
# dictResult = core.TWINcurveWindowSizeFixBin(timestamps=FtimestampMDF)
jsCom, divCom = ut_bokeh.JSandDivComPlotTemp(
binSizeList=dictResult["binSizeList"],
comList=dictResult["comList"],
varList=dictResult["varList"])
return render_template("html/ratio/ratio_plot.html",
jsMDF=jsMDF,
divMDF=divMDF,
jsCom=jsCom,
divCom=divCom,
listFiles=listFiles)
def MDFPlot():
listFiles = ut_mdf.getListMDF()
if request.method == 'POST':
fileName = str(request.form.get('selectFile'))
FdataMDF, FtimestampMDF = ut_service.getMDFData(fileName)
# sketch
windowSize = int(request.form.get('inputWindowSize'))
initialBin = int(request.form.get('inputInitialBin'))
inputOnline = request.form.get('inputOnline')
# if inputOnline == "True":
# isOnline = True
# else:
# isOnline = False
isOnline = False
# print("**************{}***********".format(isOnline))
corePlot = sketchDyBinService(windowSize=windowSize,
initialBin=initialBin,
isOnline=isOnline)
threshold_tTest = float(request.form.get('threshold_tTest'))
threshold_fTest = float(request.form.get('threshold_fTest'))
sketchInstances = corePlot.sketchMode(instances=FdataMDF)
indexBinList = corePlot.getBinNameInstances()
getListMeanBin = corePlot.getListMeanBin()
print(getListMeanBin)
dictResult = corePlot.getListResultTest()
js, div = ut_bokeh.JSandDivRawAndBin(x_axis=FtimestampMDF,
y_raw=FdataMDF,
y_axisList=[sketchInstances],
legend_labelList=["dyBin"],
namesList=[indexBinList],
title="{} window size :{}".format(
fileName, windowSize))
return render_template("html/mdf/mdf_plot.html",
jsMDF=js,
divMDF=div,
listFiles=listFiles)
import utility.utility_db as ut_db
import utility.utility_webservice as ut_service
import utility.utility_genData as ut_gen
import utility.utility_astronomy as ut_astro
import utility.utility_exportBokeh as ut_bokeh
inis_bin = [3, 5]
windows_size = [10, 20]
if __name__ == '__main__':
list_files = ut_db.getAnswerMDF_toDB()
for index, row in enumerate(list_files):
file_name = row['file_name']
FdataMDF, FtimestampMDF = ut_service.getMDFData(file_name)
UTC = ut_astro.convertList_MJDtoUTC(FtimestampMDF)
raw = {"timestamp": FtimestampMDF, 'instances': FdataMDF, 'UTC': UTC}
rows_data = []
for ini_bin in inis_bin:
for window_size in windows_size:
FdataMDF_dy = ut_gen.genListDyBin(instances=FdataMDF,
timestamp=FtimestampMDF,
windowSize=window_size,
isOnline=False,
initialBin=ini_bin)
rows_data.append({
"timestamp":
FtimestampMDF,
'instances':
FdataMDF_dy['sketchInstances'],
'UTC':
import utility.utility_webservice as ut_service
from scipy.stats import linregress
import matplotlib.pyplot as plt
from math import ceil
from data_structure.Bin.clusterBin import clus_binning as topBin
import utility.utility as ut
from bokeh.plotting import figure, show
from bokeh.models import ColumnDataSource
numberSegment = 20
# mainFile = "light_curve_Gaia-DR2_3314137846679343872_date20191205" # normal
# mainFile = "light_curve_Gaia-DR2_3410076150374417408_date20191203" # small
mainFile = "light_curve_Gaia-DR2_146236558322961024_date20191205" # sim
mainRaw, mainTime = ut_service.getMDFData(mainFile)
resultSeg = ut.genSegmentIndex(lenInput=mainTime, numberSegment=numberSegment)
window = []
line = []
for index in resultSeg["resultList"]:
bin = topBin()
bin.set_instances(instances=mainRaw[index["start"]:index["end"]],
timestamps=mainTime[index["start"]:index["end"]])
line = line + bin.genLine()
indexBin = [*range(1, numberSegment, 1)]
TOOLTIPS = [
("index", "$index"),
("(x,y)", "(@x{0,0}, @y{0,0.000}"),
]
plot = figure(tooltips=TOOLTIPS,
def clusterPlot():
listFiles = ut_mdf.getListMDF()
if request.method == 'POST':
windowSize = int(request.form.get('inputWindowSize'))
fileName = str(request.form.get('selectFile'))
isDybin = request.form.get('inputDybin')
initialBin = int(request.form.get('inputInitialBin'))
FdataMDF, FtimestampMDF = ut_service.getMDFData(fileName)
corePlot = sketchDyBinService(windowSize=windowSize,
initialBin=5,
isOnline=False)
sketchInstances = corePlot.sketchMode(instances=FdataMDF)
indexBinList = corePlot.getBinNameInstances()
window = corePlot.getWindow()
indexTimestamp = 0
segment3D = []
for index, bin in enumerate(window):
raw = bin.get_raw()
start = copy.copy(indexTimestamp)
indexTimestamp = indexTimestamp + len(raw)
tempTimestamp = FtimestampMDF[start:indexTimestamp]
slope = ut_cal.calSlope(timestamps=tempTimestamp, instances=raw)
result = {
"raw": raw,
"index": index,
"variance": bin.get_variance(),
"mean": bin.get_representation(),
"slope": slope
}
segment3D.append(result)
cluster = ut_cluster.cluster_xMean_Dy(binsList=window,
kmax=windowSize,
hist_bin=20)
ut_plotly.plot3DClusterDYTemp(samples=cluster["samples"],
clusters=cluster["clusters"],
centers=cluster["centers"])
resultCluster = ut_gen.genHistrogram_Dict(binsList=segment3D,
hist_bin=20)
jsMDF, divMDF = ut_bokeh.JSandDivRawAndBin(
x_axis=FtimestampMDF,
y_raw=FdataMDF,
y_axisList=[sketchInstances],
legend_labelList=["dyBin"],
namesList=[indexBinList],
title="{} window size :{}".format(fileName, windowSize))
jsCl, divCl = ut_bokeh.JSandDivHistrogramCluster(
meansClass=resultCluster['meansClass'],
varsClass=resultCluster['varsClass'],
slopesClass=resultCluster['slopesClass'])
return render_template("html/cluster/cluster_plot.html",
listFiles=listFiles,
jsMDF=jsMDF,
divMDF=divMDF,
jsCl=jsCl,
divCl=divCl)
import utility.utility_genData as ut_gen
import utility.utility_calculator as ut_cal
import utility.utility as ut
ini_bin = 5
# window_size = 10
# sql_file = 'getFileForSim.sql'
sql_file = 'getFileForSim2.sql'
distance = 2000
if __name__ == '__main__':
list_files = ut_db.getFileListFromSQL(sqlFileName=sql_file)
for index, main_file in enumerate(list_files):
FdataMain, FtimestampMain = ut_service.getMDFData(main_file)
windowMain = int(len(FdataMain) / 50)
mainDy = ut_gen.genListDyBin(instances=FdataMain,
windowSize=windowMain,
timestamp=FtimestampMain,
initialBin=ini_bin)
near_files = ut_db.getNearStars(fileName=main_file,
maxDistance=distance)
rows_data = []
for near_file in near_files:
try:
FdataMDF, FtimestampMDF = ut_service.getMDFData(
near_file["file_target"])
windowTarget = int(len(FdataMDF) / 50)
targetDy = ut_gen.genListDyBin(instances=FdataMDF,
timestamp=FtimestampMDF,
ini_bin = 5
window_size = 10
sql_file = 'select_top100.sql'
distance = 2000
if __name__ == '__main__':
list_files = ut_db.getFileListFromSQL(sqlFileName=sql_file)
for index,main_file in enumerate(list_files[11:51]):
near_files = ut_db.getNearStars(fileName=main_file, maxDistance=distance)
rows_data = []
for near_file in near_files:
FdataMDF, FtimestampMDF = ut_service.getMDFData(near_file["file_target"])
UTC = ut_astro.convertList_MJDtoUTC(FtimestampMDF)
FdataMDF_dy = ut_gen.genListDyBin(instances=FdataMDF, timestamp=FtimestampMDF,
windowSize=window_size, isOnline=False,
initialBin=ini_bin)
rows_data.append({"timestamp": FtimestampMDF,
'instances': FdataMDF, 'UTC': UTC,
'title_name': near_file["file_target"].replace('light_curve_Gaia-DR2_','')})
rows_data.append({"timestamp": FtimestampMDF,
'instances': FdataMDF_dy['sketchInstances'], 'UTC': UTC,
'title_name': near_file["file_target"].replace('light_curve_Gaia-DR2_','dy_')})
ut_bokeh.export_HTML_mutilples(fileName= "{:02}_{}.html".format(index+11,main_file),
rows_data=rows_data,line_width=3,
from scipy.stats import ks_2samp
import utility.utility as ut
ini_bin = 5
window_size = 10
sql_file = 'select_grubbs_win10.sql'
distance = 2000
if __name__ == '__main__':
list_files = ut_db.getFileListFromSQL(sqlFileName=sql_file)
for index,file_name in enumerate(list_files):
row_result = []
FdataMDF, FtimestampMDF = ut_service.getMDFData(file_name)
UTC = ut_astro.convertList_MJDtoUTC(FtimestampMDF)
# raw = {"timestamp": FtimestampMDF, 'instances': FdataMDF, 'UTC': UTC}
# rows_data = []
near_files = ut_db.getNearStars(fileName=file_name,maxDistance=distance)
FdataMDF_dy = ut_gen.genListDyBin(instances=FdataMDF, timestamp=FtimestampMDF,
windowSize=window_size, isOnline=False,
initialBin=ini_bin)
a = FdataMDF_dy['sketchInstances']
for near_file in near_files:
FdataMDF, FtimestampMDF = ut_service.getMDFData(near_file['file_target'])
UTC = ut_astro.convertList_MJDtoUTC(FtimestampMDF)
target_dy = ut_gen.genListDyBin(instances=FdataMDF, timestamp=FtimestampMDF,
windowSize=window_size, isOnline=False,
initialBin=ini_bin)
# listFile = getListTop()
# listFile = ["light_curve_Gaia-DR2_3314137846679343872_date20191205"]
# fullFile = getListFile()
# listFile = getListFile(start="light_curve_Gaia-DR2_2029297284882370944_date20191005")
fullFile = utility.utility_mdf.getListMDF(pattern="date20191205")
#
# fullFile = fullFile[ut.getStartIndexFromString(fullFile,"light_curve_Gaia-DR2_3408221441761775872_date20191205"):]
listFile = utility.utility_mdf.getListMDF(pattern="date20191205")
listFile = listFile[ut.getStartIndexFromString(
fullFile, "light_curve_Gaia-DR2_3314137846679343872_date20191205"):]
for i, mainFile in enumerate(listFile):
patterMainDate = mainFile.split("_date")[1]
mainRaw, mainTime = ut_service.getMDFData(mainFile)
# corePlot = sketchService(windowSize=windowSize, initialBin=initialBin, isOnline=False)
# mainInput = corePlot.sketchMode(instances=mainRaw)
mainInput = ut_mdf.getDydata(mainFile)
for j, targetFile in enumerate(fullFile):
patterTargetDate = targetFile.split("_date")[1]
if i != j and patterTargetDate == patterMainDate:
targetRaw, targetTime = ut_service.getMDFData(targetFile)
if checkPeriod(startMain=mainTime[0],
endMain=mainTime[-1],
startTarget=targetTime[0],
endTarget=targetTime[-1]):
# corePlot = sketchService(windowSize=windowSize, initialBin=initialBin, isOnline=False)
# targetInput = corePlot.sketchMode(instances=targetRaw)
targetInput = ut_mdf.getDydata(targetFile)
