def ratioPlot(): listFiles = utility.utility_mdf.getListMDF() if request.method == 'POST': fileName = str(request.form.get('selectFile')) FdataMDF, FtimestampMDF = ut_service.getMDFData(fileName) # sketch windowSize = int(request.form.get('inputWindowSize')) initialBin = int(request.form.get('inputInitialBin')) x_axis = FtimestampMDF core = coreMean(inputList=FdataMDF) dictResult = core.getCompressFixWindow(windowSize=windowSize) a = core.getAccuracy(windowSize=windowSize) print("com w_{} = {} ".format(windowSize, core.getAccuracy(windowSize=windowSize))) js, div = ut_bokeh.JSandDivDoublePlotStandard( x_axis=x_axis, y_axisRaw=FdataMDF, y_axisBin=dictResult["compressInstances"]) return render_template("html/ratio/ratio_plot.html", jsMDF=js, divMDF=div, jsCom="", divCom="", listFiles=listFiles)
def xmeanPlot(): listFiles = ut_service.getListLC_test() if request.method == 'POST': fileName = str(request.form.get('selectFile')) amount_initial_centers = int( request.form.get('amount_initial_centers')) kmax = int(request.form.get('kmax')) FdataMDF, FtimestampMDF = ut_service.getMDFData(fileName) resultDic = ut_cluster.cluster_xMean( listInput=FdataMDF, amount_initial_centers=amount_initial_centers, kmax=kmax) TOOLTIPS = [ ("index", "$index"), ("(x,y)", "@x{0,0.000000}, $y"), ] js, div = ut_bokeh.JSandDivXMeanPlot(x_axis=FtimestampMDF, y_axisRaw=FdataMDF, resultDic=resultDic, title=fileName, TOOLTIPS=TOOLTIPS) # js, div = ut_bokeh.JSandDivDoublePlot(x_axis=FtimestampMDF, y_axisRaw=FdataMDF, y_axisBin=sketchInstances, # title=fileName, transList=transList,alarmList=alarmList) # # rows = corePlot.getDataTableMeanVarianceBin() return render_template("html/LC_test/lc_test_plot.html", jsResult=js, divResult=div, listFiles=listFiles)
def dy_tran(): if request.method == 'POST': listKeys = [ 'inputWindowSize', 'inputInitialBin', 'isOnline', 'fileName' ] inputKeys = request.form.keys() if ut_service.isCompleteInputKey(listKeys=listKeys, inputKeys=inputKeys): FdataMDF, FtimestampMDF = ut_service.getMDFData( request.form.get('fileName')) windowSize = int(request.form.get('inputWindowSize')) initialBin = int(request.form.get('inputInitialBin')) inputOnline = bool(request.form.get('isOnline')) corePlot = sketchDyBinService(windowSize=windowSize, initialBin=initialBin, isOnline=inputOnline) sketchInstances = corePlot.sketchMode(instances=FdataMDF) indexBinList = corePlot.getBinNameInstances() window = corePlot.getWindow() else: return jsonify( status="404", message="Keys are incorrect", ) return render_template("html/mdf/mdf_main.html", listFiles=[])
def ratioPlot(): listFiles = ut_mdf.getListMDF() if request.method == 'POST': fileName = str(request.form.get('selectFile')) FdataMDF, FtimestampMDF = ut_service.getMDFData(fileName) # sketch windowSize = int(request.form.get('inputWindowSize')) initialBin = int(request.form.get('inputInitialBin')) x_axis = FtimestampMDF # corePlot = sketchService(windowSize=windowSize, initialBin=3) # sketch3Instances = corePlot.sketchMode(instances=FdataMDF) corePlot = sketchDyBinService(windowSize=windowSize, initialBin=3) sketchInstances = corePlot.sketchMode(instances=FdataMDF) print("sketch {}".format(initialBin)) bin3Instnace = ut_gen.genFixBin(binSize=200, instances=FdataMDF) print("fix 3") fixBinInstnace = ut_gen.genFixBin(binSize=initialBin, instances=FdataMDF) print("fix {}".format(initialBin)) # jsMDF, divMDF = ut_bokeh.JSandDivMutliplePlot(x_axis=x_axis, # y_axisList = [FdataMDF,sketch3Instances,sketchInstances,bin3Instnace,fixBinInstnace], # circleList = [True,False,False,False,False], # legend_labelList = ['Raw','DyBin3','DyBin{}'.format(initialBin),'Bin size = 3','Bin size = {}'.format(initialBin)], # line_alphaList = [0.4,1,1,1,1], # title=fileName) jsMDF, divMDF = ut_bokeh.JSandDivMutliplePlot( x_axis=x_axis, y_axisList=[FdataMDF, sketchInstances, bin3Instnace], circleList=[True, False, False], legend_labelList=['Raw', 'DyBin', 'Bin size = 200'], line_alphaList=[0.4, 1, 1], title=fileName) core = com_service(inputList=FdataMDF) # dictResult = core.TWINcurveBinSize(maxBinSize=len(x_axis)) # jsCom, divCom = ut_bokeh.JSandDivComPlot(binSizeList=dictResult["binSizeList"], comList=dictResult["comList"], # varList=dictResult["varList"]) # return render_template("html/ratio/ratio_plot.html", jsMDF=jsMDF, divMDF=divMDF # , jsCom=jsCom, divCom=divCom, listFiles=listFiles) dictResult = core.TWINcurveWindowSize() # dictResult = core.TWINcurveWindowSizeFixBin(timestamps=FtimestampMDF) jsCom, divCom = ut_bokeh.JSandDivComPlotTemp( binSizeList=dictResult["binSizeList"], comList=dictResult["comList"], varList=dictResult["varList"]) return render_template("html/ratio/ratio_plot.html", jsMDF=jsMDF, divMDF=divMDF, jsCom=jsCom, divCom=divCom, listFiles=listFiles)
def MDFPlot(): listFiles = ut_mdf.getListMDF() if request.method == 'POST': fileName = str(request.form.get('selectFile')) FdataMDF, FtimestampMDF = ut_service.getMDFData(fileName) # sketch windowSize = int(request.form.get('inputWindowSize')) initialBin = int(request.form.get('inputInitialBin')) inputOnline = request.form.get('inputOnline') # if inputOnline == "True": # isOnline = True # else: # isOnline = False isOnline = False # print("**************{}***********".format(isOnline)) corePlot = sketchDyBinService(windowSize=windowSize, initialBin=initialBin, isOnline=isOnline) threshold_tTest = float(request.form.get('threshold_tTest')) threshold_fTest = float(request.form.get('threshold_fTest')) sketchInstances = corePlot.sketchMode(instances=FdataMDF) indexBinList = corePlot.getBinNameInstances() getListMeanBin = corePlot.getListMeanBin() print(getListMeanBin) dictResult = corePlot.getListResultTest() js, div = ut_bokeh.JSandDivRawAndBin(x_axis=FtimestampMDF, y_raw=FdataMDF, y_axisList=[sketchInstances], legend_labelList=["dyBin"], namesList=[indexBinList], title="{} window size :{}".format( fileName, windowSize)) return render_template("html/mdf/mdf_plot.html", jsMDF=js, divMDF=div, listFiles=listFiles)
import utility.utility_db as ut_db import utility.utility_webservice as ut_service import utility.utility_genData as ut_gen import utility.utility_astronomy as ut_astro import utility.utility_exportBokeh as ut_bokeh inis_bin = [3, 5] windows_size = [10, 20] if __name__ == '__main__': list_files = ut_db.getAnswerMDF_toDB() for index, row in enumerate(list_files): file_name = row['file_name'] FdataMDF, FtimestampMDF = ut_service.getMDFData(file_name) UTC = ut_astro.convertList_MJDtoUTC(FtimestampMDF) raw = {"timestamp": FtimestampMDF, 'instances': FdataMDF, 'UTC': UTC} rows_data = [] for ini_bin in inis_bin: for window_size in windows_size: FdataMDF_dy = ut_gen.genListDyBin(instances=FdataMDF, timestamp=FtimestampMDF, windowSize=window_size, isOnline=False, initialBin=ini_bin) rows_data.append({ "timestamp": FtimestampMDF, 'instances': FdataMDF_dy['sketchInstances'], 'UTC':
import utility.utility_webservice as ut_service from scipy.stats import linregress import matplotlib.pyplot as plt from math import ceil from data_structure.Bin.clusterBin import clus_binning as topBin import utility.utility as ut from bokeh.plotting import figure, show from bokeh.models import ColumnDataSource numberSegment = 20 # mainFile = "light_curve_Gaia-DR2_3314137846679343872_date20191205" # normal # mainFile = "light_curve_Gaia-DR2_3410076150374417408_date20191203" # small mainFile = "light_curve_Gaia-DR2_146236558322961024_date20191205" # sim mainRaw, mainTime = ut_service.getMDFData(mainFile) resultSeg = ut.genSegmentIndex(lenInput=mainTime, numberSegment=numberSegment) window = [] line = [] for index in resultSeg["resultList"]: bin = topBin() bin.set_instances(instances=mainRaw[index["start"]:index["end"]], timestamps=mainTime[index["start"]:index["end"]]) line = line + bin.genLine() indexBin = [*range(1, numberSegment, 1)] TOOLTIPS = [ ("index", "$index"), ("(x,y)", "(@x{0,0}, @y{0,0.000}"), ] plot = figure(tooltips=TOOLTIPS,
def clusterPlot(): listFiles = ut_mdf.getListMDF() if request.method == 'POST': windowSize = int(request.form.get('inputWindowSize')) fileName = str(request.form.get('selectFile')) isDybin = request.form.get('inputDybin') initialBin = int(request.form.get('inputInitialBin')) FdataMDF, FtimestampMDF = ut_service.getMDFData(fileName) corePlot = sketchDyBinService(windowSize=windowSize, initialBin=5, isOnline=False) sketchInstances = corePlot.sketchMode(instances=FdataMDF) indexBinList = corePlot.getBinNameInstances() window = corePlot.getWindow() indexTimestamp = 0 segment3D = [] for index, bin in enumerate(window): raw = bin.get_raw() start = copy.copy(indexTimestamp) indexTimestamp = indexTimestamp + len(raw) tempTimestamp = FtimestampMDF[start:indexTimestamp] slope = ut_cal.calSlope(timestamps=tempTimestamp, instances=raw) result = { "raw": raw, "index": index, "variance": bin.get_variance(), "mean": bin.get_representation(), "slope": slope } segment3D.append(result) cluster = ut_cluster.cluster_xMean_Dy(binsList=window, kmax=windowSize, hist_bin=20) ut_plotly.plot3DClusterDYTemp(samples=cluster["samples"], clusters=cluster["clusters"], centers=cluster["centers"]) resultCluster = ut_gen.genHistrogram_Dict(binsList=segment3D, hist_bin=20) jsMDF, divMDF = ut_bokeh.JSandDivRawAndBin( x_axis=FtimestampMDF, y_raw=FdataMDF, y_axisList=[sketchInstances], legend_labelList=["dyBin"], namesList=[indexBinList], title="{} window size :{}".format(fileName, windowSize)) jsCl, divCl = ut_bokeh.JSandDivHistrogramCluster( meansClass=resultCluster['meansClass'], varsClass=resultCluster['varsClass'], slopesClass=resultCluster['slopesClass']) return render_template("html/cluster/cluster_plot.html", listFiles=listFiles, jsMDF=jsMDF, divMDF=divMDF, jsCl=jsCl, divCl=divCl)
import utility.utility_genData as ut_gen import utility.utility_calculator as ut_cal import utility.utility as ut ini_bin = 5 # window_size = 10 # sql_file = 'getFileForSim.sql' sql_file = 'getFileForSim2.sql' distance = 2000 if __name__ == '__main__': list_files = ut_db.getFileListFromSQL(sqlFileName=sql_file) for index, main_file in enumerate(list_files): FdataMain, FtimestampMain = ut_service.getMDFData(main_file) windowMain = int(len(FdataMain) / 50) mainDy = ut_gen.genListDyBin(instances=FdataMain, windowSize=windowMain, timestamp=FtimestampMain, initialBin=ini_bin) near_files = ut_db.getNearStars(fileName=main_file, maxDistance=distance) rows_data = [] for near_file in near_files: try: FdataMDF, FtimestampMDF = ut_service.getMDFData( near_file["file_target"]) windowTarget = int(len(FdataMDF) / 50) targetDy = ut_gen.genListDyBin(instances=FdataMDF, timestamp=FtimestampMDF,
ini_bin = 5 window_size = 10 sql_file = 'select_top100.sql' distance = 2000 if __name__ == '__main__': list_files = ut_db.getFileListFromSQL(sqlFileName=sql_file) for index,main_file in enumerate(list_files[11:51]): near_files = ut_db.getNearStars(fileName=main_file, maxDistance=distance) rows_data = [] for near_file in near_files: FdataMDF, FtimestampMDF = ut_service.getMDFData(near_file["file_target"]) UTC = ut_astro.convertList_MJDtoUTC(FtimestampMDF) FdataMDF_dy = ut_gen.genListDyBin(instances=FdataMDF, timestamp=FtimestampMDF, windowSize=window_size, isOnline=False, initialBin=ini_bin) rows_data.append({"timestamp": FtimestampMDF, 'instances': FdataMDF, 'UTC': UTC, 'title_name': near_file["file_target"].replace('light_curve_Gaia-DR2_','')}) rows_data.append({"timestamp": FtimestampMDF, 'instances': FdataMDF_dy['sketchInstances'], 'UTC': UTC, 'title_name': near_file["file_target"].replace('light_curve_Gaia-DR2_','dy_')}) ut_bokeh.export_HTML_mutilples(fileName= "{:02}_{}.html".format(index+11,main_file), rows_data=rows_data,line_width=3,
from scipy.stats import ks_2samp import utility.utility as ut ini_bin = 5 window_size = 10 sql_file = 'select_grubbs_win10.sql' distance = 2000 if __name__ == '__main__': list_files = ut_db.getFileListFromSQL(sqlFileName=sql_file) for index,file_name in enumerate(list_files): row_result = [] FdataMDF, FtimestampMDF = ut_service.getMDFData(file_name) UTC = ut_astro.convertList_MJDtoUTC(FtimestampMDF) # raw = {"timestamp": FtimestampMDF, 'instances': FdataMDF, 'UTC': UTC} # rows_data = [] near_files = ut_db.getNearStars(fileName=file_name,maxDistance=distance) FdataMDF_dy = ut_gen.genListDyBin(instances=FdataMDF, timestamp=FtimestampMDF, windowSize=window_size, isOnline=False, initialBin=ini_bin) a = FdataMDF_dy['sketchInstances'] for near_file in near_files: FdataMDF, FtimestampMDF = ut_service.getMDFData(near_file['file_target']) UTC = ut_astro.convertList_MJDtoUTC(FtimestampMDF) target_dy = ut_gen.genListDyBin(instances=FdataMDF, timestamp=FtimestampMDF, windowSize=window_size, isOnline=False, initialBin=ini_bin)
# listFile = getListTop() # listFile = ["light_curve_Gaia-DR2_3314137846679343872_date20191205"] # fullFile = getListFile() # listFile = getListFile(start="light_curve_Gaia-DR2_2029297284882370944_date20191005") fullFile = utility.utility_mdf.getListMDF(pattern="date20191205") # # fullFile = fullFile[ut.getStartIndexFromString(fullFile,"light_curve_Gaia-DR2_3408221441761775872_date20191205"):] listFile = utility.utility_mdf.getListMDF(pattern="date20191205") listFile = listFile[ut.getStartIndexFromString( fullFile, "light_curve_Gaia-DR2_3314137846679343872_date20191205"):] for i, mainFile in enumerate(listFile): patterMainDate = mainFile.split("_date")[1] mainRaw, mainTime = ut_service.getMDFData(mainFile) # corePlot = sketchService(windowSize=windowSize, initialBin=initialBin, isOnline=False) # mainInput = corePlot.sketchMode(instances=mainRaw) mainInput = ut_mdf.getDydata(mainFile) for j, targetFile in enumerate(fullFile): patterTargetDate = targetFile.split("_date")[1] if i != j and patterTargetDate == patterMainDate: targetRaw, targetTime = ut_service.getMDFData(targetFile) if checkPeriod(startMain=mainTime[0], endMain=mainTime[-1], startTarget=targetTime[0], endTarget=targetTime[-1]): # corePlot = sketchService(windowSize=windowSize, initialBin=initialBin, isOnline=False) # targetInput = corePlot.sketchMode(instances=targetRaw) targetInput = ut_mdf.getDydata(targetFile)