def ratioPlot(): listFiles = ut_mdf.getListMDF() if request.method == 'POST': fileName = str(request.form.get('selectFile')) FdataMDF, FtimestampMDF = ut_service.getMDFData(fileName) # sketch windowSize = int(request.form.get('inputWindowSize')) initialBin = int(request.form.get('inputInitialBin')) x_axis = FtimestampMDF # corePlot = sketchService(windowSize=windowSize, initialBin=3) # sketch3Instances = corePlot.sketchMode(instances=FdataMDF) corePlot = sketchDyBinService(windowSize=windowSize, initialBin=3) sketchInstances = corePlot.sketchMode(instances=FdataMDF) print("sketch {}".format(initialBin)) bin3Instnace = ut_gen.genFixBin(binSize=200, instances=FdataMDF) print("fix 3") fixBinInstnace = ut_gen.genFixBin(binSize=initialBin, instances=FdataMDF) print("fix {}".format(initialBin)) # jsMDF, divMDF = ut_bokeh.JSandDivMutliplePlot(x_axis=x_axis, # y_axisList = [FdataMDF,sketch3Instances,sketchInstances,bin3Instnace,fixBinInstnace], # circleList = [True,False,False,False,False], # legend_labelList = ['Raw','DyBin3','DyBin{}'.format(initialBin),'Bin size = 3','Bin size = {}'.format(initialBin)], # line_alphaList = [0.4,1,1,1,1], # title=fileName) jsMDF, divMDF = ut_bokeh.JSandDivMutliplePlot( x_axis=x_axis, y_axisList=[FdataMDF, sketchInstances, bin3Instnace], circleList=[True, False, False], legend_labelList=['Raw', 'DyBin', 'Bin size = 200'], line_alphaList=[0.4, 1, 1], title=fileName) core = com_service(inputList=FdataMDF) # dictResult = core.TWINcurveBinSize(maxBinSize=len(x_axis)) # jsCom, divCom = ut_bokeh.JSandDivComPlot(binSizeList=dictResult["binSizeList"], comList=dictResult["comList"], # varList=dictResult["varList"]) # return render_template("html/ratio/ratio_plot.html", jsMDF=jsMDF, divMDF=divMDF # , jsCom=jsCom, divCom=divCom, listFiles=listFiles) dictResult = core.TWINcurveWindowSize() # dictResult = core.TWINcurveWindowSizeFixBin(timestamps=FtimestampMDF) jsCom, divCom = ut_bokeh.JSandDivComPlotTemp( binSizeList=dictResult["binSizeList"], comList=dictResult["comList"], varList=dictResult["varList"]) return render_template("html/ratio/ratio_plot.html", jsMDF=jsMDF, divMDF=divMDF, jsCom=jsCom, divCom=divCom, listFiles=listFiles)
def MDFPlot(): listFiles = ut_mdf.getListMDF() if request.method == 'POST': fileName = str(request.form.get('selectFile')) FdataMDF, FtimestampMDF = ut_service.getMDFData(fileName) # sketch windowSize = int(request.form.get('inputWindowSize')) initialBin = int(request.form.get('inputInitialBin')) inputOnline = request.form.get('inputOnline') # if inputOnline == "True": # isOnline = True # else: # isOnline = False isOnline = False # print("**************{}***********".format(isOnline)) corePlot = sketchDyBinService(windowSize=windowSize, initialBin=initialBin, isOnline=isOnline) threshold_tTest = float(request.form.get('threshold_tTest')) threshold_fTest = float(request.form.get('threshold_fTest')) sketchInstances = corePlot.sketchMode(instances=FdataMDF) indexBinList = corePlot.getBinNameInstances() getListMeanBin = corePlot.getListMeanBin() print(getListMeanBin) dictResult = corePlot.getListResultTest() js, div = ut_bokeh.JSandDivRawAndBin(x_axis=FtimestampMDF, y_raw=FdataMDF, y_axisList=[sketchInstances], legend_labelList=["dyBin"], namesList=[indexBinList], title="{} window size :{}".format( fileName, windowSize)) return render_template("html/mdf/mdf_plot.html", jsMDF=js, divMDF=div, listFiles=listFiles)
import json import utility.utility_mdf as ut_mdf from astropy.time import Time import utility.utility_db as ut_db dataset_path = 'D:\\mdwarf_data\\' lc_path = "lc_flux_catalog_aperture_r7_txt\\" lc_timestamp_path = "lc_timestamp_txt\\" path_to_lc_file = "{}{}".format(dataset_path, lc_path) if __name__ == '__main__': fileList = ut_mdf.getListMDF() rows = [] for index, fileName in enumerate(fileList): data = ut_mdf.getDataFromFile(fileName=fileName) fileDate = fileName.split('_date')[1] startTimeMJD = data["timestamp"][0] startTimeUTC = Time(startTimeMJD, format='mjd').iso endTimeMJD = data["timestamp"][-1] endTimeUTC = Time(endTimeMJD, format='mjd').iso rows.append((fileName, fileDate, startTimeMJD, endTimeMJD, startTimeUTC, endTimeUTC)) ut_db.insertMJDRow(rows=rows) print("a")
import utility.utility_astronomy as ut_astro import utility.utility_exportBokeh as ut_ex dataset_path = 'D:\\mdwarf_data\\' lc_path = "lc_flux_catalog_aperture_r7_txt\\" lc_timestamp_path = "lc_timestamp_txt\\" path_to_lc_file = "{}{}".format(dataset_path, lc_path) html_path = "lc_flux_catalog_aperture_r7_html\\" png_path = "lc_flux_catalog_aperture_r7_png\\" # save file main_save_path = '{}\\mdwarf_data_common\\'.format(dataset_path) # LC if __name__ == '__main__': listFile = ut_mdf.getListMDF() for indexFile, fileName in enumerate(listFile): print(fileName) LCFile = "{}{}.txt".format(path_to_lc_file, fileName) filesize = os.path.getsize(LCFile) if filesize != 0: fileDate = fileName.split("_date")[1] # Save LC to Common LC file = open(LCFile, 'rb') FdataMDF = [float(item) for item in ut.txt_to_list(LCFile)] save_lc = "{}{}\\{}\\".format(main_save_path, "lc_flux_catalog_aperture_r7", fileDate) ut.checkFolderandCreate(save_lc) save_lc_fileName = "{}{}.txt".format(save_lc, fileName)
def ratioInput(): listFiles = ut_mdf.getListMDF() return render_template("html/ratio/ratio_input.html", listFiles=listFiles)
def ratioMain(): listFiles = ut_mdf.getListMDF() return render_template("html/ratio/ratio_main.html", listFiles=listFiles)
def xmeanMain(): listFiles = ut_mdf.getListMDF() return render_template("html/xmean/xmean_main.html", listFiles=listFiles)
def MDFMain(): listFiles = ut_mdf.getListMDF() return render_template("html/mdf/mdf_main.html", listFiles=listFiles)
def clusterPlot(): listFiles = ut_mdf.getListMDF() if request.method == 'POST': windowSize = int(request.form.get('inputWindowSize')) fileName = str(request.form.get('selectFile')) isDybin = request.form.get('inputDybin') initialBin = int(request.form.get('inputInitialBin')) FdataMDF, FtimestampMDF = ut_service.getMDFData(fileName) corePlot = sketchDyBinService(windowSize=windowSize, initialBin=5, isOnline=False) sketchInstances = corePlot.sketchMode(instances=FdataMDF) indexBinList = corePlot.getBinNameInstances() window = corePlot.getWindow() indexTimestamp = 0 segment3D = [] for index, bin in enumerate(window): raw = bin.get_raw() start = copy.copy(indexTimestamp) indexTimestamp = indexTimestamp + len(raw) tempTimestamp = FtimestampMDF[start:indexTimestamp] slope = ut_cal.calSlope(timestamps=tempTimestamp, instances=raw) result = { "raw": raw, "index": index, "variance": bin.get_variance(), "mean": bin.get_representation(), "slope": slope } segment3D.append(result) cluster = ut_cluster.cluster_xMean_Dy(binsList=window, kmax=windowSize, hist_bin=20) ut_plotly.plot3DClusterDYTemp(samples=cluster["samples"], clusters=cluster["clusters"], centers=cluster["centers"]) resultCluster = ut_gen.genHistrogram_Dict(binsList=segment3D, hist_bin=20) jsMDF, divMDF = ut_bokeh.JSandDivRawAndBin( x_axis=FtimestampMDF, y_raw=FdataMDF, y_axisList=[sketchInstances], legend_labelList=["dyBin"], namesList=[indexBinList], title="{} window size :{}".format(fileName, windowSize)) jsCl, divCl = ut_bokeh.JSandDivHistrogramCluster( meansClass=resultCluster['meansClass'], varsClass=resultCluster['varsClass'], slopesClass=resultCluster['slopesClass']) return render_template("html/cluster/cluster_plot.html", listFiles=listFiles, jsMDF=jsMDF, divMDF=divMDF, jsCl=jsCl, divCl=divCl)
def clusterMain(): listFiles = ut_mdf.getListMDF() return render_template("html/cluster/cluster_main.html", listFiles=listFiles)
def clusterInput(): listFiles = ut_mdf.getListMDF() return render_template("html/cluster/cluster_input.html", listFiles=listFiles)
if __name__ == '__main__': # windowSizeList = [500,250,100,50,20,10,5] # datePatterns = ['20191130', '20191203', '20191205', '20191224', '20200130','20200203'] # datePatterns = ['20190812', '20191004', '20191005', # '20191006', '20191106', '20191120', # '20191121', '20191123', '20191121', # '20191123', '20191124', '20191129', # '20191130', '20191202', '20191203', # '20191204', '20191205', '20191224', # '20191226', '20200129', '20200130', # '20200201', '20200202', '20200203'] datePatterns = [ '20190812', '20191004', '20191005', '20191006', '20191106', '20191120', '20191121', '20191123', '20191121', '20191123', '20191124', '20191129', '20191202', '20191204', '20191226', '20200129', '20200201', '20200202' ] for datePattern in datePatterns: fileList = ut_mdf.getListMDF(pattern=datePattern) for fileName in fileList[1:]: lightData = ut_mdf.getDataFromFile(fileName=fileName) windowSizeList = genWindowList(len(lightData["instances"])) for windowSize in windowSizeList: var_ratio, com_ratio = computeCompressgraph( lightData["instances"], windowSize) row = [fileName, datePattern, windowSize, var_ratio, com_ratio] print(row) ut.add_to_csv(row=row, csv_name="comratio_mdf.csv")
if __name__ == '__main__': windowSize = 40 upper_bound = 1.87519737 lower_bound = 0.53327720 listFile = ut.txt_to_list(csv_name="start_file.csv") # for windowSize in listWindow: for index, fileName1 in enumerate(listFile): print("File Name {}".format(fileName1)) pattern = fileName1.split("_")[4] lightData1 = ut_mdf.getDataFromFile(fileName=fileName1) dyResult1 = ut_data.genListDyBin(instances=lightData1["instances"], timestamp=lightData1["timestamp"], windowSize=windowSize) targetList = ut_mdf.getListMDF(pattern=pattern) for fileName2 in targetList: try: lightData2 = ut_mdf.getDataFromFile(fileName=fileName2) isOverlap = ut_mdf.isOverlapTimestamp(lightData1["timestamp"], lightData2["timestamp"]) if isOverlap: dyResult2 = ut_data.genListDyBin( instances=lightData2["instances"], timestamp=lightData2["timestamp"], windowSize=windowSize) # computeLocalOutlierFactor(dyResult1,windowSize=windowSize) F_test = dyResult1["variance"] / dyResult2["variance"] if lower_bound <= F_test <= upper_bound: row = [fileName1, fileName2, windowSize, F_test] print(row)
import utility.utility_mdf as ut_mdf import utility.utility as ut import utility.utility_genData as ut_gen import os from method.change_de.grabbs_test_detection import grabbs_test_detection as gb_test dataset_path = 'D:\\mdwarf_data\\' lc_path = "lc_flux_catalog_aperture_r7_txt\\" lc_timestamp_path = "lc_timestamp_txt\\" path_to_lc_file = "{}{}".format(dataset_path, lc_path) initialBin = 3 theshold_compression = 0.03 alpha = 0.05 if __name__ == '__main__': listFile = ut_mdf.getListMDF( startFile='light_curve_Gaia-DR2_610350453746204032_date20200201') for indexFile, fileName in enumerate(listFile): try: pathFile = "{}{}.txt".format(path_to_lc_file, fileName) filesize = os.path.getsize(pathFile) if filesize != 0: result_found = 0 file = open(pathFile, 'rb') FdataMDF = [float(item) for item in ut.txt_to_list(pathFile)] path_lc_timestamp_path = "{}{}{}.txt".format( dataset_path, lc_timestamp_path, fileName) x_axis = [ float(item) for item in ut.txt_to_list(path_lc_timestamp_path) ] # window_size = int((len(FdataMDF) / initialBin) * theshold_compression)