def PlotLy(t,wdsize_percent,min_interval_percent,polyIndex,fileName,pltData,isOpen=False):
strtitle=fileName[12:-7]+' Plot-'+str(len(t))
layout=go.Layout(title=strtitle,
xaxis=dict(
#title='AXIS TITLE',
titlefont=dict(
#family='Arial, sans-serif',
size=20,
color='lightgrey'
),
showticklabels=True,
tickangle=0,
tickfont=dict(
#family='Old Standard TT, serif',
size=35,
color='black'
),
exponentformat='e',
showexponent='all'
),
yaxis=dict(
#title='AXIS TITLE',
titlefont=dict(
#family='Arial, sans-serif',
size=20,
color='lightgrey'
),
showticklabels=True,
tickangle=0,
tickfont=dict(
#family='Old Standard TT, serif',
size=35,
color='black'
),
exponentformat='e',
showexponent='all'
),
showlegend=False)
#plot(pltData,layout,image='png',image_filename=fileName,filename=".\Plot_html\\"+fileName+".html")
#filePath=".\Plot_html\\SlidingWindow\\"+'wdL '+str(SGwd_length)+'\\max angle '+str(angle)+'\\'
filePath=".\Plot_html\\"
fT.mkfolder(filePath)
fig=go.Figure(data=pltData,layout=layout)
#print(fileName)
plot(fig,filename=filePath+fileName[12:]+".html",auto_open=isOpen)
def PlotLy(t,
wdsize_percent,
min_r2,
polyIndex,
fileName,
pltData,
isOpen=False):
strtitle = fileName + ' Plot-' + str(len(t))
layout = go.Layout(title=strtitle)
#plot(pltData,layout,image='png',image_filename=fileName,filename=".\Plot_html\\"+fileName+".html")
#filePath=".\Plot_html\\SlidingWindow\\"+'wdL '+str(SGwd_length)+'\\max angle '+str(angle)+'\\'
filePath = ".\Plot_html\\SlidingWindow\\" + 'wdsize_percent=' + str(
wdsize_percent) + '\\min_r2=' + str(min_r2) + '\\pIndex=' + str(
polyIndex) + '\\'
fT.mkfolder(filePath)
fig = go.Figure(data=pltData, layout=layout)
plot(fig,
config=dict(showSendToCloud=True),
filename=filePath + fileName + ".html",
auto_open=isOpen)
def Seg2poly(fileName,
sgf,
SGwd_length=11,
SGpolyOrd=5,
min_r2=0.99,
polyIndex=5,
window_size_percent=0.05,
min_interval_percent=0.0,
isplot=False):
startPt = 0
endPt = 0
delta = 0
timer = 0
'''
plot(t,v,'-')
plot(t,sgf,'-')
'''
tStart = time.time() #計時開始
#plt.plot(t,v,'-')
#plt.plot(t,sgf,'-')
if (len(t) > 100):
pass
else:
window_size_percent = 1
print('wdSize:', window_size_percent)
interval = int(len(t) * window_size_percent) - 1
start = []
end = []
#intervalAry=[]
coef_ary = []
decompData = []
#first_inl=0
R2_PnS = []
R2_PnR = []
Dlta = []
sensor = fileName[6:-1]
Maxr2SGF = 0.0
MaxEnd = 0
MaxCoeff = []
MaxYs = []
rsqSGF = 0
rsqTT = 0
ys_line = []
dirSnr = ''
DeltaCnt = 0
if (sensor == 'T'):
print(fileName)
dirSnr = sensor + '\\'
elif (sensor == 'H'):
print(fileName)
dirSnr = sensor + '\\'
#dirFoder='Data_csv\\SlidingWindow\\Chebyshev\\'+dirSnr+'\\Poly_Index '+str(pIndex)+'\\max angle '+str(angle) #'\\Poly_Index '+str(pIndex)+'\\SGwd_length '+str(SGwd_length)
#dirFoder='Data_csv\\SlidingWindow\\Chebyshev\\'+dirSnr
dirFoder = 'Data_csv\\SlidingWindow\\Chebyshev\\' + dirSnr
fT.mkfolder(dirFoder)
dirWCdetailFolder = 'Data_csv\\SlidingWindow\\Chebyshev\\Worst compressing data\\'
fT.mkfolder(dirWCdetailFolder)
#===condition====
#max_slope=2 # 反應差
min_time_interval = int(len(t) * min_interval_percent)
c = 1
while (startPt < (len(t))):
#MaxStart=0
islimit = False
# =============================================================================
#
# if ((interval-startPt)<=min_time_interval) :
# interval=interval+int((len(t)/10))
#
# =============================================================================
for i in range(interval, startPt - 1, -1):
if ((i - startPt) > min_time_interval):
endPt = i + 1
delta = endPt - startPt
coeff, ys_line, rsqSGF, rsqTT = mthT.polyLine(
startPt, endPt, polyIndex, t, sgf, data)
if (rsqSGF >= Maxr2SGF):
Maxr2SGF = rsqSGF
MaxYs = ys_line
#MaxStart=startPt
MaxEnd = endPt
MaxCoeff = coeff
break
elif ((i - startPt) <= min_time_interval + 1):
if (Maxr2SGF == 0):
endPt = len(t)
delta = endPt - startPt
coeff, ys_line, rsqSGF, rsqTT = mthT.polyLine(
startPt, endPt, polyIndex, t, sgf, data)
islimit = True
else:
endPt = MaxEnd
coeff = MaxCoeff
delta = endPt - startPt
rsqSGF = Maxr2SGF
ys_line = MaxYs
delta = endPt - startPt
islimit = True
break
if (rsqSGF >= min_r2) or islimit:
#====coeff[0] = A coeff[1] = B coeff[2] = C; Ax^2+Bx+C
#start.append(time[startPt])
#end.append(time[endPt])
# =============================================================================
# if(rsqTT<min_r2):
# print(c)
# print(Maxr2SGF)
# print(islimit)
# print(i)
# print(startPt)
# =============================================================================
start.append(times[startPt])
if (endPt != len(times)):
end.append(times[endPt])
#print(len(times),':',endPt)
else:
end.append(times[endPt - 1])
#print(len(times),':',endPt)
#coef_ary.append(coeff)
#print(np.poly1d(coeff))
coef_ary.extend(coeff)
decompData.extend(ys_line)
print()
Dlta.append(delta)
DeltaCnt = DeltaCnt + endPt - startPt
# print(len(decompData),'Delta=',DeltaCnt)
global pltData
pltData += [
go.Scatter(
x=times[
startPt:endPt], # assign x as the dataframe column 'x'
y=ys_line,
mode='lines',
name=str(c) + ':S: ' + str(rsqSGF) + ',R: ' + str(rsqTT),
marker=dict(size=5, color='rgba(255,0,0,0.9)'))
]
c += 1
#print(startPt,"-",endPt)
R2_PnR.append(round(rsqTT, 3))
R2_PnS.append(round(rsqSGF, 3))
startPt = endPt
interval = startPt + int((len(t) * window_size_percent)) - 1
# =============================================================================
if (interval >= len(t)):
interval = len(t) - 1
#intervalAry.append(interval)
MaxCoeff = []
MaxYs = []
Maxr2SGF = 0.0
MaxEnd = interval
else:
interval -= 1
tEnd = time.time() #計時結束
timer = tEnd - tStart
print('Compressing Time:', timer)
pltT.PlotLy(t, window_size_percent, min_interval_percent, polyIndex,
fileName, pltData, isplot)
return coef_ary, decompData
def CHEB_cod(fileName,
min_r2=0.95,
polyIndex=1,
window_size_percent=0.5,
maxCv=0.01,
isplot=False):
# ==========================
# 設定初始、結束位置等
# ==========================
startPt=0
endPt=0
delta=0
timer=0
'''
plot(t,v,'-')
plot(t,sgf,'-')
'''
tStart = time.time()#計時開始
#plt.plot(t,v,'-')
#plt.plot(t,sgf,'-')
# ========================================
# 預防數據來源總量過少，以及區間過小或等於0
# ===========================================
if(len(t)>100):
pass
else:
window_size_percent=1
#print('wdSize:',window_size_percent)
#==================
# 記錄多項式係數、Delta,type 為list
#==================
# 由 0 為n次，1為n-1次，如此類推
coef_ary=[]
coeff_5=[]
coeff_4=[]
coeff_3=[]
coeff_2=[]
coeff_1=[]
coeff_0=[]
Dlta=[]
#==================
# 記錄解壓縮、RMSE 數據,type 為list
#==================
decompData=[]
RMSECpData=[]
#first_inl=0
R2_PnR=[]
R2_ngt=[]
R2_noon=[]
R2_afr=[]
R2_mrg=[]
#==================
# 記錄不同時段的數據,type 為list
#==================
mrgData=[]
noonData=[]
aftData=[]
ngtData=[]
#==================
# 設定其他預設值
#==================
interval=int(len(t)*window_size_percent)-1
start=[]
end=[]
#intervalAry=[]
rsqRW=0
ys_line=[]
tot_segNum=0
dirSnr=''
DeltaCnt=0
dtMean={}
sensor=fileName[6:-1]
#==================
# 儲存分段最佳值
#==================
Maxr2RW=0.0
MaxEnd=0
MaxCoeff=[]
MaxYs=[]
# =========================
# 判斷數據集是溫度或濕度
# =========================
if(sensor=='T'):
#print(fileName)
dirSnr=sensor+'\\'
elif(sensor=='H'):
#print(fileName)
dirSnr=sensor+'\\'
#dirFoder='Data_csv\\SlidingWindow\\Chebyshev\\'+dirSnr+'\\Poly_Index '+str(pIndex)+'\\max angle '+str(angle) #'\\Poly_Index '+str(pIndex)+'\\SGwd_length '+str(SGwd_length)
#dirFoder='Data_csv\\SlidingWindow\\Chebyshev\\'+dirSnr
#========================
# 運算結果檔案位置
#========================
dirFoder='Data_csv\\SlidingWindow\\Chebyshev\\'+dirSnr
fT.mkfolder(dirFoder)
dirWCdetailFolder='Data_csv\\SlidingWindow\\Chebyshev\\Mean\\Worst compressing data\\'
fT.mkfolder(dirWCdetailFolder)
#================================
# condition
#================================
c=1
segNum=0
mrg=0
noon=0
aft=0
ngt=0
OmrgData=[]
OnonData=[]
OaftData=[]
OngtData=[]
OrgMrg=0
OrgNoon=0
OrgAft=0
OrgNgt=0
mxErr=0
while(startPt<(len(t))):
#MaxStart=0
islimit=False
# =============================================================================
#
# if ((interval-startPt)<=min_time_interval) :
# interval=interval+int((len(t)/10))
#
# =============================================================================
for i in range(interval,startPt-1,-1):
if ((i-startPt)>1):
endPt=i+1
delta=endPt-startPt
coeff,ys_line,rsqRW,MAE,Cv = mthT.polyLine(startPt,endPt,polyIndex,t,data)
if(rsqRW>=Maxr2RW):
Maxr2RW=rsqRW
MaxYs=ys_line
MaxEnd=endPt
MaxCoeff=coeff
elif((i-startPt)<=1):
if(Maxr2RW==0.0):
endPt=len(t)
delta=endPt-startPt
coeff,ys_line,rsqRW,MAE,Cv = mthT.polyLine(startPt,endPt,polyIndex,t,data)
islimit=True
else:
rsqRW=Maxr2RW
endPt=MaxEnd
coeff=MaxCoeff
delta=endPt-startPt
ys_line=MaxYs
delta=endPt-startPt
islimit=True
if (rsqRW>=min_r2 or Cv<=maxCv) or islimit :
start.append(times[startPt])
if(endPt!=len(times)):
end.append(times[endPt])
else:
end.append(times[endPt-1])
#print(coeff)
# =============================================================================
# coeff_5.append(float(coeff[5]))
# coeff_4.append(float(coeff[4]))
# coeff_3.append(float(coeff[3]))
# coeff_2.append(float(coeff[2]))
# coeff_1.append(float(coeff[1]))
# coeff_0.append(float(coeff[0]))
# =============================================================================
coef_ary.extend(coeff)
decompData.extend(ys_line)
Dlta.append(delta)
DeltaCnt =DeltaCnt+endPt-startPt
# =============================================================================
# global pltData
# =============================================================================
c+=1
R2_PnR.append(round(rsqRW,3))
startPt= endPt
interval=startPt+int((len(t)*window_size_percent))-1
# =============================================================================
if (interval>=len(t)):
interval=len(t)-1
MaxCoeff=[]
MaxYs=[]
Maxr2RW=0.0
MaxEnd=interval
break
tEnd = time.time()#計時結束
#========================
# 計算不同時間段的結果
#========================
#print(len(start))
for x in range(len(start)):
if(start[x].hour<6):
mrg+=1
elif(start[x].hour>=6 and start[x].hour<12):
noon+=1
elif(start[x].hour>=12 and start[x].hour<=18):
aft+=1
elif(start[x].hour>=18 and start[x].hour<=23):
ngt+=1
for x in range(len(data)):
if(times[x].hour<6):
OrgMrg+=1
OmrgData.append(data[x])
mrgData.append(decompData[x])
elif(times[x].hour>=6 and times[x].hour<12):
OrgNoon+=1
OnonData.append(data[x])
noonData.append(decompData[x])
elif(times[x].hour>=12 and times[x].hour<=18):
OrgAft+=1
OaftData.append(data[x])
aftData.append(decompData[x])
elif(times[x].hour>=18 and times[x].hour<=23):
OrgNgt+=1
OngtData.append(data[x])
ngtData.append(decompData[x])
# =======================
# 輸出每天壓縮後的詳細資料
# =======================
# =============================================================================
# #==each day polynorimal data
# dtPolyData={'start':start,
# 'end':end,
# 'min_r2':min_r2,
# 'poly n':pIndex,
# 'wd_size%':wdsize_percent,
# 'min_interval%':min_interval_percent,
# 'angle':angle,
# 'coef':coef_ary,
# 'Delta':Dlta,
# 'coeff_5':coeff_0,
# 'coeff_4':coeff_1,
# 'coeff_3':coeff_2,
# 'coeff_2':coeff_3,
# 'coeff_1':coeff_4,
# 'coeff_0':coeff_5,
# 'R2_Raw':R2_PnR
# }
# headerPD=['start','end','Delta','coeff_5','coeff_4','coeff_3','coeff_2','coeff_1','coeff_0']
# #輸出沒有了r2 值
# dfpolyDtlData=pd.DataFrame(dtPolyData,columns=headerPD,index=None)
# SnrCsvfileName='Data_csv\\SlidingWindow\\Chebyshev\\Mean\\Worst compressing data'+'\\'+fileName+"_DetailData.csv"
# dfpolyDtlData.to_csv(SnrCsvfileName,mode='w',index=None)
# =============================================================================
# =================================
# 計算四個時間段的R2、RMSE、壓縮比
# =================================
# =============================================================================
# print(OmrgData)
# print(mrgData)
# =============================================================================
cpMrg=compR(mrg,OrgMrg),
cpNoon=compR(noon,OrgNoon),
cpAft=compR(aft,OrgAft),
cpNgt=compR(ngt,OrgNgt),
rmseMrg=calRMSE(OmrgData,mrgData),
rmseNoon=calRMSE(OnonData,noonData),
rmseAft=calRMSE(OaftData,aftData),
rmseNgt=calRMSE(OngtData,ngtData)
# =============================================================================
# print('總結:times=',len(times),'deCompDaya=',len(decompData),'min_r2=',min_r2)
# print('分段數量: 0-6點: ',mrg,' 6-12點: ',noon,' 12-18點: ',aft,' 18-23點: ',ngt)
# print('R2: 0-6點: ',round(mthT.coeff_of_determination(np.array(OmrgData),mrgData,startPt,endPt),3),
# '6-12點: ',round(mthT.coeff_of_determination(np.array(OnonData),noonData,startPt,endPt),3),
# '12-18點: ',round(mthT.coeff_of_determination(np.array(OaftData),aftData,startPt,endPt),3),
# '18-23點: ',round(mthT.coeff_of_determination(np.array(OngtData),ngtData,startPt,endPt),3))
# print('壓縮比: 0-6點: ',compR(mrg,OrgMrg),
# '6-12點: ',compR(noon,OrgNoon),
# '12-18點: ',compR(aft,OrgAft),
# '18-23點: ',compR(ngt,OrgNgt))
# print('RMSE: 0-6點: ',calRMSE(OmrgData,mrgData),
# '6-12點: ',calRMSE(OnonData,noonData),
# '12-18點: ',calRMSE(OaftData,aftData),
# '18-23點: ',calRMSE(OngtData,ngtData))
# =============================================================================
#print('wd Length: ',SGwd_length,'SGpolyOrd: ',SGpolyOrd)
hdr_timeData=["date",
"Sensor",
"segMrg",
"segNoon",
"segAft",
"segNgt",
"cpMrg",
"cpNoon",
"cpAft",
"cpNgt",
"rmseMrg",
"rmseNoon",
"rmseAft",
"rmseNgt"
]
timeData={"date":fileName[:-3],
"Sensor":fileName[6:],
"segMrg":mrg,
"segNoon":noon,
"segAft":aft,
"segNgt":ngt,
"cpMrg":cpMrg,
"cpNoon":cpNoon,
"cpAft":cpAft,
"cpNgt":cpNgt,
"rmseMrg":rmseMrg,
"rmseNoon":rmseNoon,
"rmseAft":rmseAft,
"rmseNgt":rmseNgt
}
df4periods=pd.DataFrame(timeData,columns=hdr_timeData)
print(ssrType,end='')
MeanCsvfileName=dirMean+'\\2020_4periods_Data\\R2t'+str(min_r2)+'_wd'+str(window_size_percent)+'_pIndex'+str(polyIndex)+'_'+dt.datetime.now().strftime('%m%d')+'.csv'
if (pth.isfile(MeanCsvfileName)!=True):
df4periods.to_csv(MeanCsvfileName,mode='w',index=None)
else:
df4periods.to_csv(MeanCsvfileName,mode='a',header=None,index=None)
df4periods=pd.DataFrame(None,None)
# ============================
# 輸出解壓縮的詳細數據
# ============================
# =============================================================================
# #==each day deCompression data
# dtdeCData={'times':times,
# 'Data':decompData
# }
#
# headerDC=['times','Data']
# dfdeCData=pd.DataFrame(dtdeCData,columns=headerDC,index=None)
#
# deCompCsvfileName='Data_csv\\SlidingWindow\\Chebyshev\\Mean\\Worst compressing data'+'\\'+fileName+"_DecompressionDetailData.csv"
#
# dfdeCData.to_csv(deCompCsvfileName,mode='w',index=None)
#
# =============================================================================
segNum=len(start)# num of segement
tot_segNum+=segNum# total number of segements in all data
Nr=len(t)*2*8
Nc=segNum*(pIndex+4)*8
comp_ratio=(Nr-Nc)/Nr
#print(dfpolyDtlData)
#R2_RnS=round(mthT.coeff_of_determination(np.array(data),sgf,startPt,endPt),3)
R2=round(mthT.coeff_of_determination(np.array(data),decompData,startPt,endPt),3)
MeanR2=round(np.mean(R2),3)
STD=round(np.std(R2_PnR,ddof=1 ),3)
#==============================================================================
# this for loop is compare Compress data and Raw data
# =============================================================================
temp=[]
for n in range(len(data)):
temp.append(np.abs(np.round(data[n],5)-np.round(decompData[n],5)))
maxErr=np.max(temp)
minErr=np.min(temp)
MeanRMSE=calRMSE(data,decompData)
CRatio=round(comp_ratio,3)
timer=tEnd-tStart
#print('分段總數= ',str(mrg+noon+aft+ngt),' Mean R2=',MeanR2,' CompRatio=',CRatio,' RMSE=',MeanRMSE,)
#print('MeanR2PnR = ' ,MeanR2PnR,', MeanR2PnS = ',MeanR2PnS)
#print(fileName[:2]+fileName[3:])
dtMean={
'date':fileName[:-3],
'Sensor':fileName[6:],
'data Length':len(t),
'pIndex':pIndex,
'wdsize':window_size_percent,
'Cv':maxCv,
'segNum':segNum,
'comp_ratio':CRatio,
'timer': timer,
'Mean_R2':MeanR2,
'R2_STD':STD,
'Mean_RMSE':MeanRMSE,
'Max_Err':maxErr,
'Min_Err':minErr
}
# print(df)
# df.head()
#print(pltData)
#print(sensor,end=',')
global pltData
#print(len(plotTime),':',len(decompData))
pltData += [
go.Scatter(
x=plotTime, # assign x as the dataframe column 'x'
y=decompData,
mode='lines',
name='CHEB',
marker=dict(
size=5,
color='rgba(255,0,0,0.9)'
)
)]
# ============================
# 畫圖 to html
# ============================
if isplot:
#print(window_size_percent)
pltT.PlotLy(t,window_size_percent,min_r2,polyIndex,maxCv,fileName,pltData,plotisOpen)
return dtMean
#^^^^^^^^^^^^^^^^^^ Sub Functions ^^^^^^^^^^^^^^^
if __name__ =="__main__":
fileName=fT.getFileName(mypath)
#print(fileName)
dirMean='Data_csv\\CHEB(CoD)_Result'
fT.mkfolder(dirMean)
# ============================
# 條件選擇
# ============================
pIndex=4##多項式次數
min_r2=0.95##Rt2閥值
wdsize_percent=0.1#
maxCv=0.025##離散係數
#min_interval_percent=0.00#limit = 0.001
# ============================
Isplot=True
plotisOpen=True
f.close()
#print(data)
mean_rate = round(np.mean(data['rate']), 3)
mean_value = round(np.mean(data['value']), 3)
std = round(np.std(data['value']), 3)
print('sensor Type : ', Snrtype)
print('# of file : ', cnt)
print('# of data : ', len(data['value']))
print('mean rate : ', mean_rate)
print('max : ', np.max(data['value']))
print('min : ', np.min(data['value']))
print('mean : ', mean_value)
print('std : ', std)
from plotly.offline import plot
import cufflinks as cf
import plotly.graph_objs as go
pltData = [
go.Scatter(
x=t, # assign x as the dataframe column 'x'
y=v,
mode='lines',
marker=dict(size=5, color='rgba(255,0,0,1)'),
line=dict(width=1, ))
]
strtitle = Snrtype + ' Plot'
layout = go.Layout(title=strtitle)
filePath = ".\Plot_html\\"
fT.mkfolder(filePath)
fig = go.Figure(data=pltData, layout=layout)
#print(fileName)
plot(fig, filename=filePath + strtitle + ".html")
def Seg2poly(fileName,SGwd_length=11,SGpolyOrd=3,max_slope=1,min_r2=0.95,polyIndex=1,window_size_percent=0.5,min_interval_percent=0,isplot=False):
startPt=0
endPt=0
delta=0
timer=0
'''
plot(t,v,'-')
plot(t,sgf,'-')
'''
tStart = time.time()#計時開始
#plt.plot(t,v,'-')
#plt.plot(t,sgf,'-')
angle=int(round((np.arctan(max_slope)*180)/pi))
if(len(t)>100):
pass
else:
window_size_percent=1
#print('wdSize:',window_size_percent)
interval=int(len(t)*window_size_percent)-1
start=[]
end=[]
#intervalAry=[]
coef_ary=[]
coeff_5=[]
coeff_4=[]
coeff_3=[]
coeff_2=[]
coeff_1=[]
coeff_0=[]
decompData=[]
RMSDCData=[]
#first_inl=0
R2_PnS=[]
R2_PnR=[]
R2_ngt=[]
R2_noon=[]
R2_afr=[]
R2_mrg=[]
mrgData=[]
noonData=[]
aftData=[]
ngtData=[]
Dlta=[]
dtMean={}
sensor=fileName[6:-1]
Maxr2SGF=0.0
Maxr2RW=0.0
MaxEnd=0
MaxCoeff=[]
MaxYs=[]
rsqSGF=0
rsqRW=0
ys_line=[]
tot_segNum=0
dirSnr=''
DeltaCnt=0
if(sensor=='T'):
print(fileName)
dirSnr=sensor+'\\'
elif(sensor=='H'):
print(fileName)
dirSnr=sensor+'\\'
#dirFoder='Data_csv\\SlidingWindow\\Chebyshev\\'+dirSnr+'\\Poly_Index '+str(pIndex)+'\\max angle '+str(angle) #'\\Poly_Index '+str(pIndex)+'\\SGwd_length '+str(SGwd_length)
#dirFoder='Data_csv\\SlidingWindow\\Chebyshev\\'+dirSnr
dirFoder='Data_csv\\SlidingWindow\\Chebyshev\\'+dirSnr
fT.mkfolder(dirFoder)
dirWCdetailFolder='Data_csv\\SlidingWindow\\Chebyshev\\Mean\\Worst compressing data\\'
fT.mkfolder(dirWCdetailFolder)
#===condition====
#max_slope=2 # 反應差
min_time_interval= int(len(t)*min_interval_percent)
c=1
segNum=0
mrg=0
noon=0
aft=0
ngt=0
OmrgData=[]
OnonData=[]
OaftData=[]
OngtData=[]
OrgMrg=0
OrgNoon=0
OrgAft=0
OrgNgt=0
while(startPt<(len(t))):
#MaxStart=0
islimit=False
# =============================================================================
#
# if ((interval-startPt)<=min_time_interval) :
# interval=interval+int((len(t)/10))
#
# =============================================================================
for i in range(interval,startPt-1,-1):
if ((i-startPt)>1):
endPt=i+1
delta=endPt-startPt
coeff,ys_line,rsqSGF,rsqRW = mthT.polyLine(startPt,endPt,polyIndex,t,sgf,data)
if(rsqRW>=Maxr2RW):
Maxr2RW=rsqRW
MaxYs=ys_line
MaxEnd=endPt
MaxCoeff=coeff
elif((i-startPt)<=1):
if(Maxr2RW==0.0):
endPt=len(t)
delta=endPt-startPt
coeff,ys_line,rsqSGF,rsqRW = mthT.polyLine(startPt,endPt,polyIndex,t,sgf,data)
islimit=True
else:
rsqRW=Maxr2RW
endPt=MaxEnd
coeff=MaxCoeff
delta=endPt-startPt
ys_line=MaxYs
delta=endPt-startPt
islimit=True
if (rsqRW>=min_r2) or islimit :
start.append(times[startPt])
if(endPt!=len(times)):
end.append(times[endPt])
else:
end.append(times[endPt-1])
coeff_5.append(float(coeff[0]))
coeff_4.append(float(coeff[1]))
coeff_3.append(float(coeff[2]))
coeff_2.append(float(coeff[3]))
coeff_1.append(float(coeff[4]))
coeff_0.append(float(coeff[5]))
coef_ary.extend(coeff)
decompData.extend(ys_line)
Dlta.append(delta)
DeltaCnt =DeltaCnt+endPt-startPt
global pltData
c+=1
R2_PnR.append(round(rsqRW,3))
R2_PnS.append(round(rsqSGF,3))
startPt= endPt
interval=startPt+int((len(t)*window_size_percent))-1
# =============================================================================
if (interval>=len(t)):
interval=len(t)-1
MaxCoeff=[]
MaxYs=[]
Maxr2RW=0.0
MaxEnd=interval
break
tEnd = time.time()#計時結束
for x in range(len(data)):
if(times[x].hour<6):
OrgMrg+=1
elif(times[x].hour>=6 and times[x].hour<12):
OrgNoon+=1
elif(times[x].hour>=12 and times[x].hour<=18):
OrgAft+=1
elif(times[x].hour>=18 and times[x].hour<=23):
OrgNgt+=1
#=================每一天壓縮後數據==================================
# =============================================================================
# #==each day polynorimal data
# dtPolyData={'start':start,
# 'end':end,
# 'min_r2':min_r2,
# 'poly n':pIndex,
# 'wd_size%':wdsize_percent,
# 'min_interval%':min_interval_percent,
# 'angle':angle,
# 'coef':coef_ary,
# 'Delta':Dlta,
# 'coeff_5':coeff_5,
# 'coeff_4':coeff_4,
# 'coeff_3':coeff_3,
# 'coeff_2':coeff_2,
# 'coeff_1':coeff_1,
# 'coeff_0':coeff_0,
# 'R2_Raw':R2_PnR,
# 'R2_SGF':R2_PnS
# }
# headerPD=['start','end','Delta','coeff_5','coeff_4','coeff_3','coeff_2','coeff_1','coeff_0']
# #輸出沒有了r2 值
# dfpolyDtlData=pd.DataFrame(dtPolyData,columns=headerPD,index=None)
# SnrCsvfileName='Data_csv\\SlidingWindow\\Chebyshev\\Mean\\Worst compressing data'+'\\'+fileName+"_DetailData.csv"
# dfpolyDtlData.to_csv(SnrCsvfileName,mode='w',index=None)
# =============================================================================
# =============================================================================
#
# print('總結:times=',len(times),'deCompDaya=',len(decompData),'min_r2=',min_r2)
# print('分段數量: 0-6點: ',mrg,' 6-12點: ',noon,' 12-18點: ',aft,' 18-23點: ',ngt)
# print('R2: 0-6點: ',round(mthT.coeff_of_determination(np.array(OmrgData),mrgData,startPt,endPt),3),
# '6-12點: ',round(mthT.coeff_of_determination(np.array(OnonData),noonData,startPt,endPt),3),
# '12-18點: ',round(mthT.coeff_of_determination(np.array(OaftData),aftData,startPt,endPt),3),
# '18-23點: ',round(mthT.coeff_of_determination(np.array(OngtData),ngtData,startPt,endPt),3))
# print('壓縮比: 0-6點: ',compR(mrg,OrgMrg),
# '6-12點: ',compR(noon,OrgNoon),
# '12-18點: ',compR(aft,OrgAft),
# '18-23點: ',compR(ngt,OrgNgt))
# print('RMSD: 0-6點: ',calRMSD(OmrgData,mrgData),
# '6-12點: ',calRMSD(OnonData,noonData),
# '12-18點: ',calRMSD(OaftData,aftData),
# '18-23點: ',calRMSD(OngtData,ngtData))
#
#
#
# =============================================================================
# =============================================================================
# #==each day deCompression data
# dtdeCData={'times':times,
# 'Data':decompData
# }
#
# headerDC=['times','Data']
# dfdeCData=pd.DataFrame(dtdeCData,columns=headerDC,index=None)
#
# deCompCsvfileName='Data_csv\\SlidingWindow\\Chebyshev\\Mean\\Worst compressing data'+'\\'+fileName+"_DecompressionDetailData.csv"
#
# dfdeCData.to_csv(deCompCsvfileName,mode='w',index=None)
#
# =============================================================================
segNum=len(start)# num of segement
tot_segNum+=segNum# total number of segements in all data
Nr=len(t)*2*8
Nc=segNum*(pIndex+4)*8
comp_ratio=(Nr-Nc)/Nr
#print(dfpolyDtlData)
R2_RnS=round(mthT.coeff_of_determination(np.array(data),sgf,startPt,endPt),3)
R2=round(mthT.coeff_of_determination(np.array(data),decompData,startPt,endPt),3)
MeanR2=round(np.mean(R2),3)
STD=round(np.std(R2_PnR,ddof=1 ),3)
sumDCData=0
for n in range(len(data)):
## to compare decompress Alg data and Decompress DB data
sumDCData+=(np.round(data[n],5)-np.round(decompData[n],5))**2
RMSDCData.append(float(np.round(math.sqrt(sumDCData/len(data)),5)))
MeanRMSD=round(np.mean(RMSDCData),3)
CRatio=round(comp_ratio,3)
timer=tEnd-tStart
print('分段總數= ',str(mrg+noon+aft+ngt),' Mean R2=',MeanR2,' CompRatio=',CRatio,' RMSD=',MeanRMSD,)
#print('MeanR2PnR = ' ,MeanR2PnR,', MeanR2PnS = ',MeanR2PnS)
#print(fileName[:2]+fileName[3:])
dtMean={
'date':fileName[:-3],
'Sensor':fileName[6:],
'data Length':len(t),
'pIndex':pIndex,
'Minium interval':min_time_interval,
'angle':angle,
'segNum':segNum,
'comp_ratio':comp_ratio,
'R2_R&S':R2_RnS,
'timer': timer,
'Mean_R2':MeanR2,
'Mean_RMSD':MeanRMSD,
'STD':STD
}
# print(df)
# df.head()
#print(pltData)
#print(len(plotTime),':',len(decompData))
pltData += [
go.Scatter(
x=plotTime, # assign x as the dataframe column 'x'
y=decompData,
mode='lines',
name='CHEB',
marker=dict(
size=5,
color='rgba(255,0,0,0.9)'
)
)]
if isplot:
#print(window_size_percent)
pltT.PlotLy(t,window_size_percent,min_r2,polyIndex,fileName,pltData,plotisOpen)
return dtMean
def Seg2poly(fileName,
SGwd_length=11,
SGpolyOrd=3,
max_slope=1,
min_r2=0.95,
polyIndex=1,
window_size_percent=0.5,
min_interval_percent=0,
isplot=False):
startPt = 0
endPt = 0
delta = 0
timer = 0
'''
plot(t,v,'-')
plot(t,sgf,'-')
'''
tStart = time.time() #計時開始
#plt.plot(t,v,'-')
#plt.plot(t,sgf,'-')
angle = int(round((np.arctan(max_slope) * 180) / pi))
# =============================================================================
# if(len(t)>100):
# pass
# else:
# window_size_percent=1
# =============================================================================
interval = int(len(t) * window_size_percent) - 1
start = []
end = []
#intervalAry=[]
coef_ary = []
coeff_5 = []
coeff_4 = []
coeff_3 = []
coeff_2 = []
coeff_1 = []
coeff_0 = []
decompData = []
#first_inl=0
R2_PnS = []
R2_PnR = []
Dlta = []
dtMean = {}
sensor = fileName[6:-1]
Maxr2SGF = 0.0
MaxEnd = 0
MaxCoeff = []
MaxYs = []
rsqSGF = 0
rsqTT = 0
ys_line = []
tot_segNum = 0
dirSnr = ''
DeltaCnt = 0
if (sensor == 'T'):
print(fileName)
dirSnr = sensor + '\\'
elif (sensor == 'H'):
print(fileName)
dirSnr = sensor + '\\'
#dirFoder='Data_csv\\SlidingWindow\\Chebyshev\\'+dirSnr+'\\Poly_Index '+str(pIndex)+'\\max angle '+str(angle) #'\\Poly_Index '+str(pIndex)+'\\SGwd_length '+str(SGwd_length)
#dirFoder='Data_csv\\SlidingWindow\\Chebyshev\\'+dirSnr
dirFoder = 'Data_csv\\SlidingWindow\\Chebyshev\\' + dirSnr
fT.mkfolder(dirFoder)
dirWCdetailFolder = 'Data_csv\\SlidingWindow\\Chebyshev\\Worst compressing data\\'
fT.mkfolder(dirWCdetailFolder)
#===condition====
#max_slope=2 # 反應差
min_time_interval = int(len(t) * min_interval_percent)
c = 1
segNum = 0
while (startPt < (len(t))):
#MaxStart=0
islimit = False
# =============================================================================
#
# if ((interval-startPt)<=min_time_interval) :
# interval=interval+int((len(t)/10))
#
# =============================================================================
for i in range(interval, startPt - 1, -1):
if ((i - startPt) > min_time_interval):
endPt = i + 1
delta = endPt - startPt
coeff, ys_line, rsqSGF, rsqTT = mthT.polyLine(
startPt, endPt, polyIndex, t, sgf, data)
if (rsqTT >= Maxr2SGF):
Maxr2SGF = rsqSGF
MaxYs = ys_line
#MaxStart=startPt
MaxEnd = endPt
MaxCoeff = coeff
break
elif ((i - startPt) <= min_time_interval + 1):
if (Maxr2SGF == 0):
endPt = len(t)
delta = endPt - startPt
coeff, ys_line, rsqSGF, rsqTT = mthT.polyLine(
startPt, endPt, polyIndex, t, sgf, data)
islimit = True
else:
endPt = MaxEnd
coeff = MaxCoeff
delta = endPt - startPt
rsqTT = Maxr2SGF
ys_line = MaxYs
delta = endPt - startPt
islimit = True
break
if (rsqTT >= min_r2) or islimit:
#====coeff[0] = A coeff[1] = B coeff[2] = C; Ax^2+Bx+C
#start.append(time[startPt])
#end.append(time[endPt])
# =============================================================================
# if(rsqTT<min_r2):
# print(c)
# print(Maxr2SGF)
# print(islimit)
# print(i)
# print(startPt)
# =============================================================================
start.append(times[startPt])
if (endPt != len(times)):
end.append(times[endPt])
#print(len(times),':',endPt)
else:
end.append(times[endPt - 1])
#print(len(times),':',endPt)
#coef_ary.append(coeff)
coeff_5.append(coeff[0])
coeff_4.append(coeff[1])
coeff_3.append(coeff[2])
coeff_2.append(coeff[3])
coeff_1.append(coeff[4])
coeff_0.append(coeff[5])
Dlta.append(delta)
for deCdata in ys_line:
decompData.append(deCdata)
DeltaCnt = DeltaCnt + endPt - startPt
# print(len(decompData),'Delta=',DeltaCnt)
global pltData
pltData += [
go.Scatter(
x=times[
startPt:endPt], # assign x as the dataframe column 'x'
y=ys_line,
mode='lines',
name=str(c) + ':S: ' + str(rsqSGF) + ',R: ' + str(rsqTT),
marker=dict(size=5, color='rgba(255,0,0,0.9)'))
]
c += 1
#print(startPt,"-",endPt)
R2_PnR.append(round(rsqTT, 3))
R2_PnS.append(round(rsqSGF, 3))
startPt = endPt
interval = startPt + int((len(t) * window_size_percent)) - 1
# =============================================================================
if (interval >= len(t)):
interval = len(t) - 1
#intervalAry.append(interval)
MaxCoeff = []
MaxYs = []
Maxr2SGF = 0.0
MaxEnd = interval
else:
interval -= 1
tEnd = time.time() #計時結束
# =============================================================================
# #=================每一天壓縮後數據==================================
# #==each day polynorimal data
# dtPolyData={'start':start,
# 'end':end,
# 'min_r2':min_r2,
# 'poly n':pIndex,
# 'wd_size%':wdsize_percent,
# 'min_interval%':min_interval_percent,
# 'angle':angle,
# 'coef':coef_ary,
# 'coeff_5':coeff_5,
# 'coeff_4':coeff_4,
# 'coeff_3':coeff_3,
# 'coeff_2':coeff_2,
# 'coeff_1':coeff_1,
# 'coeff_0':coeff_0,
# 'Delta':Dlta,
# 'R2_Raw':R2_PnR,
# 'R2_SGF':R2_PnS
# }
# headerPD=['start','end','coeff_5','coeff_4','coeff_3','coeff_2','coeff_1','coeff_0','R2_Raw','Delta']
# dfpolyDtlData=pd.DataFrame(dtPolyData,columns=headerPD,index=None)
# SnrCsvfileName='Data_csv\\SlidingWindow\\Chebyshev\\Worst compressing data'+'\\'+fileName+"_DetailData.csv"
# dfpolyDtlData.to_csv(SnrCsvfileName,mode='w')
# print('總結:times=',len(times),'deCompDaya=',len(decompData))
#
#
#
#
#
# #==each day deCompression data
# dtdeCData={'times':times,
# 'Data':decompData
# }
#
# headerDC=['times','Data']
# dfdeCData=pd.DataFrame(dtdeCData,columns=headerDC,index=None)
#
# deCompCsvfileName='Data_csv\\SlidingWindow\\Chebyshev\\Worst compressing data'+'\\'+fileName+"_DecompressionDetailData.csv"
#
# dfdeCData.to_csv(deCompCsvfileName,mode='w')
#
# =============================================================================
segNum = len(start) # num of segement
tot_segNum += segNum # total number of segements in all data
Nr = len(t) * 2 * 8
Nc = segNum * (pIndex + 4) * 8
comp_ratio = (Nr - Nc) / Nr
#print(dfpolyDtlData)
R2_RnS = round(
mthT.coeff_of_determination(np.array(data), sgf, startPt, endPt), 3)
MeanR2PnR = round(np.mean(R2_PnR), 3)
MeanR2PnS = round(np.mean(R2_PnS), 3)
STD = round(np.std(R2_PnS), 3)
timer = tEnd - tStart
#print('MeanR2PnR = ' ,MeanR2PnR,', MeanR2PnS = ',MeanR2PnS)
#print(fileName[:2]+fileName[3:])
dtMean = {
'date': fileName[:-3],
'Sensor': fileName[6:],
'data Length': len(t),
'pIndex': pIndex,
'Minium interval': min_time_interval,
'angle': angle,
'segNum': segNum,
'comp_ratio': comp_ratio,
'R2_R&S': R2_RnS,
'timer': timer,
'Mean_R2_P&R': MeanR2PnR,
'Mean_R2_P&S': MeanR2PnS,
'STD': STD
}
# print(df)
# df.head()
#print(pltData)
if isplot:
pltT.PlotLy(t, window_size_percent, min_interval_percent, polyIndex,
fileName, pltData, plotisOpen)
return dtMean
