def _update_ms_insert(mdb, PE_fr, PE_to): idx_fr=PE_fr['cid']; idx_to=PE_to['cid']; # if not exist, just ignore if(idx_fr == -1 or idx_to == -1): return mdb # create new ms record ms_new=_create_new_ms_rcd(PE_fr, PE_to) # (a) find fr_to in mdb idxs="%d_%d"%(idx_fr, idx_to) if(not (idxs in mdb.MS)): mdb.MS[idxs]=[] # best-candidate ms_best=-1; err_best=tl.INF # (a-i) find closest mss for ms_idxs_i in mdb.MS[idxs]: ms_d=ms_idxs_i['medoid'] err_i=0.5*( tl.RMSE(ms_d['vn_fr'],ms_new['vn_fr']) + tl.RMSE(ms_d['v0_to'],ms_new['v0_to']) ) #tl.eprint(mdb.rho_MS, err_i) #tl.eprint(ms_d['vn_fr'], ms_new['vn_fr'], ms_d['v0_to'],ms_new['v0_to']) if(err_i<err_best): ms_best=ms_idxs_i; err_best=err_i; #tl.eprint("err_best: %f (%f)"%(err_best, mdb.rho_MS)) # (b) update MS[idxs] if(err_best<mdb.rho_MS): # if we have similar ms ms_best['object'].append(ms_new) if(DBG): tl.msg("MDB-MS (insert): # of object=%d"%(len(ms_best['object']))) else: # if unknown, then create new MS mdb.MS[idxs].append({'medoid':ms_new, 'object':[ms_new]}) return mdb
def _FM(mdb, Xc, lene, smin, PE, fixORopt): # dictMS (see, om_trans.py) dictMS = { 'cid': PE['cid'], 'si': tl.dcopy(PE['md'].si), 'rho': mdb.rho_MS, 'smin': smin, 'errV': tl.INF } if (fixORopt == 'OPT'): dictMS = om_trans.fit_forward(Xc, mdb, dictMS, W_FM) #, DPS) (Ve, Se, Re) = om_trans.gen_forward_multi_trans(Xc, mdb, dictMS, lene, MULTI_TRANS) #(Ve, Se, Re)=om_trans.gen_forward(mdb, dictMS, lene) errV = tl.RMSE(Xc, Ve[0:len(Xc)]) if (_isUnfit(Ve, mdb)): Ve[:] = np.nan Se[:] = np.nan Re[:] = REID_NULL errV = tl.INF if (DBG0): tl.msg("%s ++++++ _FM(%s): errV:%f (cid:%d) " % (IDT_F, fixORopt, errV, dictMS['cid'])) PF = {'Ve': Ve, 'Se': Se, 'Re': Re, 'errV': errV} return PF
def _plot(model, fn, pblock): observations=model.data fig=pl.figure(figsize=(8, 6)) pl.subplot(311) pl_obs=pl.plot(observations, '-', label='org') pl.title("Original sequence") # (g_state, g_obs)=_gen(model) (f_state, f_obs)=_filter(model) #(s_state, s_obs)=_smoother(model) # pl.subplot(312) pl_sta=pl.plot(f_state, label='states(filter)') pl.title("States (filter)") pl.subplot(313) #pl_obs=pl.plot(observations, '--', label='org') #tl.resetCol() pl_gen=pl.plot(f_obs, '-', label='gen(filter)') #pl.title("Generated (filter)") pl.title("Generated (filter), rmse=%f"%(tl.RMSE(observations, f_obs))) # #pl.xlabel('time') fig.set_tight_layout(True) if(fn!=''): tl.savefig(fn+'_fit','pdf') pl.show(block=pblock)
def _RU(Xc, PE_cr, fixORopt): tm_st = PE_cr['tm_st'] tm_ed = PE_cr['tm_ed'] if (_isNullP(PE_cr)): if (DBG0): tl.msg("%s ...... _RU: (null)" % (IDT_E)) else: cid = PE_cr['cid'] # put cid md_c = tl.dcopy(PE_cr['md']) # copy model md_c.n = len(Xc) md_c.data = Xc # copy data if (fixORopt == 'OPT'): md_c = md_c.fit_si(W_RR, DPS) # update init si (i.e., s(0)=si) (Sc, Vc) = md_c.gen() # generate events errV = tl.RMSE(Xc, Vc) md_c.fn = "seg:t%d-%d_e%.2f" % (tm_st, tm_ed, errV) if (DBG0): tl.msg("%s ...... _RU: errV:%f (cid:%d)" % (IDT_E, errV, cid)) PE_cr = { 'md': md_c, 'cid': cid, 'Vc': Vc, 'Sc': Sc, 'siset': {}, 'tm_st': tm_st, 'tm_ed': tm_ed, 'errV': errV } return PE_cr
def _find_nearest_t(Vs, vec, smin): n = len(Vs) errs = tl.INF * np.ones((n)) for it in range(smin, n - LST_MINS): errs[it] = tl.RMSE(Vs[it], vec) t_best = np.argmin(errs) err_best = errs[t_best] return (t_best, err_best)
def _RT(Xc, mdb, smin, PE_cr): PE_vc = tl.dcopy(PE_cr) PE_vs = [] lenc = len(Xc) dictMS = { 'cid': PE_cr['cid'], 'si': tl.dcopy(PE_cr['md'].si), 'rho': mdb.rho_MS, 'smin': smin, 'errV': tl.INF } (Ve, Se, Re) = om_trans.gen_forward(mdb, dictMS, lenc) errV = tl.RMSE(Xc, Ve[0:lenc]) #tl.eprint("_RT: before:%f -> after:%f"%(PE_cr['errV'], errV)) #-------------------------# #--- find first cpt ------# cpt = -1 for t in range(0, lenc): if (Re[t] == -1): cpt = t break if (cpt == -1 or PE_cr['errV'] <= errV): return (PE_vc, PE_vs) # if it cannot find better-trans, just ignore #-------------------------# # update params {PE_vc : PE_vs} #tl.eprint("tm_st:%d, tm_ed:%d, cpt:%d"%(PE_vc["tm_st"], PE_vc["tm_ed"], cpt)) PE_vc['tm_ed'] = PE_vc['tm_st'] + cpt PE_vs = { 'md': mdb.MD[Re[cpt + 1]]['medoid'], 'cid': Re[cpt + 1], 'Vc': Ve[cpt + 1:, ], 'Sc': Se[cpt + 1:, ], 'siset': {}, 'tm_st': PE_vc['tm_st'] + (cpt + 1), 'tm_ed': PE_cr['tm_ed'], 'errV': errV } #tl.eprint("tm_st:tm_ed: %d %d %d %d"%(PE_vc['tm_st'], PE_vc['tm_ed'], PE_vs['tm_st'], PE_vs['tm_ed'])) # compute errorV PE_vc['errV'] = tl.RMSE(Xc[0:cpt], Ve[0:cpt]) PE_vs['errV'] = tl.RMSE(Xc[(cpt + 1):lenc], Ve[(cpt + 1):lenc]) #tl.eprint("err_vc:%f - err_vs:%f (rho:%f)"%(PE_vc['errV'], PE_vs['errV'], mdb.rho_RE)) return (PE_vc, PE_vs)
def _FP(Xc, lene): (lenc, d) = np.shape(Xc) Ve = np.zeros((lene, d)) if (CUTNP == "MEAN"): Ve[0:lene, :] = tl.mynanmean(Xc, axis=0) if (CUTNP == "LAST"): Ve[0:lene, :] = Xc[-1, :] Se = -1 * np.ones((lene, 1)) Re = REID_NULL * np.ones(lene) errV = tl.RMSE(Xc, Ve[0:len(Xc)]) #=tl.INF # if (DBG0): tl.msg("%s ++++++ _FP(FIX): errV:inf (cid:%d) " % (IDT_F, REID_NULL)) PF = {'Ve': Ve, 'Se': Se, 'Re': Re, 'errV': errV, 'MSs': []} return PF
def _md_medoid_refinement_each(mdb, Snaps, cid_i): dps=Snaps['DPS'] Xorg=Snaps['Xorg'] md_c=tl.dcopy(mdb.MD[cid_i]['medoid']) # (a) find longest segment (tm_st, tm_ed) = _find_longest_segment(Snaps, cid_i) if(tm_st==-1 and tm_ed==-1): return (mdb, Snaps) # set current window Xc=Xorg[tm_st:tm_ed,:]; len_Xc=tm_ed-tm_st # (b) try current model vs. new model # (b-1) current model md_c.data=Xc md_c=md_c.fit_si(W_RR, dps) # si-fit using current param (Sc,Vc)=md_c.gen(len_Xc); err_cr=tl.RMSE(Xc,Vc) # (b-2) new model (estimate param using Xc) md_new=nl.NLDS(Xc, "medoid-refinement-t:%d:%d"%(tm_st, tm_ed)) md_new.data=Xc md_new=md_new.fit(W_RE) # lmfit (regime creation) (Sc,Vc)=md_new.gen(len_Xc); err_new=tl.RMSE(Xc,Vc) # (c) if it finds better model, then replace if(err_new < err_cr): md_c=md_new return (mdb, Snaps)
def _FS(mdb, Xc, lene, PE, fixORopt): md_c = PE['md'] #; si=md_c.si (Se, Ve) = md_c.gen(lene) #(Se,Ve)=md_c.forward(si, lene) Re = REID_NULL * np.ones(lene) errV = tl.RMSE(Xc, Ve[0:len(Xc)]) if (_isUnfit(Ve, mdb)): Ve[:] = np.nan Se[:] = np.nan Re[:] = REID_NULL errV = tl.INF if (DBG0): tl.msg("%s ++++++ _FS(%s): errV:%f (cid:%d) " % (IDT_F, fixORopt, errV, REID_NULL)) PF = {'Ve': Ve, 'Se': Se, 'Re': Re, 'errV': errV} return PF
def _fit_k_lin(nlds, k, kf_i=[]): data = nlds.data fn = nlds.fn # linear fit (KF) if (kf_i == []): kf = KF(data, k) # without init params else: kf = kf_i # with init params kf.em(ITER, ITERe) # em algorithm nlds.setKFParams(kf) # fit si nlds = nlds.fit_si() if (DBG): nlds.plot("%s_%d" % (fn, k)) (Sta, Obs) = nlds.gen(len(data)) err = tl.RMSE(Obs, data) if (DBG): print("nlds:fit_k_lin (k:%d):rmse: %f" % (k, err)) return (nlds, err)
def _RE(Xc, tm_st, tm_ed): # estimate new model dynamics (md_c) md_c = nl.NLDS(Xc, "t%d-%d" % (tm_st, tm_ed)) md_c = md_c.fit(W_RE) # lmfit (Sc, Vc) = md_c.gen() # generate events errV = tl.RMSE(Xc, Vc) md_c.fn = "seg:t%d-%d_e%.2f" % (tm_st, tm_ed, errV) if (DBG0): tl.msg("%s ...... _RE: errV:%f (cid:-1)" % (IDT_E, errV)) PE_cr = { 'md': md_c, 'cid': -1, 'Vc': Vc, 'Sc': Sc, 'siset': {}, 'tm_st': tm_st, 'tm_ed': tm_ed, 'errV': errV } return PE_cr
def _compute_rho_min(Xorg, lstep): (n, d) = np.shape(Xorg) lmin = max(LMIN, int(lstep * AVGR)) errs = [] for i in range(0, AVG_TRIAL): tm_st = int((n - lmin) * np.random.rand()) tm_ed = tm_st + lmin if (tm_st < 0 or tm_ed >= n): continue Xc = Xorg[tm_st:tm_ed, :] md_c = nl.NLDS(Xc, "t-%d" % (i)) md_c = md_c.fit(W_RE) # lmfit (Sc, Vc) = md_c.gen() # generate events err = tl.RMSE(Xc, Vc) errs.append(err) if (DBG1): tl.msg("trial: %d, st:%d, ed:%d, err:%.2f" % (i, tm_st, tm_ed, err)) #md_c.plot("%s_s_%d"%(mdb.fn,i)) rho = om_mdb.compute_RHO_W_ERRs(errs) tl.eprint("estimated initial rho: ", rho) #, errs) return rho
def _RR(Xc, mdb, PE_cr): tm_st = PE_cr['tm_st'] tm_ed = PE_cr['tm_ed'] Si = PE_cr['siset'] (Vc, md_c, Si, cid) = mdb.search_md(Xc, Si, DPS) if (md_c == []): if (DBG0): tl.msg("%s ...... _RR: mdb (null)" % (IDT_E)) else: (Sc, Vc) = md_c.gen() # generate events errV = tl.RMSE(Xc, Vc) md_c.fn = "seg:t%d-%d_e%.2f" % (tm_st, tm_ed, errV) if (DBG0): tl.msg("%s ...... _RR: errV:%f (cid:%d)" % (IDT_E, errV, cid)) PE_cr = { 'md': md_c, 'cid': cid, 'Vc': Vc, 'Sc': Sc, 'siset': Si, 'tm_st': tm_st, 'tm_ed': tm_ed, 'errV': errV } return PE_cr
def rmse(self): data = self.data (n, d) = np.shape(data) (Sta, Obs) = self.gen(n) return tl.RMSE(data, Obs)
def _distfunc_rmse(nlds): data = nlds.data (Sta, Obs) = nlds.gen(len(data)) return tl.RMSE(data, Obs)
rho = mdb.rho_MS #0.1 #1.0 #0.1 #1.0 data = Xorg[st:st + wd, :] print(mdb.MD) si = mdb.MD[cid]['medoid'].si #"+0.2 #*(1+np.random.rand()) lene = len(data) #200 dictMS = {'cid': cid, 'si': si, 'rho': rho, 'smin': smin} (Ve, Se, Re) = gen_forward(mdb, dictMS, lene) print(dictMS) tl.msg("plot original-est") tl.figure(1) tl.plt.subplot(221) tl.plt.plot(data, '--', color='grey') tl.plt.plot(Ve) tl.plt.title("rmse: %f" % (tl.RMSE(data, Ve))) tl.plt.ylim([np.min(data), np.max(data)]) tl.plt.subplot(422) tl.plt.plot(tl.normalizeZ(Se)) tl.plt.subplot(424) tl.plt.plot(Re) tl.eprint("========================================") tl.msg("model-switch-est") #dictMS['si']*=0.0; #mdb.MD[cid]['medoid'].si Xe = data dictMS = fit_forward(Xe, mdb, dictMS) print(dictMS) (Ve, Se, Re) = gen_forward(mdb, dictMS, ncast) #int(lene*ncast_r)) print(dictMS)
def _distfunc_rmse(Xe, mdb, dictMS): (n, d) = np.shape(Xe) lene = n (Oe, Se, Re) = om_trans.gen_forward(mdb, dictMS, lene) return tl.RMSE(Xe, Oe)
def _plot(nlds, disp=False, nf=-1, fn=''): if (nf == -1): nf = int(nlds.n) (Sta, Obs) = nlds.gen(nf) (StaL, ObsL) = nlds.gen_lin(nf) err_o = tl.RMSE(nlds.data, 0 * nlds.data) err = tl.RMSE(Obs[0:len(nlds.data), :], nlds.data) errL = tl.RMSE(ObsL[0:len(nlds.data), :], nlds.data) data = nlds.data mn = np.nanmin(data.flatten()) mx = np.nanmax(data.flatten()) mnx = abs(mx - mn) mx += 0.1 * mnx mn -= 0.1 * mnx #fig=tl.figure(10) tl.plt.clf() # tl.plt.subplot(221) tl.resetCol() tl.plt.plot(data, '-', lw=1) tl.plt.xlim([0, nf]) tl.plt.ylim([mn, mx]) tl.plt.title("Original data") # tl.plt.subplot(222) tl.resetCol() tl.plt.plot(data, '--', lw=1) tl.resetCol() tl.plt.plot(Obs, '-', lw=1) tl.plt.xlim([0, nf]) tl.plt.ylim([mn, mx]) tl.plt.title("Original vs. Estimation: err %.2f (%.2f)" % (err, err / err_o)) # tl.plt.subplot(223) tl.plt.plot(Sta, '-', lw=1) tl.plt.xlim([0, nf]) tl.plt.title("s(t) --- Sta (hidden) k:%d" % (nlds.k)) # tl.plt.subplot(224) for i in range(0, len(Obs[0])): tl.plt.fill_between(range(0, len(Obs)), Obs[:, i] + 2 * err, Obs[:, i] - 2 * err, facecolor=[0.8, 0.8, 0.8], alpha=1.0, linewidth=0.0) tl.plt.plot(Obs, '-', lw=1) tl.plt.xlim([0, nf]) tl.plt.ylim([mn, mx]) tl.plt.title("v(t) --- Obs (est)") # tl.plt.tight_layout(pad=1.0, w_pad=1.0, h_pad=1.0) if (False): tl.plt.subplot(322) tl.resetCol() tl.plt.plot(data, '--', lw=1) tl.resetCol() tl.plt.plot(ObsL, '-', lw=1) tl.plt.xlim([0, nf]) tl.plt.ylim([mn, mx]) tl.plt.title("Original vs. LDS: errL %.2f (%.2f)" % (errL, errL / err_o)) # tl.plt.subplot(324) tl.plt.plot(StaL, '-', lw=1) tl.plt.xlim([0, nf]) tl.plt.title("s(t) --- Sta (LDS) k:%d" % (nlds.k)) # tl.plt.subplot(326) for i in range(0, len(Obs[0])): tl.plt.fill_between(range(0, len(ObsL)), ObsL[:, i] + 2 * errL, ObsL[:, i] - 2 * errL, facecolor=[0.8, 0.8, 0.8], alpha=1.0, linewidth=0.0) tl.plt.plot(ObsL, '-', lw=1) tl.plt.xlim([0, nf]) tl.plt.ylim([mn, mx]) tl.plt.title("v(t) --- Obs (LDS)") # if (fn != ''): tl.savefig(fn + '_plot', 'pdf') tl.plt.draw() tl.plt.show(block=disp) if (disp != True): tl.plt.close() # save data (optional) if (FULLSAVE): tl.save_txt(data, fn + '_seq_data.txt') tl.save_txt(Sta, fn + '_seq_Sta.txt') tl.save_txt(Obs, fn + '_seq_Obs.txt') tl.save_txt(ObsL, fn + '_seq_ObsL.txt')