def _update_ms_insert(mdb, PE_fr, PE_to):
idx_fr=PE_fr['cid']; idx_to=PE_to['cid'];
# if not exist, just ignore
if(idx_fr == -1 or idx_to == -1): return mdb
# create new ms record
ms_new=_create_new_ms_rcd(PE_fr, PE_to)
# (a) find fr_to in mdb
idxs="%d_%d"%(idx_fr, idx_to)
if(not (idxs in mdb.MS)): mdb.MS[idxs]=[]
# best-candidate
ms_best=-1; err_best=tl.INF
# (a-i) find closest mss
for ms_idxs_i in mdb.MS[idxs]:
ms_d=ms_idxs_i['medoid']
err_i=0.5*( tl.RMSE(ms_d['vn_fr'],ms_new['vn_fr']) + tl.RMSE(ms_d['v0_to'],ms_new['v0_to']) )
#tl.eprint(mdb.rho_MS, err_i)
#tl.eprint(ms_d['vn_fr'], ms_new['vn_fr'], ms_d['v0_to'],ms_new['v0_to'])
if(err_i<err_best):
ms_best=ms_idxs_i; err_best=err_i;
#tl.eprint("err_best: %f (%f)"%(err_best, mdb.rho_MS))
# (b) update MS[idxs]
if(err_best<mdb.rho_MS): # if we have similar ms
ms_best['object'].append(ms_new)
if(DBG): tl.msg("MDB-MS (insert): # of object=%d"%(len(ms_best['object'])))
else: # if unknown, then create new MS
mdb.MS[idxs].append({'medoid':ms_new, 'object':[ms_new]})
return mdb
def _FM(mdb, Xc, lene, smin, PE, fixORopt):
# dictMS (see, om_trans.py)
dictMS = {
'cid': PE['cid'],
'si': tl.dcopy(PE['md'].si),
'rho': mdb.rho_MS,
'smin': smin,
'errV': tl.INF
}
if (fixORopt == 'OPT'):
dictMS = om_trans.fit_forward(Xc, mdb, dictMS, W_FM) #, DPS)
(Ve, Se, Re) = om_trans.gen_forward_multi_trans(Xc, mdb, dictMS, lene,
MULTI_TRANS)
#(Ve, Se, Re)=om_trans.gen_forward(mdb, dictMS, lene)
errV = tl.RMSE(Xc, Ve[0:len(Xc)])
if (_isUnfit(Ve, mdb)):
Ve[:] = np.nan
Se[:] = np.nan
Re[:] = REID_NULL
errV = tl.INF
if (DBG0):
tl.msg("%s ++++++ _FM(%s): errV:%f (cid:%d) " %
(IDT_F, fixORopt, errV, dictMS['cid']))
PF = {'Ve': Ve, 'Se': Se, 'Re': Re, 'errV': errV}
return PF
def _plot(model, fn, pblock):
observations=model.data
fig=pl.figure(figsize=(8, 6))
pl.subplot(311)
pl_obs=pl.plot(observations, '-', label='org')
pl.title("Original sequence")
#
(g_state, g_obs)=_gen(model)
(f_state, f_obs)=_filter(model)
#(s_state, s_obs)=_smoother(model)
#
pl.subplot(312)
pl_sta=pl.plot(f_state, label='states(filter)')
pl.title("States (filter)")
pl.subplot(313)
#pl_obs=pl.plot(observations, '--', label='org')
#tl.resetCol()
pl_gen=pl.plot(f_obs, '-', label='gen(filter)')
#pl.title("Generated (filter)")
pl.title("Generated (filter), rmse=%f"%(tl.RMSE(observations, f_obs)))
#
#pl.xlabel('time')
fig.set_tight_layout(True)
if(fn!=''): tl.savefig(fn+'_fit','pdf')
pl.show(block=pblock)
def _RU(Xc, PE_cr, fixORopt):
tm_st = PE_cr['tm_st']
tm_ed = PE_cr['tm_ed']
if (_isNullP(PE_cr)):
if (DBG0): tl.msg("%s ...... _RU: (null)" % (IDT_E))
else:
cid = PE_cr['cid'] # put cid
md_c = tl.dcopy(PE_cr['md']) # copy model
md_c.n = len(Xc)
md_c.data = Xc # copy data
if (fixORopt == 'OPT'):
md_c = md_c.fit_si(W_RR, DPS) # update init si (i.e., s(0)=si)
(Sc, Vc) = md_c.gen() # generate events
errV = tl.RMSE(Xc, Vc)
md_c.fn = "seg:t%d-%d_e%.2f" % (tm_st, tm_ed, errV)
if (DBG0):
tl.msg("%s ...... _RU: errV:%f (cid:%d)" % (IDT_E, errV, cid))
PE_cr = {
'md': md_c,
'cid': cid,
'Vc': Vc,
'Sc': Sc,
'siset': {},
'tm_st': tm_st,
'tm_ed': tm_ed,
'errV': errV
}
return PE_cr
def _find_nearest_t(Vs, vec, smin):
n = len(Vs)
errs = tl.INF * np.ones((n))
for it in range(smin, n - LST_MINS):
errs[it] = tl.RMSE(Vs[it], vec)
t_best = np.argmin(errs)
err_best = errs[t_best]
return (t_best, err_best)
def _RT(Xc, mdb, smin, PE_cr):
PE_vc = tl.dcopy(PE_cr)
PE_vs = []
lenc = len(Xc)
dictMS = {
'cid': PE_cr['cid'],
'si': tl.dcopy(PE_cr['md'].si),
'rho': mdb.rho_MS,
'smin': smin,
'errV': tl.INF
}
(Ve, Se, Re) = om_trans.gen_forward(mdb, dictMS, lenc)
errV = tl.RMSE(Xc, Ve[0:lenc])
#tl.eprint("_RT: before:%f -> after:%f"%(PE_cr['errV'], errV))
#-------------------------#
#--- find first cpt ------#
cpt = -1
for t in range(0, lenc):
if (Re[t] == -1):
cpt = t
break
if (cpt == -1 or PE_cr['errV'] <= errV):
return (PE_vc, PE_vs) # if it cannot find better-trans, just ignore
#-------------------------#
# update params {PE_vc : PE_vs}
#tl.eprint("tm_st:%d, tm_ed:%d, cpt:%d"%(PE_vc["tm_st"], PE_vc["tm_ed"], cpt))
PE_vc['tm_ed'] = PE_vc['tm_st'] + cpt
PE_vs = {
'md': mdb.MD[Re[cpt + 1]]['medoid'],
'cid': Re[cpt + 1],
'Vc': Ve[cpt + 1:, ],
'Sc': Se[cpt + 1:, ],
'siset': {},
'tm_st': PE_vc['tm_st'] + (cpt + 1),
'tm_ed': PE_cr['tm_ed'],
'errV': errV
}
#tl.eprint("tm_st:tm_ed: %d %d %d %d"%(PE_vc['tm_st'], PE_vc['tm_ed'], PE_vs['tm_st'], PE_vs['tm_ed']))
# compute errorV
PE_vc['errV'] = tl.RMSE(Xc[0:cpt], Ve[0:cpt])
PE_vs['errV'] = tl.RMSE(Xc[(cpt + 1):lenc], Ve[(cpt + 1):lenc])
#tl.eprint("err_vc:%f - err_vs:%f (rho:%f)"%(PE_vc['errV'], PE_vs['errV'], mdb.rho_RE))
return (PE_vc, PE_vs)
def _FP(Xc, lene):
(lenc, d) = np.shape(Xc)
Ve = np.zeros((lene, d))
if (CUTNP == "MEAN"): Ve[0:lene, :] = tl.mynanmean(Xc, axis=0)
if (CUTNP == "LAST"): Ve[0:lene, :] = Xc[-1, :]
Se = -1 * np.ones((lene, 1))
Re = REID_NULL * np.ones(lene)
errV = tl.RMSE(Xc, Ve[0:len(Xc)]) #=tl.INF #
if (DBG0):
tl.msg("%s ++++++ _FP(FIX): errV:inf (cid:%d) " % (IDT_F, REID_NULL))
PF = {'Ve': Ve, 'Se': Se, 'Re': Re, 'errV': errV, 'MSs': []}
return PF
def _md_medoid_refinement_each(mdb, Snaps, cid_i):
dps=Snaps['DPS']
Xorg=Snaps['Xorg']
md_c=tl.dcopy(mdb.MD[cid_i]['medoid'])
# (a) find longest segment
(tm_st, tm_ed) = _find_longest_segment(Snaps, cid_i)
if(tm_st==-1 and tm_ed==-1): return (mdb, Snaps)
# set current window
Xc=Xorg[tm_st:tm_ed,:]; len_Xc=tm_ed-tm_st
# (b) try current model vs. new model
# (b-1) current model
md_c.data=Xc
md_c=md_c.fit_si(W_RR, dps) # si-fit using current param
(Sc,Vc)=md_c.gen(len_Xc); err_cr=tl.RMSE(Xc,Vc)
# (b-2) new model (estimate param using Xc)
md_new=nl.NLDS(Xc, "medoid-refinement-t:%d:%d"%(tm_st, tm_ed))
md_new.data=Xc
md_new=md_new.fit(W_RE) # lmfit (regime creation)
(Sc,Vc)=md_new.gen(len_Xc); err_new=tl.RMSE(Xc,Vc)
# (c) if it finds better model, then replace
if(err_new < err_cr): md_c=md_new
return (mdb, Snaps)
def _FS(mdb, Xc, lene, PE, fixORopt):
md_c = PE['md'] #; si=md_c.si
(Se, Ve) = md_c.gen(lene) #(Se,Ve)=md_c.forward(si, lene)
Re = REID_NULL * np.ones(lene)
errV = tl.RMSE(Xc, Ve[0:len(Xc)])
if (_isUnfit(Ve, mdb)):
Ve[:] = np.nan
Se[:] = np.nan
Re[:] = REID_NULL
errV = tl.INF
if (DBG0):
tl.msg("%s ++++++ _FS(%s): errV:%f (cid:%d) " %
(IDT_F, fixORopt, errV, REID_NULL))
PF = {'Ve': Ve, 'Se': Se, 'Re': Re, 'errV': errV}
return PF
def _fit_k_lin(nlds, k, kf_i=[]):
data = nlds.data
fn = nlds.fn
# linear fit (KF)
if (kf_i == []):
kf = KF(data, k) # without init params
else:
kf = kf_i # with init params
kf.em(ITER, ITERe) # em algorithm
nlds.setKFParams(kf)
# fit si
nlds = nlds.fit_si()
if (DBG): nlds.plot("%s_%d" % (fn, k))
(Sta, Obs) = nlds.gen(len(data))
err = tl.RMSE(Obs, data)
if (DBG): print("nlds:fit_k_lin (k:%d):rmse: %f" % (k, err))
return (nlds, err)
def _RE(Xc, tm_st, tm_ed):
# estimate new model dynamics (md_c)
md_c = nl.NLDS(Xc, "t%d-%d" % (tm_st, tm_ed))
md_c = md_c.fit(W_RE) # lmfit
(Sc, Vc) = md_c.gen() # generate events
errV = tl.RMSE(Xc, Vc)
md_c.fn = "seg:t%d-%d_e%.2f" % (tm_st, tm_ed, errV)
if (DBG0): tl.msg("%s ...... _RE: errV:%f (cid:-1)" % (IDT_E, errV))
PE_cr = {
'md': md_c,
'cid': -1,
'Vc': Vc,
'Sc': Sc,
'siset': {},
'tm_st': tm_st,
'tm_ed': tm_ed,
'errV': errV
}
return PE_cr
def _compute_rho_min(Xorg, lstep):
(n, d) = np.shape(Xorg)
lmin = max(LMIN, int(lstep * AVGR))
errs = []
for i in range(0, AVG_TRIAL):
tm_st = int((n - lmin) * np.random.rand())
tm_ed = tm_st + lmin
if (tm_st < 0 or tm_ed >= n): continue
Xc = Xorg[tm_st:tm_ed, :]
md_c = nl.NLDS(Xc, "t-%d" % (i))
md_c = md_c.fit(W_RE) # lmfit
(Sc, Vc) = md_c.gen() # generate events
err = tl.RMSE(Xc, Vc)
errs.append(err)
if (DBG1):
tl.msg("trial: %d, st:%d, ed:%d, err:%.2f" %
(i, tm_st, tm_ed, err))
#md_c.plot("%s_s_%d"%(mdb.fn,i))
rho = om_mdb.compute_RHO_W_ERRs(errs)
tl.eprint("estimated initial rho: ", rho) #, errs)
return rho
def _RR(Xc, mdb, PE_cr):
tm_st = PE_cr['tm_st']
tm_ed = PE_cr['tm_ed']
Si = PE_cr['siset']
(Vc, md_c, Si, cid) = mdb.search_md(Xc, Si, DPS)
if (md_c == []):
if (DBG0): tl.msg("%s ...... _RR: mdb (null)" % (IDT_E))
else:
(Sc, Vc) = md_c.gen() # generate events
errV = tl.RMSE(Xc, Vc)
md_c.fn = "seg:t%d-%d_e%.2f" % (tm_st, tm_ed, errV)
if (DBG0):
tl.msg("%s ...... _RR: errV:%f (cid:%d)" % (IDT_E, errV, cid))
PE_cr = {
'md': md_c,
'cid': cid,
'Vc': Vc,
'Sc': Sc,
'siset': Si,
'tm_st': tm_st,
'tm_ed': tm_ed,
'errV': errV
}
return PE_cr
def rmse(self):
data = self.data
(n, d) = np.shape(data)
(Sta, Obs) = self.gen(n)
return tl.RMSE(data, Obs)
def _distfunc_rmse(nlds):
data = nlds.data
(Sta, Obs) = nlds.gen(len(data))
return tl.RMSE(data, Obs)
rho = mdb.rho_MS #0.1 #1.0 #0.1 #1.0
data = Xorg[st:st + wd, :]
print(mdb.MD)
si = mdb.MD[cid]['medoid'].si #"+0.2 #*(1+np.random.rand())
lene = len(data) #200
dictMS = {'cid': cid, 'si': si, 'rho': rho, 'smin': smin}
(Ve, Se, Re) = gen_forward(mdb, dictMS, lene)
print(dictMS)
tl.msg("plot original-est")
tl.figure(1)
tl.plt.subplot(221)
tl.plt.plot(data, '--', color='grey')
tl.plt.plot(Ve)
tl.plt.title("rmse: %f" % (tl.RMSE(data, Ve)))
tl.plt.ylim([np.min(data), np.max(data)])
tl.plt.subplot(422)
tl.plt.plot(tl.normalizeZ(Se))
tl.plt.subplot(424)
tl.plt.plot(Re)
tl.eprint("========================================")
tl.msg("model-switch-est")
#dictMS['si']*=0.0; #mdb.MD[cid]['medoid'].si
Xe = data
dictMS = fit_forward(Xe, mdb, dictMS)
print(dictMS)
(Ve, Se, Re) = gen_forward(mdb, dictMS, ncast) #int(lene*ncast_r))
print(dictMS)
def _distfunc_rmse(Xe, mdb, dictMS):
(n, d) = np.shape(Xe)
lene = n
(Oe, Se, Re) = om_trans.gen_forward(mdb, dictMS, lene)
return tl.RMSE(Xe, Oe)
def _plot(nlds, disp=False, nf=-1, fn=''):
if (nf == -1): nf = int(nlds.n)
(Sta, Obs) = nlds.gen(nf)
(StaL, ObsL) = nlds.gen_lin(nf)
err_o = tl.RMSE(nlds.data, 0 * nlds.data)
err = tl.RMSE(Obs[0:len(nlds.data), :], nlds.data)
errL = tl.RMSE(ObsL[0:len(nlds.data), :], nlds.data)
data = nlds.data
mn = np.nanmin(data.flatten())
mx = np.nanmax(data.flatten())
mnx = abs(mx - mn)
mx += 0.1 * mnx
mn -= 0.1 * mnx
#fig=tl.figure(10)
tl.plt.clf()
#
tl.plt.subplot(221)
tl.resetCol()
tl.plt.plot(data, '-', lw=1)
tl.plt.xlim([0, nf])
tl.plt.ylim([mn, mx])
tl.plt.title("Original data")
#
tl.plt.subplot(222)
tl.resetCol()
tl.plt.plot(data, '--', lw=1)
tl.resetCol()
tl.plt.plot(Obs, '-', lw=1)
tl.plt.xlim([0, nf])
tl.plt.ylim([mn, mx])
tl.plt.title("Original vs. Estimation: err %.2f (%.2f)" %
(err, err / err_o))
#
tl.plt.subplot(223)
tl.plt.plot(Sta, '-', lw=1)
tl.plt.xlim([0, nf])
tl.plt.title("s(t) --- Sta (hidden) k:%d" % (nlds.k))
#
tl.plt.subplot(224)
for i in range(0, len(Obs[0])):
tl.plt.fill_between(range(0, len(Obs)),
Obs[:, i] + 2 * err,
Obs[:, i] - 2 * err,
facecolor=[0.8, 0.8, 0.8],
alpha=1.0,
linewidth=0.0)
tl.plt.plot(Obs, '-', lw=1)
tl.plt.xlim([0, nf])
tl.plt.ylim([mn, mx])
tl.plt.title("v(t) --- Obs (est)")
#
tl.plt.tight_layout(pad=1.0, w_pad=1.0, h_pad=1.0)
if (False):
tl.plt.subplot(322)
tl.resetCol()
tl.plt.plot(data, '--', lw=1)
tl.resetCol()
tl.plt.plot(ObsL, '-', lw=1)
tl.plt.xlim([0, nf])
tl.plt.ylim([mn, mx])
tl.plt.title("Original vs. LDS: errL %.2f (%.2f)" %
(errL, errL / err_o))
#
tl.plt.subplot(324)
tl.plt.plot(StaL, '-', lw=1)
tl.plt.xlim([0, nf])
tl.plt.title("s(t) --- Sta (LDS) k:%d" % (nlds.k))
#
tl.plt.subplot(326)
for i in range(0, len(Obs[0])):
tl.plt.fill_between(range(0, len(ObsL)),
ObsL[:, i] + 2 * errL,
ObsL[:, i] - 2 * errL,
facecolor=[0.8, 0.8, 0.8],
alpha=1.0,
linewidth=0.0)
tl.plt.plot(ObsL, '-', lw=1)
tl.plt.xlim([0, nf])
tl.plt.ylim([mn, mx])
tl.plt.title("v(t) --- Obs (LDS)")
#
if (fn != ''): tl.savefig(fn + '_plot', 'pdf')
tl.plt.draw()
tl.plt.show(block=disp)
if (disp != True): tl.plt.close()
# save data (optional)
if (FULLSAVE):
tl.save_txt(data, fn + '_seq_data.txt')
tl.save_txt(Sta, fn + '_seq_Sta.txt')
tl.save_txt(Obs, fn + '_seq_Obs.txt')
tl.save_txt(ObsL, fn + '_seq_ObsL.txt')
