def _plot(model, fn, pblock):
observations=model.data
fig=pl.figure(figsize=(8, 6))
pl.subplot(311)
pl_obs=pl.plot(observations, '-', label='org')
pl.title("Original sequence")
#
(g_state, g_obs)=_gen(model)
(f_state, f_obs)=_filter(model)
#(s_state, s_obs)=_smoother(model)
#
pl.subplot(312)
pl_sta=pl.plot(f_state, label='states(filter)')
pl.title("States (filter)")
pl.subplot(313)
#pl_obs=pl.plot(observations, '--', label='org')
#tl.resetCol()
pl_gen=pl.plot(f_obs, '-', label='gen(filter)')
#pl.title("Generated (filter)")
pl.title("Generated (filter), rmse=%f"%(tl.RMSE(observations, f_obs)))
#
#pl.xlabel('time')
fig.set_tight_layout(True)
if(fn!=''): tl.savefig(fn+'_fit','pdf')
pl.show(block=pblock)
def _forward(mdb, dictMS, lene, dps, MULTI_TRANS):
dps = 1 # current setting: dps=1
#--------------------------------------#
# set current-regime md_c
idx_c = dictMS['cid']
si = dictMS['si']
rho = dictMS['rho']
smin = dictMS['smin']
#--------------------------------------#
if (not idx_c in mdb.MD):
tl.warning("_forward: cannot find %d" % (idx_c))
return ([], [], [])
#--------------------------------------#
md_c = mdb.MD[idx_c]['medoid']
d = md_c.d
k = md_c.k
#--------------------------------------#
Se = np.zeros((lene, md_c.kmax)) # hidden states
Ve = np.zeros((lene, d)) # estimated events
Re = np.zeros((lene)) # regime_id
#--------------------------------------#
#--------------------------------------#
t_c = 0
t_e = lene
# set misc parameters
misc = {'t_e': t_e, 'mdb': mdb, 'rho': rho, 'smin': smin, 'dps': dps}
# init Stack
Stc = _create_Stc(t_c, idx_c, md_c, si, Se, Ve, Re)
Final = []
Stack = []
Stack.append(Stc)
# find all candidate set
#--------------------------------------#
while (len(Stack) >= 1):
(Stack, Final) = _gen_trans(Stack, Final, misc, MULTI_TRANS)
if (DBG): tl.eprint("stack: %d" % len(Stack))
if (DBG): tl.eprint("final: %d" % len(Final))
#--------------------------------------#
stc_best = []
#--- DBG -------------#
if (tl.NO):
idx = 0
for f in Final:
tl.plt.clf()
tl.plt.subplot(211)
tl.plt.plot(f['Ve'])
tl.plt.subplot(212)
tl.plt.plot(f['Re'])
tl.savefig(
"./tmp/tc_%d_te_%d_cid_%d_idx_%d" % (t_c, t_e, idx_c, idx),
'pdf')
idx += 1
#--- DBG -------------#
#--------------------------------------#
Stc = Final[0]
Ve = Stc['Ve']
Se = Stc['Se']
Re = Stc['Re']
#--------------------------------------#
return (Ve, Se, Re, Final)
def _create_multiscale_seqs(Xorg, lstep, mscale_h, outdir):
tl.msg("_create_multiscale_seq (lstep: %d, mscale_h: %d)"%(lstep, mscale_h))
tl.save_seq_txt_pdf(Xorg, "%sXorg"%outdir)
# multi-scale sequences
XHs={}; Xagg=tl.dcopy(Xorg); outdirHs={}
wdHs={} # window size list
for i in range(0,mscale_h):
# window size at i-th level
if(i<mscale_h-1): wdHs[i]=lstep*(2**(mscale_h-1-i)) # if HT=4: 3,2,1 ... (i.e., 8xlstep, 4xlstep, 2xlstep, 1)
#if(i<mscale_h-1): wdHs[i]=lstep*(2**(mscale_h-2-i)) # if HT=4: 2,1,0 ... (i.e., 4xlstep, 2xlstep, lstep, 1)
#if(i<mscale_h-1): wdHs[i]=lstep*(2**(mscale_h-3-i)) # if HT=4: 1,0,-1 ... (i.e., 2xlstep, lstep, 1/2xlstep, 1
else: wdHs[i]=1 # if, last seq
for i in range(0,mscale_h):
XHs[i]=tl.smoothWMAo(Xagg, wdHs[i]) # compute i-th level
Xagg=Xagg-XHs[i] # delete current scale
for i in range(0,mscale_h):
#XHs[i]= XHs[i][int(wdHs[0]):,:] # delete longest-window
XHs[i][0:int(wdHs[0]),:]=tl.NAN # delete longest-window
outdirHs[i]='%s_Level%d/'%(outdir,i); tl.mkdir(outdirHs[i]) # directory
# save sequence at i-th level
tl.save_txt(XHs[i], '%sXorg.txt'%(outdirHs[i])) # save data
tl.plt.clf(); tl.plt.plot(XHs[i]) # plot and save figure
tl.savefig("%sXorg"%(outdirHs[i]), 'pdf')
tl.msg("wdHs: ")
tl.msg(wdHs)
return (XHs, wdHs, outdirHs)
def _plotResultsF(Snaps, mdb, outdir):
Xorg = Snaps['Xorg']
mn = np.nanmin(Xorg.flatten())
mx = np.nanmax(Xorg.flatten())
tl.plt.clf()
tl.plt.subplot(511)
tl.plt.plot(Xorg, ) # 'black')
tl.plt.xlim([0, len(Xorg)])
tl.plt.ylim([mn, mx])
tl.plt.ylabel('Original')
tl.plt.xticks([])
tl.plt.subplot(512)
tl.plt.plot(Xorg, 'lightgrey')
tl.resetCol()
if (SMTH):
tl.plt.plot(tl.smoothWMAo(Snaps['Vf_full'], int(Snaps['pstep'])))
else:
tl.plt.plot(Snaps['Vf_full'])
tl.plt.xlim([0, len(Xorg)])
tl.plt.ylim([mn, mx])
tl.plt.ylabel('Forecast')
tl.plt.xticks([])
tl.plt.subplot(513)
tl.plt.plot(Snaps['Es_full'], 'lightgrey')
tl.plt.plot(Snaps['Ef_full'], 'yellowgreen')
tl.plt.xlim([0, len(Xorg)])
tl.plt.ylabel('RMSE (cast)')
tl.plt.xticks([])
emx = np.nanmax(Snaps['Es_full'].flatten())
tl.plt.ylim([0, emx])
tl.plt.subplot(514)
#tl.plt.plot(Snaps['Rf_full'])
tl.plt.imshow(_createImap(mdb, Snaps['Rf_full']),
cmap='gray',
interpolation='nearest',
aspect='auto')
tl.plt.xlim([0, len(Xorg)]) #; tl.plt.yticks([])
tl.plt.yticks(range(0, mdb.get_c()), range(1, mdb.get_c() + 1))
tl.plt.ylabel('R-ID (cast)')
tl.plt.xticks([])
tl.plt.subplot(515)
tl.plt.semilogy(Snaps['T_full'], '.', color='yellowgreen')
tl.plt.xlim([0, len(Xorg)])
tl.plt.ylabel('Speed')
#tl.savefig("%sout_Vf"%(outdir),'pdf')
tl.savefig("%sout_Vf" % (outdir), 'png')
tl.plt.close()
def _combine_multi_trans(Xc, Stack, Ve, Re):
(lenc, d) = np.shape(Xc)
(lene, k) = np.shape(Ve)
r = len(Stack)
tl.eprint("combine_multi_trans: (size: d:%d, lenc:%d, #candis:%d)" %
(d, lenc, r))
Ve_r = np.zeros((r, lene, d))
Re_r = np.zeros((r, lene))
for i in range(0, r):
Ve_r[i] = Stack[i]['Ve'][0:lene]
Re_r[i] = Stack[i]['Re'][0:lene]
Vc_r = Ve_r[:, 0:lenc, :]
Wt = _decompWt(Xc, Vc_r, r, d, lenc)
Ve = np.zeros((lene, d))
Ve = _mix_Ve_r(Ve_r, Ve, r, Wt)
#--- DBG -------------#
if (tl.NO):
for i in range(0, r):
tl.plt.clf()
tl.plt.subplot(211)
tl.plt.plot(Ve_r[i])
tl.plt.plot(Xc, '--')
tl.plt.subplot(212)
tl.plt.plot(Re_r[i])
tl.plt.title(Wt)
tl.savefig("./tmp/lenc_%d_lene_%d_stack_id_%d" % (lenc, lene, i),
'pdf')
if (tl.NO):
tl.plt.clf()
tl.plt.subplot(211)
tl.plt.plot(Ve)
tl.plt.plot(Xc, '--')
tl.plt.subplot(212)
#tl.plt.plot(Rem)
tl.plt.plot(Re_r.T)
tl.plt.title(Wt)
tl.savefig("./tmp/lenc_%d_lene_%d_stack_full" % (lenc, lene), 'pdf')
#--- DBG -------------#
return (Ve)
import tool
import predict_tool as ptl
# coin_name_list = ['BTC', 'ETH']
coin_name_list = ['BTC']
coin_list = []
#### Get Data
# Coin
for coin in coin_name_list:
df = tool.loadChart(price='USDT_%s'%coin)
globals()[coin] = tool.unifyCoinDF(df)
coin_list.append(globals()[coin])
#### Predict BTC by Monte Carlo
start = 200
split = 315
train = BTC.iloc[start:split]
test = BTC['Open'].iloc[split-1:]
pred = ptl.predictMonteCarlo(train, 50, 50, runs=100)
plt.figure(figsize=(15, 5))
plt.plot(pred['Open'])
plt.plot(test)
plt.plot(pred['Best'])
plt.legend(['Train','Test', 'Predict'])
tool.savefig('monte_carlo_btc')
# plt.plot(pred)
# plt.plot(test)
# plt.legend(pred.columns)
# tool.savefig('monte_carlo_btc_10simu')
tl.msg("model-switch-est")
#dictMS['si']*=0.0; #mdb.MD[cid]['medoid'].si
Xe = data
dictMS = fit_forward(Xe, mdb, dictMS)
print(dictMS)
(Ve, Se, Re) = gen_forward(mdb, dictMS, ncast) #int(lene*ncast_r))
print(dictMS)
tl.save_txt(Xorg, '%sXorg' % (OUTDIR))
tl.save_txt(Ve, '%sVe' % (OUTDIR))
tl.save_txt(Re, '%sRe' % (OUTDIR))
tl.plt.subplot(223)
tl.plt.plot(Xorg[st:st + len(Ve), :], '--', color='lightgrey')
tl.plt.plot(data, '--', color='grey')
tl.plt.plot(Ve, '-')
tl.plt.title("rmse: %f" % (tl.RMSE(data, Ve[0:lene, :])))
tl.plt.xlim([0, len(Ve)])
tl.plt.ylim([np.min(data), np.max(data)])
tl.plt.subplot(4, 4, 11)
tl.plt.plot(tl.normalizeZ(Se))
tl.plt.subplot(4, 4, 12)
tl.plt.plot(Ve[:, 0], Ve[:, 1], '-+')
tl.plt.subplot(428)
tl.plt.plot(Re)
tl.plt.xlim([0, len(Ve)])
#
tl.plt.show()
tl.savefig('%s_tmp' % (OUTDIR), 'pdf')
def _plotSS_t_G(idx, Snaps, outdir, SnapsHs):
tc = idx
tf_st = tc + Snaps['lstep']
tf_ed = tf_st + Snaps['pstep']
n = Snaps['n']
Xorg = Snaps['Xorg']
mn = np.nanmin(Xorg.flatten())
mx = np.nanmax(Xorg.flatten())
if (DBG): tl.msg([tc, tf_st, tf_ed])
tl.plt.clf()
tl.plt.subplot(511)
tl.plt.plot(Xorg[0:idx, :]) #, 'black')
tl.plt.plot(range(idx, n), Xorg[idx:n, :], 'lightgrey')
tl.plt.xlim([0, len(Xorg)])
tl.plt.ylim([mn, mx])
tl.plt.ylabel('Original')
tl.plt.xticks([])
_plot_lines([tc], mn, mx, 0.5, 'b')
_plot_lines([tf_st, tf_ed], mn, mx, 0.5, 'r')
tl.plt.subplot(512)
tl.plt.plot(Xorg[0:idx, :], 'lightgrey')
tl.resetCol()
tl.plt.plot(Snaps['Ve_full'][0:idx]) #, 'royalblue')
tl.plt.xlim([0, len(Xorg)])
tl.plt.ylim([mn, mx])
tl.plt.ylabel('Estimation')
tl.plt.xticks([])
_plot_lines([tc], mn, mx, 0.5, 'b')
_plot_lines([tf_st, tf_ed], mn, mx, 0.5, 'r')
tl.plt.subplot(513)
tl.plt.plot(Xorg[0:tf_ed, :], 'lightgrey')
tl.resetCol()
if (SMTH):
tl.plt.plot(
tl.smoothWMAo(Snaps['Vf_full'][0:tf_ed], int(Snaps['pstep'])))
else:
tl.plt.plot(Snaps['Vf_full'][0:tf_ed])
tl.plt.xlim([0, len(Xorg)])
tl.plt.ylim([mn, mx])
tl.plt.ylabel('Forecast')
tl.plt.xticks([])
_plot_lines([tc], mn, mx, 0.5, 'b')
_plot_lines([tf_st, tf_ed], mn, mx, 0.5, 'r')
tl.plt.subplot(514)
tl.plt.plot(Snaps['Ef_full'][0:idx], 'yellowgreen')
tl.plt.plot(Snaps['Es_full'][0:idx], 'lightgrey')
tl.plt.xlim([0, len(Xorg)])
tl.plt.ylabel('RMSE')
tl.plt.xticks([])
emx = np.nanmax(Snaps['Es_full'].flatten())
_plot_lines([tc], 0, emx, 0.5, 'b')
_plot_lines([tf_st, tf_ed], 0, emx, 0.5, 'r')
tl.plt.ylim([0, emx])
tl.plt.subplot(515)
tmx = -tl.INF
tmn = tl.INF
for i in range(0, MSCALE_H):
tl.plt.semilogy(SnapsHs[i]['T_full'][0:idx], '.', label='h=%d' % (i))
mn = np.nanmin(SnapsHs[i]['T_full'])
mx = np.nanmax(SnapsHs[i]['T_full'])
if (tmx < mx): tmx = mx
if (tmn > mn): tmn = mn
tl.plt.xlim([0, len(Xorg)])
_plot_lines([tc], tmn, tmx, 0.5, 'b')
_plot_lines([tf_st, tf_ed], tmn, tmx, 0.5, 'r')
tl.plt.ylim([tmn, tmx])
tl.plt.ylabel('Speed')
if (outdir != ''):
tl.savefig("%sresult_G" % (outdir), 'pdf')
tl.savefig("%sresult_G" % (outdir), 'png')
def _plot(nlds, disp=False, nf=-1, fn=''):
if (nf == -1): nf = int(nlds.n)
(Sta, Obs) = nlds.gen(nf)
(StaL, ObsL) = nlds.gen_lin(nf)
err_o = tl.RMSE(nlds.data, 0 * nlds.data)
err = tl.RMSE(Obs[0:len(nlds.data), :], nlds.data)
errL = tl.RMSE(ObsL[0:len(nlds.data), :], nlds.data)
data = nlds.data
mn = np.nanmin(data.flatten())
mx = np.nanmax(data.flatten())
mnx = abs(mx - mn)
mx += 0.1 * mnx
mn -= 0.1 * mnx
#fig=tl.figure(10)
tl.plt.clf()
#
tl.plt.subplot(221)
tl.resetCol()
tl.plt.plot(data, '-', lw=1)
tl.plt.xlim([0, nf])
tl.plt.ylim([mn, mx])
tl.plt.title("Original data")
#
tl.plt.subplot(222)
tl.resetCol()
tl.plt.plot(data, '--', lw=1)
tl.resetCol()
tl.plt.plot(Obs, '-', lw=1)
tl.plt.xlim([0, nf])
tl.plt.ylim([mn, mx])
tl.plt.title("Original vs. Estimation: err %.2f (%.2f)" %
(err, err / err_o))
#
tl.plt.subplot(223)
tl.plt.plot(Sta, '-', lw=1)
tl.plt.xlim([0, nf])
tl.plt.title("s(t) --- Sta (hidden) k:%d" % (nlds.k))
#
tl.plt.subplot(224)
for i in range(0, len(Obs[0])):
tl.plt.fill_between(range(0, len(Obs)),
Obs[:, i] + 2 * err,
Obs[:, i] - 2 * err,
facecolor=[0.8, 0.8, 0.8],
alpha=1.0,
linewidth=0.0)
tl.plt.plot(Obs, '-', lw=1)
tl.plt.xlim([0, nf])
tl.plt.ylim([mn, mx])
tl.plt.title("v(t) --- Obs (est)")
#
tl.plt.tight_layout(pad=1.0, w_pad=1.0, h_pad=1.0)
if (False):
tl.plt.subplot(322)
tl.resetCol()
tl.plt.plot(data, '--', lw=1)
tl.resetCol()
tl.plt.plot(ObsL, '-', lw=1)
tl.plt.xlim([0, nf])
tl.plt.ylim([mn, mx])
tl.plt.title("Original vs. LDS: errL %.2f (%.2f)" %
(errL, errL / err_o))
#
tl.plt.subplot(324)
tl.plt.plot(StaL, '-', lw=1)
tl.plt.xlim([0, nf])
tl.plt.title("s(t) --- Sta (LDS) k:%d" % (nlds.k))
#
tl.plt.subplot(326)
for i in range(0, len(Obs[0])):
tl.plt.fill_between(range(0, len(ObsL)),
ObsL[:, i] + 2 * errL,
ObsL[:, i] - 2 * errL,
facecolor=[0.8, 0.8, 0.8],
alpha=1.0,
linewidth=0.0)
tl.plt.plot(ObsL, '-', lw=1)
tl.plt.xlim([0, nf])
tl.plt.ylim([mn, mx])
tl.plt.title("v(t) --- Obs (LDS)")
#
if (fn != ''): tl.savefig(fn + '_plot', 'pdf')
tl.plt.draw()
tl.plt.show(block=disp)
if (disp != True): tl.plt.close()
# save data (optional)
if (FULLSAVE):
tl.save_txt(data, fn + '_seq_data.txt')
tl.save_txt(Sta, fn + '_seq_Sta.txt')
tl.save_txt(Obs, fn + '_seq_Obs.txt')
tl.save_txt(ObsL, fn + '_seq_ObsL.txt')
