def return_stats_for_various_vol():
univ_ib_cl = cr_cret.retrieve(univ_ib_ext + "Close")
univ_ib_rt1 = univ_ib_cl.copy()
univ_ib_rt2 = univ_ib_cl.copy()
univ_ib_rt3 = univ_ib_cl.copy()
for i in univ_ib_cl.tick_cols():
univ_ib_cl[i] = filt.ret(univ_ib_cl[i].values)
for i in univ_ib_rt1.tick_cols():
univ_ib_rt1[i] = filt.ret(univ_ib_rt1[i].values, 27)
univ_ib_rt1[i] = filt.lag(univ_ib_rt1[i].values)
for i in univ_ib_rt2.tick_cols():
univ_ib_rt2[i] = filt.ret(univ_ib_rt2[i].values, 80)
univ_ib_rt2[i] = filt.lag(univ_ib_rt2[i].values)
for i in univ_ib_rt3.tick_cols():
univ_ib_rt3[i] = filt.ret(univ_ib_rt3[i].values, 240)
univ_ib_rt3[i] = filt.lag(univ_ib_rt3[i].values)
univ_ib_cl_data = univ_ib_cl[univ_ib_cl.tick_cols()].values.reshape(-1)
univ_ib_rt1_data = univ_ib_rt1[univ_ib_rt1.tick_cols()].values.reshape(-1)
univ_ib_rt2_data = univ_ib_rt2[univ_ib_rt2.tick_cols()].values.reshape(-1)
univ_ib_rt3_data = univ_ib_rt3[univ_ib_rt3.tick_cols()].values.reshape(-1)
univ_ib_cl_data = np.abs(univ_ib_cl_data)
import warnings
warnings.simplefilter("ignore", RuntimeWarning)
univ_ib_rt1_data = np.sign(univ_ib_rt1_data)
univ_ib_rt2_data = np.sign(univ_ib_rt2_data)
univ_ib_rt3_data = np.sign(univ_ib_rt3_data)
warnings.simplefilter("default", RuntimeWarning)
for j in vol_names:
for k in range(30, 330, 30):
if k < 100:
univ_ib_vol = cr_vol_all.retrieve(univ_ib_ext + j + "_0" + str(k) + "D")
else:
univ_ib_vol = cr_vol_all.retrieve(univ_ib_ext + j + "_" + str(k) + "D")
for i in univ_ib_vol.tick_cols():
univ_ib_vol[i] = filt.lag(univ_ib_vol[i].values)
univ_ib_vol_data = univ_ib_vol[univ_ib_vol.tick_cols()].values.reshape(-1)
# univ_ib_nn = ~np.isnan(univ_ib_cl_data) & ~np.isnan(univ_ib_vol_data)
univ_ib_nn = ~np.isnan(univ_ib_cl_data) & ~np.isnan(univ_ib_vol_data) & (univ_ib_rt1_data < 0)
univ_ib_vol_data = univ_ib_vol_data[univ_ib_nn]
univ_ib_cl_data_ = univ_ib_cl_data[univ_ib_nn]
rho = ss.spearmanr(univ_ib_vol_data, univ_ib_cl_data_).correlation
if k < 100:
print(j + "_0" + str(k) + "D:\t", rho)
else:
print(j + "_" + str(k) + "D:\t", rho)
print("\n")
def gradient_boosting_result(_ticker='SP500'):
univ_ib_gd = cr_cret.retrieve(univ_ib_eqidx_ext + 'ExchOpen')[_ticker].values
z = list(np.where(univ_ib_gd.astype('int') == 1)[0])
_prc_names = ['CLS', 'A2D', 'A3D', 'A4D', 'A2W', 'A3W', 'A4W', 'A2M', 'A3M', 'A4M']
_trn_names = ['LVL', 'CH0', 'CH1', 'CH2']
_fil_names = ['LRB', 'QRB', 'QRG']
_hoz_names = []
for j in range(5, 305, 5):
if j < 10:
_hoz_names.append('00' + str(j))
elif j < 100:
_hoz_names.append('0' + str(j))
else:
_hoz_names.append(str(j))
# get volatility forecast
univ_ib_vl = cr_vol_all_adj.retrieve(univ_ib_eqidx_ext + 'vol_pb_120')[_ticker].values[z]
# get return
univ_ib_cl = cr_cret.retrieve(univ_ib_eqidx_ext + 'Close')[_ticker].values[z]
univ_ib_cl = filt.ret(univ_ib_cl)
univ_ib_sig_all = None
for k in _trn_names:
for i in _prc_names:
for j in _fil_names:
for mi, m in enumerate(_hoz_names):
fn = univ_ib_eqidx_ext+i+'_'+j+'_'+k+'_'+m
univ_ib_sig = cr_sig_mr_sg.retrieve(fn)[_ticker].values[z]
if univ_ib_sig_all is None:
univ_ib_sig_all = DataFrame({i+'_'+j+'_'+k+'_'+m: univ_ib_sig})
else:
univ_ib_sig_all.col_bind(DataFrame({i+'_'+j+'_'+k+'_'+m: univ_ib_sig}))
# just for storage - incase something happens
# univ_ib_gb = DataFrame({'Close': univ_ib_cl, 'Volatility': univ_ib_vl})
# univ_ib_gb.col_bind(univ_ib_sig_all)
# cr_cret.store(univ_ib_eqidx_ext + 'GBM', univ_ib_gb)
reg_lookback = [120, 240, 360, 480]
new_col_names = list(univ_ib_sig_all.columns)
for i in new_col_names:
# i = new_col_names[0]
test_ = np.empty(0)
for j in reg_lookback:
# j = reg_lookback[0]
uic = univ_ib_cl
uiv = univ_ib_vl
uis = filt.lag(univ_ib_sig_all[i].values, 1)
uic, uis, uiv = reduce_nonnan(uic, uis, uiv)
b1 = qreg.roll_e_ladreg_1d(uic, uis, j)
b2 = qreg.roll_e_ladreg_1d(uic/uiv, uis/uiv, j)
resid1 = uic - filt.lag(b1)*uis
resid2 = uic - filt.lag(b2)*uis
resid1, resid2 = reduce_nonnan(resid1, resid2)
test_ = np.hstack((test_, np.array([np.median(abs(resid1)), np.median(abs(resid2))])))
print(i+' : '+np_to_str(test_))
def response_curve(x1, _ticker='SP500', f=None, md=True, sigd=False):
univ_ib_gd = cr_cret.retrieve(univ_ib_eqidx_ext + 'ExchOpen')[_ticker].values
univ_ib_vl = cr_vol_all_adj.retrieve(univ_ib_eqidx_ext + 'vol_pb_120')[_ticker].values
univ_ib_s1 = cr_sig_mr_sg.retrieve(univ_ib_eqidx_ext + x1)[_ticker].values
univ_ib_cl = cr_cret.retrieve(univ_ib_eqidx_ext + 'Close')[_ticker].values
z = list(np.where(univ_ib_gd.astype('int') == 1)[0])
univ_ib_s1 = univ_ib_s1[z]
univ_ib_cl = univ_ib_cl[z]
univ_ib_vl = univ_ib_vl[z]
univ_ib_cl = filt.ret(univ_ib_cl)/filt.lag(univ_ib_vl, 1)
univ_ib_s1 = filt.lag(univ_ib_s1, 1)/filt.lag(univ_ib_vl, 1)
univ_ib_cl, univ_ib_s1 = reduce_nonnan(univ_ib_cl, univ_ib_s1)
_bins = 20
_range = np.maximum(np.percentile(univ_ib_s1, 99), -np.percentile(univ_ib_s1, 1))
_delta = _range/_bins
if f is not None:
pyl.figure(f)
else:
pyl.figure(1)
for i in range(0, 16):
if i == 0:
uis1 = univ_ib_s1
uic1 = univ_ib_cl
else:
uis1 = filt.lag(univ_ib_s1, i)
uic1 = filt.sma(univ_ib_cl, i+1)
uis1, uic1 = reduce_nonnan(uis1, uic1)
uis1_b = np.linspace(-_range, _range, num=_bins+1)
uic1_b = np.zeros(_bins+1)*np.nan
for j in range(0, _bins+1):
# j = 1
if j==0:
tmp__ = np.where(uis1 <= uis1_b[j]+_delta)[0]
elif j == _bins+1:
tmp__ = np.where(uis1 > uis1_b[j]-_delta)[0]
else:
tmp__ = np.where((uis1 <= uis1_b[j]+_delta) & (uis1 > uis1_b[j]-_delta))[0]
if tmp__.shape[0] > 0:
if md:
if not sigd:
uic1_b[j] = np.nanmedian(uic1[tmp__]) #/np.nanstd(uic1[tmp__])
else:
uic1_b[j] = np.nanmedian(uic1[tmp__])/np.nanstd(uic1[tmp__])
else:
if not sigd:
uic1_b[j] = np.nanmean(uic1[tmp__]) #/np.nanstd(uic1[tmp__])
else:
uic1_b[j] = np.nanmean(uic1[tmp__])/np.nanstd(uic1[tmp__])
pyl.subplot(4, 4, i+1)
pyl.plot(uis1_b, uic1_b)
def get_best_fit(k):
# for k in range(5, 16):
good1k = (k - 1) / (k + 1)
good1k_ = k / (k + 2)
print('Looking for EMA %s equivalent with autocorr = %s' % (str(k), str(good1k)))
# metric = [np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan]
good2b = np.nan
for n1 in range(10, 40, 5): # range(20, 160, 20):
# n1 = 160
# print('Processing n1 = %s' % str(n1))
for n2 in range(5, 50, 5):
# n2 = 100
for f1 in range(10, 60, 5): #range(10, 300, 10): # range(2, 100, 2):
# f1 = 200
for f2 in range(6, 48, 2): # range(2, 36, 2):
# f2 = 28
for lthresh in range(0, 1, 1): # (0, 12, 1)
# lthresh = 1
for oshpr in range(7, 30, 1):
# oshpr = 28
_loss = n1 + n2 + f1 + f2 + oshpr
if _loss <= 250: # 150
if np.isnan(good2b) | (~np.isnan(good2b) & (good2b > 0.0001)):
# print(n1, n2, f1, f2, lthresh, oshpr)
test1p = filt.smth_price(test3p_, test2p_, n1, n2, f1, f2, lthresh, oshpr)
test1p, test2p, test3p = reduce_nonnan(test1p, test2p_, test3p_)
# test1r =
good1 = autocorr(filt.ret(test1p)[-7000:])
if (good1 >= good1k) & (good1 < good1k_):
# get the overshoot measure
xoc_mx = test1p-np.maximum(test2p, test3p)
xoc_mn = test1p-np.minimum(test2p, test3p)
good2_ = np.abs(200*((xoc_mx>0)*xoc_mx+(xoc_mn<0)*xoc_mn)/(test2p+test3p))
good2 = np.mean(good2_[-7000:])*2/(k+1)+np.median(good2_[-7000:])*(k-1)/(k+1)
good3 = good2
good2 = good2*np.sqrt(_loss)
if np.isnan(good2b):
metric = [k, n1, n2, f1, f2, lthresh, oshpr, good1, good3, _loss, good2]
good2b = good2
print(metric)
elif good2 < good2b:
metric = [k, n1, n2, f1, f2, lthresh, oshpr, good1, good3, _loss, good2]
good2b = good2
print(metric)
return None
def pred_stats_for_various_vol():
univ_ib_cl = cr_cret.retrieve(univ_ib_ext + "Close")
univ_ib_rt = univ_ib_cl.copy()
for i in univ_ib_cl.tick_cols():
univ_ib_cl[i] = filt.ret(univ_ib_cl[i].values)
for i in univ_ib_rt.tick_cols():
univ_ib_rt[i] = filt.ret(univ_ib_rt[i].values, 30)
univ_ib_rt[i] = filt.lag(univ_ib_rt[i].values)
univ_ib_cl_data = univ_ib_cl[univ_ib_cl.tick_cols()].values.reshape(-1)
univ_ib_rt_data = univ_ib_rt[univ_ib_rt.tick_cols()].values.reshape(-1)
# univ_ib_cl_data = np.abs(univ_ib_cl_data)
import warnings
warnings.simplefilter("ignore", RuntimeWarning)
univ_ib_rt_data = np.sign(univ_ib_rt_data)
warnings.simplefilter("default", RuntimeWarning)
for j in vol_names_sm:
for k in range(30, 330, 30): # changed here
if k < 100:
univ_ib_vol = cr_vol_all.retrieve(univ_ib_ext + j + "_0" + str(k) + "D")
else:
univ_ib_vol = cr_vol_all.retrieve(univ_ib_ext + j + "_" + str(k) + "D")
for i in univ_ib_vol.tick_cols():
univ_ib_vol[i] = filt.chg(filt.lag(1 / univ_ib_vol[i].values))
univ_ib_vol_data = univ_ib_vol[univ_ib_vol.tick_cols()].values.reshape(-1)
# univ_ib_nn = ~np.isnan(univ_ib_cl_data) & ~np.isnan(univ_ib_vol_data)
univ_ib_nn = ~np.isnan(univ_ib_cl_data) & ~np.isnan(univ_ib_vol_data) # & (univ_ib_rt_data < 0)
univ_ib_vol_data = univ_ib_vol_data[univ_ib_nn]
univ_ib_cl_data_ = univ_ib_cl_data[univ_ib_nn]
rho = ss.spearmanr(univ_ib_vol_data, univ_ib_cl_data_).correlation
if k < 100:
print(j + "_0" + str(k) + "D:\t", rho)
else:
print(j + "_" + str(k) + "D:\t", rho)
print("\n")
def check_pos_vs_neg_carry():
univ_ib_cl = cr_cret.retrieve(univ_ib_ext + "Close")[["Date", "SP500"]]
univ_ib_vol1 = cr_vol_all.retrieve(univ_ib_ext + "vol_rs_zlf_zdt_300D")[["Date", "SP500"]]
univ_ib_vol2 = cr_vol_all.retrieve(univ_ib_ext + "vol_cc_zlt_zdt_300D")[["Date", "SP500"]]
univ_ib_cl["Return"] = filt.ret(univ_ib_cl["SP500"].values)
univ_ib_cl["Lev1"] = 0.15 / univ_ib_vol1["SP500"].values
univ_ib_cl["Lev2"] = 0.15 / univ_ib_vol2["SP500"].values
univ_ib_cl["DyRet1"] = univ_ib_cl["Return"] * univ_ib_cl["Lev1"]
univ_ib_cl["DyRet2"] = univ_ib_cl["Return"] * univ_ib_cl["Lev2"]
t1 = univ_ib_cl["DyRet1"].values
t2 = univ_ib_cl["DyRet2"].values
tnan = ~np.isnan(t1) & ~np.isnan(t2)
t1 = t1[tnan]
t2 = t2[tnan]
print(16 * np.mean(t1) / np.std(t1))
print(16 * np.mean(t2) / np.std(t2))
def spline_curve(x1, _ticker='SP500', f=None, x1_=None, x2_=None, scat=True):
univ_ib_gd = cr_cret.retrieve(univ_ib_eqidx_ext + 'ExchOpen')[_ticker].values
univ_ib_vl = cr_vol_all_adj.retrieve(univ_ib_eqidx_ext + 'vol_pb_120')[_ticker].values
univ_ib_s1 = cr_sig_mr_sg.retrieve(univ_ib_eqidx_ext + x1)[_ticker].values
univ_ib_cl = cr_cret.retrieve(univ_ib_eqidx_ext + 'Close')[_ticker].values
z = list(np.where(univ_ib_gd.astype('int') == 1)[0])
univ_ib_s1 = univ_ib_s1[z]
univ_ib_cl = univ_ib_cl[z]
univ_ib_vl = univ_ib_vl[z]
univ_ib_cl = filt.ret(univ_ib_cl) / filt.lag(univ_ib_vl, 1)
univ_ib_s1 = filt.lag(univ_ib_s1, 1) / filt.lag(univ_ib_vl, 1)
univ_ib_cl, univ_ib_s1 = reduce_nonnan(univ_ib_cl, univ_ib_s1)
print(spearmanr(univ_ib_cl, univ_ib_s1).correlation)
test_weight = np.ones(univ_ib_cl.shape[0]) / univ_ib_cl.shape[0]
if x1_ is None:
x1 = np.percentile(univ_ib_s1, 25)
else:
x1 = x1_
if x2_ is None:
x2 = np.percentile(univ_ib_s1, 75)
else:
x2 = x2_
testa = rpf2.cubic_fit_linreg(univ_ib_cl, univ_ib_s1, x1, x2, test_weight)
x_ = np.linspace(-1, 1, num=101)
b0 = testa[0]
b1 = testa[1]
b2 = testa[2]
b3 = testa[3]
a0 = testa[4]
c0 = testa[5]
y_ = (b0+a0*(x_<x1)+c0*(x_>x2))+(b1-3*(a0/x_)*(x_<x1)-3*(c0/x_)*(x_>x2))*x_+\
(b2+3*(a0/(x_**2))*(x_<x1)+3*(c0/(x_**2))*(x_>x2))*(x_**2)+\
(b3-(a0/(x_**3))*(x_<x1)-(c0/(x_**3))*(x_>x2))*(x_**3)
y_[np.where(np.isnan(y_))[0]] = b0
z_ = np.zeros(len(y_))
pyl.plot(x_, y_)
pyl.plot(x_, z_)
if scat:
pyl.scatter(univ_ib_s1, univ_ib_cl, c='c')
return None
def test_measures():
univ_ib_gd = cr_cret.retrieve(univ_ib_eqidx_ext + 'ExchOpen')
_prc_names = ['CLS', 'A2D', 'A3D', 'A4D', 'A2W', 'A3W', 'A4W', 'A2M', 'A3M', 'A4M']
_trn_names = ['LVL', 'CH0', 'CH1', 'CH2']
_fil_names = ['LRB', 'QRB', 'QRG']
_hoz_names = []
for j in range(5, 305, 5):
if j < 10:
_hoz_names.append('00'+str(j))
elif j < 100:
_hoz_names.append('0'+str(j))
else:
_hoz_names.append(str(j))
_ticker = 'SP500'
z = list(np.where(univ_ib_gd[_ticker].values.astype('int') ==1)[0])
univ_ib_cl = cr_cret.retrieve(univ_ib_eqidx_ext + 'Close')[_ticker].values[z]
univ_ib_cl = filt.ret(univ_ib_cl)
univ_ib_cl = univ_ib_cl[-2780:]
# _len = 99999
for i in _prc_names:
# i = _prc_names[0]
for j in _fil_names:
# j = _fil_names[0]
for k in _trn_names:
# k = _trn_names[0]
for m in _hoz_names:
# m = _hoz_names[0]
fn = univ_ib_eqidx_ext+i+'_'+j+'_'+k+'_'+m
# print(fn)
univ_ib_sig = cr_sig_mr_sg.retrieve(fn)[_ticker].values[z]
univ_ib_sig1 = filt.lag(univ_ib_sig)
univ_ib_sig2 = filt.lag(univ_ib_sig, 2)
univ_ib_sig1 = univ_ib_sig1[-2780:]
univ_ib_sig2 = univ_ib_sig2[-2780:]
t1 = spearmanr(univ_ib_cl, univ_ib_sig1).correlation
t2 = spearmanr(univ_ib_cl, univ_ib_sig2).correlation
d1 = spearmanr(np.sign(univ_ib_cl), np.sign(univ_ib_sig1)).correlation
d2 = spearmanr(np.sign(univ_ib_cl), np.sign(univ_ib_sig2)).correlation
td = np.array([t1, t2, d1, d2])*100
print(i+','+j+','+k+','+m+' :\t', np_to_str(td))
return None
def refresh_correl(_window=240):
_cls = cr_cret.retrieve(univ_ib_eqidx_ext + 'Close')
_vol = cr_cret.retrieve(univ_ib_eqidx_ext + 'vol_gk240')
tcl = _cls.tick_cols()
for i in tcl:
_cls[i] = filt.ret(_cls[i].values)/_vol[i].values
_corr = dummy_df(_cls)
n = _corr.shape[0]
# need to make this code parallel
for j in range(0, len(tcl)-1):
for k in range(j+1, len(tcl)):
print('Processing rolling correlation of %s and %s' % (tcl[j], tcl[k]))
_corr[tcl[j]+'_'+tcl[k]] = np.empty(n)*np.nan
_tmp1 = _cls[tcl[k]].values
_tmp2 = _cls[tcl[j]].values
ny1 = np.maximum(filt.fst_nan(_tmp1), filt.fst_nan(_tmp2))
for _lbck in range(_window + ny1, n):
_corr[_lbck, tcl[j]+'_'+tcl[k]] = bst.kendall_tau(_tmp1[_lbck-_window:_lbck],
_tmp2[_lbck-_window:_lbck])
cr_cret.store(univ_ib_eqidx_ext+'Correl', _corr)
return None
def get_vol_adj():
univ_ib_cl = cr_cret.retrieve(univ_ib_eqidx_ext + 'Close')
univ_ib_gd = cr_cret.retrieve(univ_ib_eqidx_ext + 'ExchOpen')
tick_cols = univ_ib_cl.tick_cols()
for k in range(0, len(vol_list)):
print('Processing volatility %s' % vol_names[k])
for j in vol_lookbacks:
print('Currently working on lookback %s' % str(j))
if j < 100:
univ_ib_vol = cr_vol_all.retrieve(univ_ib_eqidx_ext + vol_names[k] + '_0' + str(j))
else:
univ_ib_vol = cr_vol_all.retrieve(univ_ib_eqidx_ext + vol_names[k] + '_' + str(j))
for i in tick_cols:
z = np.where(univ_ib_gd[i].values.astype('int') == 1)[0]
if z.shape[0] > 0:
univ_ib_vol_ = univ_ib_vol[i].values[z]
univ_ib_cl_ = univ_ib_cl[i].values[z]
# find the beta
univ_ib_vol__ = filt.lag(univ_ib_vol_)
univ_ib_cl_ = np.abs(filt.ret(univ_ib_cl_))
regp = qreg4.roll_e_ladreg_1d(univ_ib_cl_, univ_ib_vol__, adj_lookback)
# get the beta-adjusted volatility
vol__ = univ_ib_vol_ * regp * _lapl_mult
# smooth out the volatility
vol__1 = filt.lrma(vol__, 61, lg=True)
vol__2 = filt.lrma(vol__, 7, lg=True)
vol__3 = vol__1 + filt.lrma(vol__2 - vol__1, 16)
# push the new volatility back
univ_ib_vol[i] = np.nan
univ_ib_vol[list(z), i] = vol__3
univ_ib_vol[i] = filt.fill(univ_ib_vol[i].values)
else:
univ_ib_vol[i] = np.nan
if j < 100:
cr_vol_all_adj.store(univ_ib_eqidx_ext + vol_names[k] + '_0' + str(j), univ_ib_vol)
else:
cr_vol_all_adj.store(univ_ib_eqidx_ext + vol_names[k] + '_' + str(j), univ_ib_vol)
return None
testv1 = mkt_retrieve(i, 'Stats', 'Volatility')
for vv in ['vol_gk240']:
# vv = 'vol_gk240'
# lag the volatility
testv = filt.lag(testv1[vv].values)
test1 = mkt_retrieve(i, 'Stats', 'Returns')
# lag the signal
test2 = mkt_retrieve(i, 'MovReg', 'Signals')
tcl2 = test2.tick_cols()
for k in tcl2:
test2[k] = filt.lag(test2[k].values)/testv
# get the average of returns
test1_ = test1[['Date', 'Close']]
test1_['Returns'] = filt.ret(test1_['Close'].values)
fret1 = test1_['Returns'].values / testv
num_col = len(tcl2)
correl_vec = np.zeros(num_col)
for z in range(0, num_col):
correl_vec[z] = bst.kendall_tau(fret1, test2[tcl2[z]].values)
_mom_sig = []
_mom_sig_idx = []
_mom_val = []
for z in range(0, 50):
# z = 0
if z == 0:
_idx = bst.argmax_fixed(correl_vec)
_mom_sig.append(tcl2[_idx])
def check_closest_volatility():
univ_ib_cl = cr_cret.retrieve(univ_ib_ext+'Close')
univ_ib_rt1 = univ_ib_cl.copy()
univ_ib_rt2 = univ_ib_cl.copy()
univ_ib_rt3 = univ_ib_cl.copy()
for i in univ_ib_cl.tick_cols():
# i = 'SP500'
univ_ib_cl[i] = filt.ret(univ_ib_cl[i].values)
univ_ib_rt1[i] = filt.ret(univ_ib_rt1[i].values, 30)
univ_ib_rt1[i] = filt.lag(univ_ib_rt1[i].values)
univ_ib_rt2[i] = filt.ret(univ_ib_rt2[i].values, 60)
univ_ib_rt2[i] = filt.lag(univ_ib_rt2[i].values)
univ_ib_rt3[i] = filt.ret(univ_ib_rt3[i].values, 120)
univ_ib_rt3[i] = filt.lag(univ_ib_rt3[i].values)
univ_ib_cl_data = univ_ib_cl[univ_ib_cl.tick_cols()].values.reshape(-1, )
univ_ib_rt1_data = univ_ib_rt1[univ_ib_rt1.tick_cols()].values.reshape(-1, )
univ_ib_rt2_data = univ_ib_rt2[univ_ib_rt2.tick_cols()].values.reshape(-1, )
univ_ib_rt3_data = univ_ib_rt3[univ_ib_rt3.tick_cols()].values.reshape(-1, )
univ_ib_cl_data = np.abs(univ_ib_cl_data)
import warnings
warnings.simplefilter('ignore', RuntimeWarning)
univ_ib_rt1_data = np.sign(univ_ib_rt1_data)
univ_ib_rt2_data = np.sign(univ_ib_rt2_data)
univ_ib_rt3_data = np.sign(univ_ib_rt3_data)
warnings.simplefilter('default', RuntimeWarning)
# vol_names_ = [i for i in vol_names if 'reg' not in i]
for j in vol_names:
for k in range(_min_range, _max_range+_step_range, _step_range):
if k < 100:
univ_ib_vol = cr_vol_all.retrieve(univ_ib_ext + j + '_0' + str(k))
else:
univ_ib_vol = cr_vol_all.retrieve(univ_ib_ext + j + '_' + str(k))
for i in univ_ib_vol.tick_cols():
univ_ib_vol[i] = filt.lag(univ_ib_vol[i].values)
univ_ib_vol_data = univ_ib_vol[univ_ib_vol.tick_cols()].values.reshape(-1, )
univ_ib_nn = ~np.isnan(univ_ib_cl_data) & ~np.isnan(univ_ib_vol_data)
univ_ib_vol_data_ = univ_ib_vol_data[univ_ib_nn]*_norm_mult
univ_ib_cl_data_ = univ_ib_cl_data[univ_ib_nn]
res0 = simple_quant_reg(univ_ib_cl_data_, univ_ib_vol_data_)
# res0 = QuantReg(univ_ib_cl_data_, univ_ib_vol_data_).fit(q=0.5).params
univ_ib_nn = ~np.isnan(univ_ib_cl_data) & ~np.isnan(univ_ib_vol_data) & (univ_ib_rt1_data < 0)
univ_ib_vol_data_ = univ_ib_vol_data[univ_ib_nn]*_norm_mult
univ_ib_cl_data_ = univ_ib_cl_data[univ_ib_nn]
res1 = simple_quant_reg(univ_ib_cl_data_, univ_ib_vol_data_)
# res1 = QuantReg(univ_ib_cl_data_, univ_ib_vol_data_).fit(q=0.5).params
univ_ib_nn = ~np.isnan(univ_ib_cl_data) & ~np.isnan(univ_ib_vol_data) & (univ_ib_rt2_data < 0)
univ_ib_vol_data_ = univ_ib_vol_data[univ_ib_nn]*_norm_mult
univ_ib_cl_data_ = univ_ib_cl_data[univ_ib_nn]
res2 = simple_quant_reg(univ_ib_cl_data_, univ_ib_vol_data_)
# res2 = QuantReg(univ_ib_cl_data_, univ_ib_vol_data_).fit(q=0.5).params
univ_ib_nn = ~np.isnan(univ_ib_cl_data) & ~np.isnan(univ_ib_vol_data) & (univ_ib_rt3_data < 0)
univ_ib_vol_data_ = univ_ib_vol_data[univ_ib_nn]*_norm_mult
univ_ib_cl_data_ = univ_ib_cl_data[univ_ib_nn]
# res3 = QuantReg(univ_ib_cl_data_, univ_ib_vol_data_).fit(q=0.5).params
res3 = simple_quant_reg(univ_ib_cl_data_, univ_ib_vol_data_)
univ_ib_nn = ~np.isnan(univ_ib_cl_data) & ~np.isnan(univ_ib_vol_data) & (univ_ib_rt1_data > 0)
univ_ib_vol_data_ = univ_ib_vol_data[univ_ib_nn] * _norm_mult
univ_ib_cl_data_ = univ_ib_cl_data[univ_ib_nn]
# res4 = QuantReg(univ_ib_cl_data_, univ_ib_vol_data_).fit(q=0.5).params
res4 = simple_quant_reg(univ_ib_cl_data_, univ_ib_vol_data_)
univ_ib_nn = ~np.isnan(univ_ib_cl_data) & ~np.isnan(univ_ib_vol_data) & (univ_ib_rt2_data > 0)
univ_ib_vol_data_ = univ_ib_vol_data[univ_ib_nn] * _norm_mult
univ_ib_cl_data_ = univ_ib_cl_data[univ_ib_nn]
# res5 = QuantReg(univ_ib_cl_data_, univ_ib_vol_data_).fit(q=0.5).params
res5 = simple_quant_reg(univ_ib_cl_data_, univ_ib_vol_data_)
univ_ib_nn = ~np.isnan(univ_ib_cl_data) & ~np.isnan(univ_ib_vol_data) & (univ_ib_rt3_data > 0)
univ_ib_vol_data_ = univ_ib_vol_data[univ_ib_nn] * _norm_mult
univ_ib_cl_data_ = univ_ib_cl_data[univ_ib_nn]
# res6 = QuantReg(univ_ib_cl_data_, univ_ib_vol_data_).fit(q=0.5).params
res6 = simple_quant_reg(univ_ib_cl_data_, univ_ib_vol_data_)
if k < 100:
print(j + '_0' + str(k)+'\t', np_to_str(res0), '\t', np_to_str(res1), '\t', np_to_str(res2), '\t', np_to_str(res3), '\t',
np_to_str(res4), '\t', np_to_str(res5), '\t', np_to_str(res6), '\t')
else:
print(j + '_' + str(k)+'\t', np_to_str(res0), '\t', np_to_str(res1), '\t', np_to_str(res2), '\t', np_to_str(res3), '\t',
np_to_str(res4), '\t', np_to_str(res5), '\t', np_to_str(res6), '\t')
print('\n')
def get_cs_factor_portfolios():
# just testing here
# read the closing prices, convert it to returns
_cls = cr_cret.retrieve(univ_ib_eqidx_ext + 'Close')
tcl = _cls.tick_cols()
for j in tcl:
_cls[j] = filt.ret(_cls[j].values)
# read the volatility, signal, pca beta, lag them
_vol = cr_cret.retrieve(univ_ib_eqidx_ext + 'vol_gk240')
_sig = cr_sret.retrieve(univ_ib_eqidx_ext + 'D10S26_521_QRB_LVL')
# _sig = cr_sret.retrieve(univ_ib_eqidx_ext + _all_signals_p1_55[2])
_pca1 = cr_cret.retrieve(univ_ib_eqidx_ext + 'PCA1_Beta')
_pca2 = cr_cret.retrieve(univ_ib_eqidx_ext + 'PCA2_Beta')
for j in tcl:
_sig[j] = filt.lag(_sig[j].values)
_pca1[j] = filt.lag(_pca1[j].values)
_pca2[j] = filt.lag(_pca2[j].values)
_vol[j] = filt.lag(_vol[j].values)
# find starting point, at least 20 markets are live (from both beta, sig)
nc = len(tcl)
ny = np.zeros(nc)
for j in range(0, nc):
ny[j] = np.maximum(filt.fst_nan(_sig[tcl[j]].values), filt.fst_nan(_pca1[tcl[j]].values))
_ny_mn = int(np.min(ny))
_ny_mx = int(np.max(ny))
_ny = _ny_mn
for j in range(_ny_mn, _ny_mx):
if np.where(ny >= j)[0].shape[0] >= 20:
_ny = j
break
# cross-sectionally normalize the signal
_sign1 = _sig.copy()
_sign2 = _sig.copy()
_sign3 = _sig.copy()
_sign4 = _sig.copy()
_sign5 = _sig.copy()
_sign6 = _sig.copy()
for j in tcl:
_sign1[j] = np.nan
_sign2[j] = np.nan
_sign3[j] = np.nan
_sign4[j] = np.nan
_sign5[j] = np.nan
_sign6[j] = np.nan
for j in range(_ny, _sig.shape[0]):
_tmp = _sig[j, tcl] / _vol[j, tcl]
_sign1[j, tcl] = half_norm_rankit(_tmp)
_sign2[j, tcl] = _sign1[j, tcl].values
_sign3[j, tcl] = _sign1[j, tcl].values
_sign4[j, tcl] = full_norm_rankit(_tmp)
_sign5[j, tcl] = _sign4[j, tcl].values
_sign6[j, tcl] = _sign4[j, tcl].values
# calculate returns using risk parity portfolio approaches
bk_test = dummy_df(_vol)
bk_test['H1'] = np.nan
bk_test['H2'] = np.nan
bk_test['H3'] = np.nan
bk_test['F1'] = np.nan
bk_test['F2'] = np.nan
bk_test['F3'] = np.nan
for j in range(_ny, _sig.shape[0]):
# j = _ny
_sign1_tmp = _sign1[j:j, tcl].values
_sign2_tmp = _sign2[j:j, tcl].values
_sign3_tmp = _sign3[j:j, tcl].values
_sign4_tmp = _sign4[j:j, tcl].values
_sign5_tmp = _sign5[j:j, tcl].values
_sign6_tmp = _sign6[j:j, tcl].values
_cls_tmp = _cls[j:j, tcl].values
_vol_tmp = _vol[j:j, tcl].values
_pca1_tmp = _pca1[j:j, tcl].values
_pca2_tmp = _pca2[j:j, tcl].values
_sign1_tmp, _sign2_tmp, _sign3_tmp, _sign4_tmp, _sign5_tmp, _sign6_tmp, _cls_tmp, _vol_tmp, _pca1_tmp,\
_pca2_tmp = reduce_nonnan(_sign1_tmp, _sign2_tmp, _sign3_tmp, _sign4_tmp, _sign5_tmp, _sign6_tmp,
_cls_tmp, _vol_tmp, _pca1_tmp, _pca2_tmp)
_tmp11 = np.dot(_pca1_tmp, _pca1_tmp)
_tmp22 = np.dot(_pca2_tmp, _pca2_tmp)
_tmps11 = np.dot(_sign1_tmp, _pca1_tmp)
_tmps12 = np.dot(_sign1_tmp, _pca2_tmp)
_tmps21 = np.dot(_sign4_tmp, _pca1_tmp)
_tmps22 = np.dot(_sign4_tmp, _pca2_tmp)
_sign2_tmp = _sign2_tmp - _pca1_tmp * _tmps11 / _tmp11
_sign5_tmp = _sign5_tmp - _pca1_tmp * _tmps21 / _tmp11
_sign3_tmp = _sign3_tmp - _pca1_tmp * _tmps11 / _tmp11 - _pca2_tmp * _tmps12 / _tmp22
_sign6_tmp = _sign6_tmp - _pca1_tmp * _tmps21 / _tmp11 - _pca2_tmp * _tmps22 / _tmp22
_sign1_tmp_sum = np.sum(np.abs(_sign1_tmp))
_sign2_tmp_sum = np.sum(np.abs(_sign2_tmp))
_sign3_tmp_sum = np.sum(np.abs(_sign3_tmp))
_sign4_tmp_sum = np.sum(np.abs(_sign4_tmp))
_sign5_tmp_sum = np.sum(np.abs(_sign5_tmp))
_sign6_tmp_sum = np.sum(np.abs(_sign6_tmp))
_sign1_tmp = _sign1_tmp / _sign1_tmp_sum
_sign2_tmp = _sign2_tmp / _sign2_tmp_sum
_sign3_tmp = _sign3_tmp / _sign3_tmp_sum
_sign4_tmp = _sign4_tmp / _sign4_tmp_sum
_sign5_tmp = _sign5_tmp / _sign5_tmp_sum
_sign6_tmp = _sign6_tmp / _sign6_tmp_sum
_lev_tmp = 0.005 / _vol_tmp
_ret_tmp = _cls_tmp * _lev_tmp
bk_test[j, 'H1'] = np.dot(_sign1_tmp, _ret_tmp)
bk_test[j, 'H2'] = np.dot(_sign2_tmp, _ret_tmp)
bk_test[j, 'H3'] = np.dot(_sign3_tmp, _ret_tmp)
bk_test[j, 'F1'] = np.dot(_sign4_tmp, _ret_tmp)
bk_test[j, 'F2'] = np.dot(_sign5_tmp, _ret_tmp)
bk_test[j, 'F3'] = np.dot(_sign6_tmp, _ret_tmp)
print('done')
_dt = bk_test['Date'].values[_ny:]
f1 = plt.figure(1)
for jidx, j in enumerate(['H1', 'H2', 'H3']):
_sh = np.nanmean(bk_test[j].values)*16/np.nanstd(bk_test[j].values)
if _sh > 0:
_mx = avg_drawdown(bk_test[_ny:, j].values) / (16 * np.std(bk_test[_ny:, j].values))
testh = conv_to_price(bk_test[j].values)
else:
_mx = avg_drawdown(-bk_test[_ny:, j].values) / (16 * np.std(bk_test[_ny:, j].values))
testh = conv_to_price(-bk_test[j].values)
_sh = -_sh
print([_sh, _mx])
plt.subplot(3, 1, jidx+1)
plot_ts_new(_dt, testh[_ny:])
f2 = plt.figure(2)
for jidx, j in enumerate(['F1', 'F2', 'F3']):
_sh = np.nanmean(bk_test[j].values) * 16 / np.nanstd(bk_test[j].values)
if _sh > 0:
_mx = avg_drawdown(bk_test[_ny:, j].values) / (16 * np.std(bk_test[_ny:, j].values))
testh = conv_to_price(bk_test[j].values)
else:
_mx = avg_drawdown(-bk_test[_ny:, j].values) / (16 * np.std(bk_test[_ny:, j].values))
testh = conv_to_price(-bk_test[j].values)
_sh = -_sh
print([_sh, _mx])
plt.subplot(3, 1, jidx + 1)
plot_ts_new(_dt, testh[_ny:])
_bk_test = bk_test[_ny:, bk_test.tick_cols()].values
print(np.corrcoef(_bk_test.T))
# bk_test3 = bk_test.copy()
f1.clear()
f2.clear()
plt.subplot(3, 1, 1)
tmp = bk_test1[:, 'F1'].values
plot_ts_new(_dt, conv_to_price(tmp)[_ny:])
print(np.nanmean(tmp)*16/np.nanstd(tmp))
plt.subplot(3, 1, 2)
tmp = bk_test2[:, 'F1'].values
plot_ts_new(_dt, conv_to_price(tmp)[_ny:])
print(np.nanmean(tmp)*16/np.nanstd(tmp))
plt.subplot(3, 1, 3)
tmp = 0.5 * -bk_test3[:, 'F1'].values + 0.5 * bk_test2[:, 'F2'].values
plot_ts_new(_dt, conv_to_price(tmp)[_ny:])
print(np.nanmean(tmp) * 16 / np.nanstd(tmp))
test2 = mkt_retrieve(i, 'MovReg', 'Signals_Pct_' + str(j))
tcl2 = test2.tick_cols()
for lmb in [100]:
# lmb = 0.01
print('Using lookback of %s with lambda of %s' % (str(j), str(lmb)))
for k in range(2, 3):
# k = 1
# lag the signal data
test2_ = test2.copy()
for tcl2_ in tcl2:
test2_[tcl2_] = filt.lag(test2_[tcl2_].values, k)
# get the average of returns
test1_ = test1[['Date', 'Close']]
test1_['Returns'] = filt.ret(test1_['Close'].values)
test1_['MultiReturns'] = filt.ret(test1_['Close'].values, k)/k
test1_['MultiReturns'] = filt.mpc(test1_['MultiReturns'].values, j)
test1_['ReturnPct'] = filt.mpc(test1_['Returns'].values, j)
# get beta
# beta1 = qreg5.roll_s_ladreg_2d_l2n(test1_['MultiReturns'].values,
# np.ascontiguousarray(test2_[test2_.tick_cols()].values),
# j, lmb, 30)
beta2 = qreg5.roll_w_ladreg_2d_l2n(test1_['MultiReturns'].values,
np.ascontiguousarray(test2_[test2_.tick_cols()].values),
j, lmb, 30)
# beta1_ = test2.copy()
beta2_ = test2.copy()
test2_ = mkt_retrieve(i, 'MovReg', 'Changes7')[['Date']+_sig_set_95_ch7]
test2_.set_columns(['Date']+[k + '_CH7' for k in _sig_set_95_ch7])
test2 = DataFrame.merge(test2, test2_, on='Date')
test2_ = mkt_retrieve(i, 'MovReg', 'Changes9')[['Date']+_sig_set_95_ch9]
test2_.set_columns(['Date']+[k + '_CH9' for k in _sig_set_95_ch9])
test2 = DataFrame.merge(test2, test2_, on='Date')
tcl2 = test2.tick_cols()
for k in tcl2:
test2[k] = filt.lag(test2[k].values)/testv
# get the returns
test1 = mkt_retrieve(i, 'Stats', 'Returns')[['Date', 'Close']]
test1['Returns'] = filt.ret(test1['Close'].values)/testv
del test1['Close']
tcl2 = test2.tick_cols()
ny = filt.fst_nan(test1['Returns'].values)
for k in tcl2:
ny = np.maximum(ny, filt.fst_nan(test2[k].values))
test1 = test1[ny:, :]
test2 = test2[ny:, :]
if i == _eq_idx[0]:
test1_all = test1.copy()
test2_all = test2.copy()
else:
for j in range(i, e2):
x_.append((i, j))
return x_
n_range = get_permut(5, 50, 50)
f_range = get_permut(5, 70, 70)
results = {}
for n in n_range:
for f in f_range:
for oshpr in range(4, 40, 1):
_loss = n[0]+n[1]+f[0]+f[1]+oshpr
if _loss <= 250:
test1p = filt.smth_price(test3p_, test2p_, n[0], n[1], f[0], f[1], 0, oshpr)
test1p, test2p, test3p = reduce_nonnan(test1p, test2p_, test3p_)
g1 = autocorr(filt.ret(test1p)[-7000:])
k = int((1+g1)/(1-g1))
if k > 5:
xoc_mx = test1p - np.maximum(test2p, test3p)
xoc_mn = test1p - np.minimum(test2p, test3p)
good2_ = np.abs(200 * ((xoc_mx > 0) * xoc_mx + (xoc_mn < 0) * xoc_mn) / (test2p + test3p))
good2 = np.mean(good2_[-7000:]) * 2 / (k + 1) + np.median(good2_[-7000:]) * (k - 1) / (k + 1)
good2 *= np.sqrt(_loss)
if k in results.keys():
tmp = results[k]
if tmp[5] > good2:
results[k] = [n[0], n[1], f[0], f[1], oshpr, good2]
print([k]+results[k])
else:
results[k] = [n[0], n[1], f[0], f[1], oshpr, good2]
print([k] + results[k])
def check_closest_volatility():
univ_ib_cl = cr_cret.retrieve(univ_ib_ext + "Close")
univ_ib_rt1 = univ_ib_cl.copy()
univ_ib_rt2 = univ_ib_cl.copy()
univ_ib_rt3 = univ_ib_cl.copy()
for i in univ_ib_cl.tick_cols():
univ_ib_cl[i] = filt.ret(univ_ib_cl[i].values)
univ_ib_rt1[i] = filt.ret(univ_ib_rt1[i].values, 30)
univ_ib_rt1[i] = filt.lag(univ_ib_rt1[i].values)
univ_ib_rt2[i] = filt.ret(univ_ib_rt2[i].values, 60)
univ_ib_rt2[i] = filt.lag(univ_ib_rt2[i].values)
univ_ib_rt3[i] = filt.ret(univ_ib_rt3[i].values, 120)
univ_ib_rt3[i] = filt.lag(univ_ib_rt3[i].values)
univ_ib_cl_data = univ_ib_cl[univ_ib_cl.tick_cols()].values.reshape(-1)
univ_ib_rt1_data = univ_ib_rt1[univ_ib_rt1.tick_cols()].values.reshape(-1)
univ_ib_rt2_data = univ_ib_rt2[univ_ib_rt2.tick_cols()].values.reshape(-1)
univ_ib_rt3_data = univ_ib_rt3[univ_ib_rt3.tick_cols()].values.reshape(-1)
univ_ib_cl_data = np.abs(univ_ib_cl_data)
import warnings
warnings.simplefilter("ignore", RuntimeWarning)
univ_ib_rt1_data = np.sign(univ_ib_rt1_data)
univ_ib_rt2_data = np.sign(univ_ib_rt2_data)
univ_ib_rt3_data = np.sign(univ_ib_rt3_data)
warnings.simplefilter("default", RuntimeWarning)
# vol_names_ = [i for i in vol_names if 'reg' not in i]
for j in vol_names_sm:
for k in range(_min_range, _max_range, _step_range * 3):
if k < 100:
univ_ib_vol = cr_vol_all.retrieve(univ_ib_ext + j + "_0" + str(k) + "D")
else:
univ_ib_vol = cr_vol_all.retrieve(univ_ib_ext + j + "_" + str(k) + "D")
for i in univ_ib_vol.tick_cols():
univ_ib_vol[i] = filt.lag(univ_ib_vol[i].values)
univ_ib_vol_data = univ_ib_vol[univ_ib_vol.tick_cols()].values.reshape(-1)
univ_ib_nn = ~np.isnan(univ_ib_cl_data) & ~np.isnan(univ_ib_vol_data)
univ_ib_vol_data_ = univ_ib_vol_data[univ_ib_nn] * _norm_mult
univ_ib_cl_data_ = univ_ib_cl_data[univ_ib_nn]
res0 = QuantReg(univ_ib_cl_data_, univ_ib_vol_data_).fit(q=0.5).params
univ_ib_nn = ~np.isnan(univ_ib_cl_data) & ~np.isnan(univ_ib_vol_data) & (univ_ib_rt1_data < 0)
univ_ib_vol_data_ = univ_ib_vol_data[univ_ib_nn] * _norm_mult
univ_ib_cl_data_ = univ_ib_cl_data[univ_ib_nn]
res1 = QuantReg(univ_ib_cl_data_, univ_ib_vol_data_).fit(q=0.5).params
univ_ib_nn = ~np.isnan(univ_ib_cl_data) & ~np.isnan(univ_ib_vol_data) & (univ_ib_rt2_data < 0)
univ_ib_vol_data_ = univ_ib_vol_data[univ_ib_nn] * _norm_mult
univ_ib_cl_data_ = univ_ib_cl_data[univ_ib_nn]
res2 = QuantReg(univ_ib_cl_data_, univ_ib_vol_data_).fit(q=0.5).params
univ_ib_nn = ~np.isnan(univ_ib_cl_data) & ~np.isnan(univ_ib_vol_data) & (univ_ib_rt3_data < 0)
univ_ib_vol_data_ = univ_ib_vol_data[univ_ib_nn] * _norm_mult
univ_ib_cl_data_ = univ_ib_cl_data[univ_ib_nn]
res3 = QuantReg(univ_ib_cl_data_, univ_ib_vol_data_).fit(q=0.5).params
univ_ib_nn = ~np.isnan(univ_ib_cl_data) & ~np.isnan(univ_ib_vol_data) & (univ_ib_rt1_data > 0)
univ_ib_vol_data_ = univ_ib_vol_data[univ_ib_nn] * _norm_mult
univ_ib_cl_data_ = univ_ib_cl_data[univ_ib_nn]
res4 = QuantReg(univ_ib_cl_data_, univ_ib_vol_data_).fit(q=0.5).params
univ_ib_nn = ~np.isnan(univ_ib_cl_data) & ~np.isnan(univ_ib_vol_data) & (univ_ib_rt2_data > 0)
univ_ib_vol_data_ = univ_ib_vol_data[univ_ib_nn] * _norm_mult
univ_ib_cl_data_ = univ_ib_cl_data[univ_ib_nn]
res5 = QuantReg(univ_ib_cl_data_, univ_ib_vol_data_).fit(q=0.5).params
univ_ib_nn = ~np.isnan(univ_ib_cl_data) & ~np.isnan(univ_ib_vol_data) & (univ_ib_rt3_data > 0)
univ_ib_vol_data_ = univ_ib_vol_data[univ_ib_nn] * _norm_mult
univ_ib_cl_data_ = univ_ib_cl_data[univ_ib_nn]
res6 = QuantReg(univ_ib_cl_data_, univ_ib_vol_data_).fit(q=0.5).params
if k < 100:
print(
j + "_0" + str(k) + "\t",
np_to_str(res0),
"\t",
np_to_str(res1),
"\t",
np_to_str(res2),
"\t",
np_to_str(res3),
"\t",
np_to_str(res4),
"\t",
np_to_str(res5),
"\t",
np_to_str(res6),
"\t",
)
else:
print(
j + "_" + str(k) + "\t",
np_to_str(res0),
"\t",
np_to_str(res1),
"\t",
np_to_str(res2),
"\t",
np_to_str(res3),
"\t",
np_to_str(res4),
"\t",
np_to_str(res5),
"\t",
np_to_str(res6),
"\t",
)
print("\n")
# f1 = 200
for f2 in range(6, 36, 2): # range(2, 36, 2):
# f2 = 28
for lthresh in range(0, 2, 1): #[1]:
# lthresh = 1
for oshpr in range(13, 16, 3):
# oshpr = 28
if (np.isnan(good2b) | (~np.isnan(good2b) & (good2b > 0.0001))) & (n1+n2+f1+f2 < 180):
# print(n1, n2, f1, f2, lthresh, oshpr)
test1p = spec_smth(univ_ib_op[_ticker].values, univ_ib_cl[_ticker].values, n1, n2, f1, f2, lthresh, oshpr)
test2p = univ_ib_cl[_ticker].values
test3p = univ_ib_op[_ticker].values
testkp = np.copy(testk)
test1p, test2p, test3p, testkp = reduce_nonnan(test1p, test2p, test3p, testkp)
test1r = filt.ret(test1p)
testkr = filt.ret(testkp)
test1r, testkr = reduce_nonnan(test1r, testkr)
# good1 = autocorr(test1r)-autocorr(test2r)
good1 = autocorr(test1r[-7000:])
good1k = autocorr(testkr[-7000:])
# get the overshoot measure
xoc_mx = test1p-np.maximum(test2p, test3p)
xoc_mn = test1p-np.minimum(test2p, test3p)
good2_ = np.abs(200*((xoc_mx>0)*xoc_mx+(xoc_mn<0)*xoc_mn)/(test2p+test3p))
# good2 = np.mean(good2_[-7000:])
good2 = np.median(good2_[-7000:])
xoc_mxk = testkp-np.maximum(test2p, test3p)
def test_measures2():
pd.set_option('display.max_columns', 30)
pd.set_option('display.max_rows', 100)
univ_ib_gd = cr_cret.retrieve(univ_ib_eqidx_ext + 'ExchOpen')
_prc_names = ['CLS', 'A2D', 'A3D', 'A4D', 'A2W', 'A3W', 'A4W', 'A2M', 'A3M', 'A4M']
_trn_names = ['LVL', 'CH0', 'CH1', 'CH2']
_fil_names = ['LRB', 'QRB', 'QRG']
_hoz_names = []
for j in range(5, 305, 5):
if j < 10:
_hoz_names.append('00'+str(j))
elif j < 100:
_hoz_names.append('0'+str(j))
else:
_hoz_names.append(str(j))
univ_ib_cl = cr_cret.retrieve(univ_ib_eqidx_ext + 'Close')
tick_cols = univ_ib_cl.tick_cols()
for n in tick_cols:
z = list(np.where(univ_ib_gd[n].values.astype('int') == 1)[0])
univ_ib_cl_ = filt.ret(univ_ib_cl[n].values[z])
univ_ib_cl[n] = np.nan
univ_ib_cl[z, n] = univ_ib_cl_
univ_ib_cl[n] = filt.fill1(univ_ib_cl[n].values, 0)
# tick_cols_ = tick_cols[0:1]
tick_cols = ['SP500', 'SP400Mid', 'Nikkei225', 'R2000', 'FTSE100', 'DAX',
'SMI', 'CAC40', 'AEX', 'MIBFTSE', 'IBEX35', 'OMXS30']
siz_ = [5330, 7115, 6757, 7349, 5324, 6785, 6794, 6199, 7232, 7936, 7180, 7295]
# siz = [2784, 999, 1357, 765, 2790, 1329, 1320, 1915, 882, 178, 934, 819]
tick_cols = tick_cols[1:]
siz_ = siz_[1:]
for k in _trn_names:
dummy_cols = []
for i in _prc_names:
for j in _fil_names:
dummy_cols.append(i+'_'+j)
x_ = np.random.random(len(_hoz_names))*np.nan
y_ = np.random.random(len(_hoz_names))*np.nan
dummy_df = DataFrame({dummy_cols[0]: x_, dummy_cols[1]: y_})
for i in range(2, 30):
dummy_df[dummy_cols[i]] = x_
_col = 0
for i in _prc_names:
# j = _fil_names[0]
for j in _fil_names:
# k = _trn_names[0]
for mi, m in enumerate(_hoz_names):
# m = _hoz_names[0]
fn = univ_ib_eqidx_ext+i+'_'+j+'_'+k+'_'+m
univ_ib_sig = cr_sig_mr_sg.retrieve(fn)
ret_, sig_ = np.empty(0), np.empty(0)
for ni, n in enumerate(tick_cols):
# ni = 0
# n = tick_cols[ni]
z = list(np.where(univ_ib_gd[n].values.astype('int') == 1)[0])
z = [z_ for z_ in z if z_ > siz_[ni]] # 5 is just a buffer
sig__ = univ_ib_sig[n].values[z]
ret__ = univ_ib_cl[n].values[z]
sig__ = filt.lag(sig__, 2)
ret_ = np.hstack((ret_, ret__[2:]))
sig_ = np.hstack((sig_, sig__[2:]))
gidx = np.where(~np.isnan(ret_) & ~np.isnan(sig_))[0]
dummy_df[mi, dummy_cols[_col]] = 100*spearmanr(ret_[gidx], sig_[gidx]).correlation
_col += 1
dummy_df.show_all()
univ_ib_vol6 = cr_vol_all.retrieve(univ_ib_ext+'vol_gk_'+lookback)[['Date', ticker]]
univ_ib_vol7 = cr_vol_all.retrieve(univ_ib_ext+'vol_yz_'+lookback)[['Date', ticker]]
z = list(np.where(univ_ib_gd[ticker].values.astype('int') == 1)[0])
univ_ib_cl = univ_ib_cl[z, :]
univ_ib_vol0 = univ_ib_vol0[z, :]
univ_ib_vol1 = univ_ib_vol1[z, :]
univ_ib_vol2 = univ_ib_vol2[z, :]
univ_ib_vol3 = univ_ib_vol3[z, :]
univ_ib_vol4 = univ_ib_vol4[z, :]
univ_ib_vol5 = univ_ib_vol5[z, :]
univ_ib_vol6 = univ_ib_vol6[z, :]
univ_ib_vol7 = univ_ib_vol7[z, :]
univ_ib_cl[ticker] = np.abs(filt.ret(univ_ib_cl[ticker].values))
univ_ib_vol0[ticker] = filt.lag(univ_ib_vol0[ticker].values)
univ_ib_vol1[ticker] = filt.lag(univ_ib_vol1[ticker].values)
univ_ib_vol2[ticker] = filt.lag(univ_ib_vol2[ticker].values)
univ_ib_vol3[ticker] = filt.lag(univ_ib_vol3[ticker].values)
univ_ib_vol4[ticker] = filt.lag(univ_ib_vol4[ticker].values)
univ_ib_vol5[ticker] = filt.lag(univ_ib_vol5[ticker].values)
univ_ib_vol6[ticker] = filt.lag(univ_ib_vol6[ticker].values)
univ_ib_vol7[ticker] = filt.lag(univ_ib_vol7[ticker].values)
univ_ib_dt_ = univ_ib_cl['Date'].values
univ_ib_cl_ = univ_ib_cl[ticker].values
univ_ib_vol0_ = univ_ib_vol0[ticker].values
univ_ib_vol1_ = univ_ib_vol1[ticker].values
univ_ib_vol2_ = univ_ib_vol2[ticker].values
univ_ib_vol3_ = univ_ib_vol3[ticker].values
def check_price_forecast():
univ_ib_dt = cr_cret.retrieve(univ_ib_eqidx_ext + 'ExchOpen')['Date'].values
univ_ib_gd = cr_cret.retrieve(univ_ib_eqidx_ext + 'ExchOpen')['SP500'].values
z = np.where(univ_ib_gd.astype('int') == 1)[0]
univ_ib_cl = cr_cret.retrieve(univ_ib_eqidx_ext + 'Close')['SP500'].values[z]
univ_ib_vl = cr_vol_all_adj.retrieve(univ_ib_eqidx_ext + 'vol_pb_120')['SP500'].values[z]
univ_ib_dt = univ_ib_dt[z]
nlag = 2
n1 = filt.fst_nan(univ_ib_cl)
n1 = np.maximum(n1, filt.fst_nan(univ_ib_vl))
univ_ib_sig = np.empty(0)
univ_ib_sig_ = np.empty(0)
for days in range(1, 11):
for smth in range(5, 33):
if days < 10:
if smth < 10:
tmp = cr_aret.retrieve(univ_ib_eqidx_ext + 'AdjSerD0' + str(days)+'S0' + str(smth))['SP500'].values[z]
else:
tmp = cr_aret.retrieve(univ_ib_eqidx_ext + 'AdjSerD0' + str(days)+'S' + str(smth))['SP500'].values[z]
else:
if smth < 10:
tmp = cr_aret.retrieve(univ_ib_eqidx_ext + 'AdjSerD' + str(days)+'S0' + str(smth))['SP500'].values[z]
else:
tmp = cr_aret.retrieve(univ_ib_eqidx_ext + 'AdjSerD' + str(days)+'S' + str(smth))['SP500'].values[z]
# get the change in the signal
# tmp = np.sign(filt.chg(tmp))
# lag the data
# tmp = filt.lag(tmp, nlag)
if univ_ib_sig.shape[0] == 0:
univ_ib_sig = tmp
else:
univ_ib_sig = np.vstack((univ_ib_sig, tmp))
n1 = np.maximum(n1, filt.fst_nan(tmp))
univ_ib_cl_ = np.sign(filt.chg(univ_ib_cl, nlag))
univ_ib_cl_ = univ_ib_cl_[n1:]
univ_ib_vl_ = univ_ib_vl[n1:]
univ_ib_sig = univ_ib_sig[:, n1:]
names = []
for days in range(1, 11):
for smth in range(5, 33):
if (days < 10) and (smth < 10):
names.append('0'+str(days)+'_0'+str(smth))
elif (days < 10) and (smth >= 10):
names.append('0'+str(days)+'_'+str(smth))
elif (days >= 10) and (smth < 10):
names.append(str(days)+'_0'+str(smth))
else:
names.append(str(days)+'_'+str(smth))
for i in range(0, len(names)):
print(names[i], ':', smart_kendall(univ_ib_cl_, univ_ib_sig[i, :]))
plot_ts_new(univ_ib_dt, univ_ib_cl)
plot_ts_new(univ_ib_dt, univ_ib_sig[167, :])
t1 = filt.ret(filt.lag(univ_ib_sig[167, :]))
t1_ = filt.ret(univ_ib_sig[167, :])
t2 = filt.ret(univ_ib_cl)
t3 = t2*np.sign(t1)
t1_, t2, t3 = reduce_nonnan(t1_, t2, t3)
ct1_ = np.cumprod(1+t1_)*100
ct2 = np.cumprod(1+t2)*100
ct3 = np.cumprod(1-t3)*100
f = pyl.figure(1)
f.clear()
pyl.subplot(3, 1, 1)
pyl.semilogy(ct2[-5000:])
pyl.subplot(3, 1, 2)
pyl.semilogy(ct1_[-5000:])
pyl.subplot(3, 1, 3)
pyl.semilogy(ct3[-5000:])
lookbacks = ['120', '240', '360'] np_nice_options(linelen=250, numpres=8) _norm_mult = np.sqrt(2/np.pi) _lapl_mult = 1/np.sqrt(2) _rand_mult = 0.60 tickers = ['SP500', 'DAX', 'Nikkei225', 'ESTX50', 'SMI', 'RDX', 'MSCIEM'] i = 'SP500' univ_ib_cl = mkt_retrieve(i, 'Stats', 'Returns')['Close'].values univ_ib_vl = mkt_retrieve(i, 'Stats', 'Volatility')['vol_pb240'].values univ_ib_cl = np.abs(filt.ret(univ_ib_cl)) univ_ib_vl = filt.lag(univ_ib_vl) univ_ib_cl, univ_ib_vl = reduce_nonnan(univ_ib_cl, univ_ib_vl ) univ_ib_vl *= _lapl_mult univ_ib_vl2 = np.sqrt(univ_ib_vl) univ_ib_vl2 = univ_ib_vl2 * med_abs_dev(univ_ib_vl)/med_abs_dev(univ_ib_vl2) b41 = qreg.roll_e_ladreg_1d(univ_ib_cl, univ_ib_vl, 240) b51 = qreg.roll_e_ladreg_2d(univ_ib_cl, mcc(univ_ib_vl, univ_ib_vl2), 240) resid0 = univ_ib_cl - univ_ib_vl resid1 = univ_ib_cl - univ_ib_vl * filt.lag(b41) resid2 = univ_ib_cl - univ_ib_vl * filt.lag(np.ascontiguousarray(b51[:, 0])) - univ_ib_vl2 * filt.lag(np.ascontiguousarray(b51[:, 1]))
