def gradient_boosting_result(_ticker='SP500'):
univ_ib_gd = cr_cret.retrieve(univ_ib_eqidx_ext + 'ExchOpen')[_ticker].values
z = list(np.where(univ_ib_gd.astype('int') == 1)[0])
_prc_names = ['CLS', 'A2D', 'A3D', 'A4D', 'A2W', 'A3W', 'A4W', 'A2M', 'A3M', 'A4M']
_trn_names = ['LVL', 'CH0', 'CH1', 'CH2']
_fil_names = ['LRB', 'QRB', 'QRG']
_hoz_names = []
for j in range(5, 305, 5):
if j < 10:
_hoz_names.append('00' + str(j))
elif j < 100:
_hoz_names.append('0' + str(j))
else:
_hoz_names.append(str(j))
# get volatility forecast
univ_ib_vl = cr_vol_all_adj.retrieve(univ_ib_eqidx_ext + 'vol_pb_120')[_ticker].values[z]
# get return
univ_ib_cl = cr_cret.retrieve(univ_ib_eqidx_ext + 'Close')[_ticker].values[z]
univ_ib_cl = filt.ret(univ_ib_cl)
univ_ib_sig_all = None
for k in _trn_names:
for i in _prc_names:
for j in _fil_names:
for mi, m in enumerate(_hoz_names):
fn = univ_ib_eqidx_ext+i+'_'+j+'_'+k+'_'+m
univ_ib_sig = cr_sig_mr_sg.retrieve(fn)[_ticker].values[z]
if univ_ib_sig_all is None:
univ_ib_sig_all = DataFrame({i+'_'+j+'_'+k+'_'+m: univ_ib_sig})
else:
univ_ib_sig_all.col_bind(DataFrame({i+'_'+j+'_'+k+'_'+m: univ_ib_sig}))
# just for storage - incase something happens
# univ_ib_gb = DataFrame({'Close': univ_ib_cl, 'Volatility': univ_ib_vl})
# univ_ib_gb.col_bind(univ_ib_sig_all)
# cr_cret.store(univ_ib_eqidx_ext + 'GBM', univ_ib_gb)
reg_lookback = [120, 240, 360, 480]
new_col_names = list(univ_ib_sig_all.columns)
for i in new_col_names:
# i = new_col_names[0]
test_ = np.empty(0)
for j in reg_lookback:
# j = reg_lookback[0]
uic = univ_ib_cl
uiv = univ_ib_vl
uis = filt.lag(univ_ib_sig_all[i].values, 1)
uic, uis, uiv = reduce_nonnan(uic, uis, uiv)
b1 = qreg.roll_e_ladreg_1d(uic, uis, j)
b2 = qreg.roll_e_ladreg_1d(uic/uiv, uis/uiv, j)
resid1 = uic - filt.lag(b1)*uis
resid2 = uic - filt.lag(b2)*uis
resid1, resid2 = reduce_nonnan(resid1, resid2)
test_ = np.hstack((test_, np.array([np.median(abs(resid1)), np.median(abs(resid2))])))
print(i+' : '+np_to_str(test_))
def test_measures():
univ_ib_gd = cr_cret.retrieve(univ_ib_eqidx_ext + 'ExchOpen')
_prc_names = ['CLS', 'A2D', 'A3D', 'A4D', 'A2W', 'A3W', 'A4W', 'A2M', 'A3M', 'A4M']
_trn_names = ['LVL', 'CH0', 'CH1', 'CH2']
_fil_names = ['LRB', 'QRB', 'QRG']
_hoz_names = []
for j in range(5, 305, 5):
if j < 10:
_hoz_names.append('00'+str(j))
elif j < 100:
_hoz_names.append('0'+str(j))
else:
_hoz_names.append(str(j))
_ticker = 'SP500'
z = list(np.where(univ_ib_gd[_ticker].values.astype('int') ==1)[0])
univ_ib_cl = cr_cret.retrieve(univ_ib_eqidx_ext + 'Close')[_ticker].values[z]
univ_ib_cl = filt.ret(univ_ib_cl)
univ_ib_cl = univ_ib_cl[-2780:]
# _len = 99999
for i in _prc_names:
# i = _prc_names[0]
for j in _fil_names:
# j = _fil_names[0]
for k in _trn_names:
# k = _trn_names[0]
for m in _hoz_names:
# m = _hoz_names[0]
fn = univ_ib_eqidx_ext+i+'_'+j+'_'+k+'_'+m
# print(fn)
univ_ib_sig = cr_sig_mr_sg.retrieve(fn)[_ticker].values[z]
univ_ib_sig1 = filt.lag(univ_ib_sig)
univ_ib_sig2 = filt.lag(univ_ib_sig, 2)
univ_ib_sig1 = univ_ib_sig1[-2780:]
univ_ib_sig2 = univ_ib_sig2[-2780:]
t1 = spearmanr(univ_ib_cl, univ_ib_sig1).correlation
t2 = spearmanr(univ_ib_cl, univ_ib_sig2).correlation
d1 = spearmanr(np.sign(univ_ib_cl), np.sign(univ_ib_sig1)).correlation
d2 = spearmanr(np.sign(univ_ib_cl), np.sign(univ_ib_sig2)).correlation
td = np.array([t1, t2, d1, d2])*100
print(i+','+j+','+k+','+m+' :\t', np_to_str(td))
return None
# resid1, resid2, resid3, resid4, resid5, resid6 = reduce_nonnan(resid1, resid2, resid3, resid4, resid5, resid6)
f1 = []
f1.append(np.median(np.abs(resid0)))
f1.append(np.median(np.abs(resid1)))
f1.append(np.median(np.abs(resid2)))
f1.append(np.median(np.abs(resid3)))
f1.append(np.median(np.abs(resid4)))
# f1.append(np.mean(np.abs(resid0)))
# f1.append(np.mean(np.abs(resid1)))
# f1.append(np.mean(np.abs(resid2)))
# f1.append(np.mean(np.abs(resid3)))
# f1.append(np.mean(np.abs(resid4)))
# f1.append(np.median(np.abs(resid6)))
# f1.append(np.median(np.abs(resid7)))
# f1.append(np.median(np.abs(resid8)))
# f1.append(np.median(np.abs(resid9)))
f1 = np.array(f1)
print(np_to_str(f1))
# f1.append(np.median(np.abs(resid1)))
# f2.append(np.median(np.abs(resid2)))
# f3.append(np.median(np.abs(resid3)))
# f4.append(np.median(np.abs(resid4)))
# [print(i) for i in f3]
# [print(i) for i in f4]
# [print(i) for i in f1]
# [print(i) for i in f2]
def check_closest_volatility():
univ_ib_cl = cr_cret.retrieve(univ_ib_ext+'Close')
univ_ib_rt1 = univ_ib_cl.copy()
univ_ib_rt2 = univ_ib_cl.copy()
univ_ib_rt3 = univ_ib_cl.copy()
for i in univ_ib_cl.tick_cols():
# i = 'SP500'
univ_ib_cl[i] = filt.ret(univ_ib_cl[i].values)
univ_ib_rt1[i] = filt.ret(univ_ib_rt1[i].values, 30)
univ_ib_rt1[i] = filt.lag(univ_ib_rt1[i].values)
univ_ib_rt2[i] = filt.ret(univ_ib_rt2[i].values, 60)
univ_ib_rt2[i] = filt.lag(univ_ib_rt2[i].values)
univ_ib_rt3[i] = filt.ret(univ_ib_rt3[i].values, 120)
univ_ib_rt3[i] = filt.lag(univ_ib_rt3[i].values)
univ_ib_cl_data = univ_ib_cl[univ_ib_cl.tick_cols()].values.reshape(-1, )
univ_ib_rt1_data = univ_ib_rt1[univ_ib_rt1.tick_cols()].values.reshape(-1, )
univ_ib_rt2_data = univ_ib_rt2[univ_ib_rt2.tick_cols()].values.reshape(-1, )
univ_ib_rt3_data = univ_ib_rt3[univ_ib_rt3.tick_cols()].values.reshape(-1, )
univ_ib_cl_data = np.abs(univ_ib_cl_data)
import warnings
warnings.simplefilter('ignore', RuntimeWarning)
univ_ib_rt1_data = np.sign(univ_ib_rt1_data)
univ_ib_rt2_data = np.sign(univ_ib_rt2_data)
univ_ib_rt3_data = np.sign(univ_ib_rt3_data)
warnings.simplefilter('default', RuntimeWarning)
# vol_names_ = [i for i in vol_names if 'reg' not in i]
for j in vol_names:
for k in range(_min_range, _max_range+_step_range, _step_range):
if k < 100:
univ_ib_vol = cr_vol_all.retrieve(univ_ib_ext + j + '_0' + str(k))
else:
univ_ib_vol = cr_vol_all.retrieve(univ_ib_ext + j + '_' + str(k))
for i in univ_ib_vol.tick_cols():
univ_ib_vol[i] = filt.lag(univ_ib_vol[i].values)
univ_ib_vol_data = univ_ib_vol[univ_ib_vol.tick_cols()].values.reshape(-1, )
univ_ib_nn = ~np.isnan(univ_ib_cl_data) & ~np.isnan(univ_ib_vol_data)
univ_ib_vol_data_ = univ_ib_vol_data[univ_ib_nn]*_norm_mult
univ_ib_cl_data_ = univ_ib_cl_data[univ_ib_nn]
res0 = simple_quant_reg(univ_ib_cl_data_, univ_ib_vol_data_)
# res0 = QuantReg(univ_ib_cl_data_, univ_ib_vol_data_).fit(q=0.5).params
univ_ib_nn = ~np.isnan(univ_ib_cl_data) & ~np.isnan(univ_ib_vol_data) & (univ_ib_rt1_data < 0)
univ_ib_vol_data_ = univ_ib_vol_data[univ_ib_nn]*_norm_mult
univ_ib_cl_data_ = univ_ib_cl_data[univ_ib_nn]
res1 = simple_quant_reg(univ_ib_cl_data_, univ_ib_vol_data_)
# res1 = QuantReg(univ_ib_cl_data_, univ_ib_vol_data_).fit(q=0.5).params
univ_ib_nn = ~np.isnan(univ_ib_cl_data) & ~np.isnan(univ_ib_vol_data) & (univ_ib_rt2_data < 0)
univ_ib_vol_data_ = univ_ib_vol_data[univ_ib_nn]*_norm_mult
univ_ib_cl_data_ = univ_ib_cl_data[univ_ib_nn]
res2 = simple_quant_reg(univ_ib_cl_data_, univ_ib_vol_data_)
# res2 = QuantReg(univ_ib_cl_data_, univ_ib_vol_data_).fit(q=0.5).params
univ_ib_nn = ~np.isnan(univ_ib_cl_data) & ~np.isnan(univ_ib_vol_data) & (univ_ib_rt3_data < 0)
univ_ib_vol_data_ = univ_ib_vol_data[univ_ib_nn]*_norm_mult
univ_ib_cl_data_ = univ_ib_cl_data[univ_ib_nn]
# res3 = QuantReg(univ_ib_cl_data_, univ_ib_vol_data_).fit(q=0.5).params
res3 = simple_quant_reg(univ_ib_cl_data_, univ_ib_vol_data_)
univ_ib_nn = ~np.isnan(univ_ib_cl_data) & ~np.isnan(univ_ib_vol_data) & (univ_ib_rt1_data > 0)
univ_ib_vol_data_ = univ_ib_vol_data[univ_ib_nn] * _norm_mult
univ_ib_cl_data_ = univ_ib_cl_data[univ_ib_nn]
# res4 = QuantReg(univ_ib_cl_data_, univ_ib_vol_data_).fit(q=0.5).params
res4 = simple_quant_reg(univ_ib_cl_data_, univ_ib_vol_data_)
univ_ib_nn = ~np.isnan(univ_ib_cl_data) & ~np.isnan(univ_ib_vol_data) & (univ_ib_rt2_data > 0)
univ_ib_vol_data_ = univ_ib_vol_data[univ_ib_nn] * _norm_mult
univ_ib_cl_data_ = univ_ib_cl_data[univ_ib_nn]
# res5 = QuantReg(univ_ib_cl_data_, univ_ib_vol_data_).fit(q=0.5).params
res5 = simple_quant_reg(univ_ib_cl_data_, univ_ib_vol_data_)
univ_ib_nn = ~np.isnan(univ_ib_cl_data) & ~np.isnan(univ_ib_vol_data) & (univ_ib_rt3_data > 0)
univ_ib_vol_data_ = univ_ib_vol_data[univ_ib_nn] * _norm_mult
univ_ib_cl_data_ = univ_ib_cl_data[univ_ib_nn]
# res6 = QuantReg(univ_ib_cl_data_, univ_ib_vol_data_).fit(q=0.5).params
res6 = simple_quant_reg(univ_ib_cl_data_, univ_ib_vol_data_)
if k < 100:
print(j + '_0' + str(k)+'\t', np_to_str(res0), '\t', np_to_str(res1), '\t', np_to_str(res2), '\t', np_to_str(res3), '\t',
np_to_str(res4), '\t', np_to_str(res5), '\t', np_to_str(res6), '\t')
else:
print(j + '_' + str(k)+'\t', np_to_str(res0), '\t', np_to_str(res1), '\t', np_to_str(res2), '\t', np_to_str(res3), '\t',
np_to_str(res4), '\t', np_to_str(res5), '\t', np_to_str(res6), '\t')
print('\n')
def check_closest_volatility():
univ_ib_cl = cr_cret.retrieve(univ_ib_ext + "Close")
univ_ib_rt1 = univ_ib_cl.copy()
univ_ib_rt2 = univ_ib_cl.copy()
univ_ib_rt3 = univ_ib_cl.copy()
for i in univ_ib_cl.tick_cols():
univ_ib_cl[i] = filt.ret(univ_ib_cl[i].values)
univ_ib_rt1[i] = filt.ret(univ_ib_rt1[i].values, 30)
univ_ib_rt1[i] = filt.lag(univ_ib_rt1[i].values)
univ_ib_rt2[i] = filt.ret(univ_ib_rt2[i].values, 60)
univ_ib_rt2[i] = filt.lag(univ_ib_rt2[i].values)
univ_ib_rt3[i] = filt.ret(univ_ib_rt3[i].values, 120)
univ_ib_rt3[i] = filt.lag(univ_ib_rt3[i].values)
univ_ib_cl_data = univ_ib_cl[univ_ib_cl.tick_cols()].values.reshape(-1)
univ_ib_rt1_data = univ_ib_rt1[univ_ib_rt1.tick_cols()].values.reshape(-1)
univ_ib_rt2_data = univ_ib_rt2[univ_ib_rt2.tick_cols()].values.reshape(-1)
univ_ib_rt3_data = univ_ib_rt3[univ_ib_rt3.tick_cols()].values.reshape(-1)
univ_ib_cl_data = np.abs(univ_ib_cl_data)
import warnings
warnings.simplefilter("ignore", RuntimeWarning)
univ_ib_rt1_data = np.sign(univ_ib_rt1_data)
univ_ib_rt2_data = np.sign(univ_ib_rt2_data)
univ_ib_rt3_data = np.sign(univ_ib_rt3_data)
warnings.simplefilter("default", RuntimeWarning)
# vol_names_ = [i for i in vol_names if 'reg' not in i]
for j in vol_names_sm:
for k in range(_min_range, _max_range, _step_range * 3):
if k < 100:
univ_ib_vol = cr_vol_all.retrieve(univ_ib_ext + j + "_0" + str(k) + "D")
else:
univ_ib_vol = cr_vol_all.retrieve(univ_ib_ext + j + "_" + str(k) + "D")
for i in univ_ib_vol.tick_cols():
univ_ib_vol[i] = filt.lag(univ_ib_vol[i].values)
univ_ib_vol_data = univ_ib_vol[univ_ib_vol.tick_cols()].values.reshape(-1)
univ_ib_nn = ~np.isnan(univ_ib_cl_data) & ~np.isnan(univ_ib_vol_data)
univ_ib_vol_data_ = univ_ib_vol_data[univ_ib_nn] * _norm_mult
univ_ib_cl_data_ = univ_ib_cl_data[univ_ib_nn]
res0 = QuantReg(univ_ib_cl_data_, univ_ib_vol_data_).fit(q=0.5).params
univ_ib_nn = ~np.isnan(univ_ib_cl_data) & ~np.isnan(univ_ib_vol_data) & (univ_ib_rt1_data < 0)
univ_ib_vol_data_ = univ_ib_vol_data[univ_ib_nn] * _norm_mult
univ_ib_cl_data_ = univ_ib_cl_data[univ_ib_nn]
res1 = QuantReg(univ_ib_cl_data_, univ_ib_vol_data_).fit(q=0.5).params
univ_ib_nn = ~np.isnan(univ_ib_cl_data) & ~np.isnan(univ_ib_vol_data) & (univ_ib_rt2_data < 0)
univ_ib_vol_data_ = univ_ib_vol_data[univ_ib_nn] * _norm_mult
univ_ib_cl_data_ = univ_ib_cl_data[univ_ib_nn]
res2 = QuantReg(univ_ib_cl_data_, univ_ib_vol_data_).fit(q=0.5).params
univ_ib_nn = ~np.isnan(univ_ib_cl_data) & ~np.isnan(univ_ib_vol_data) & (univ_ib_rt3_data < 0)
univ_ib_vol_data_ = univ_ib_vol_data[univ_ib_nn] * _norm_mult
univ_ib_cl_data_ = univ_ib_cl_data[univ_ib_nn]
res3 = QuantReg(univ_ib_cl_data_, univ_ib_vol_data_).fit(q=0.5).params
univ_ib_nn = ~np.isnan(univ_ib_cl_data) & ~np.isnan(univ_ib_vol_data) & (univ_ib_rt1_data > 0)
univ_ib_vol_data_ = univ_ib_vol_data[univ_ib_nn] * _norm_mult
univ_ib_cl_data_ = univ_ib_cl_data[univ_ib_nn]
res4 = QuantReg(univ_ib_cl_data_, univ_ib_vol_data_).fit(q=0.5).params
univ_ib_nn = ~np.isnan(univ_ib_cl_data) & ~np.isnan(univ_ib_vol_data) & (univ_ib_rt2_data > 0)
univ_ib_vol_data_ = univ_ib_vol_data[univ_ib_nn] * _norm_mult
univ_ib_cl_data_ = univ_ib_cl_data[univ_ib_nn]
res5 = QuantReg(univ_ib_cl_data_, univ_ib_vol_data_).fit(q=0.5).params
univ_ib_nn = ~np.isnan(univ_ib_cl_data) & ~np.isnan(univ_ib_vol_data) & (univ_ib_rt3_data > 0)
univ_ib_vol_data_ = univ_ib_vol_data[univ_ib_nn] * _norm_mult
univ_ib_cl_data_ = univ_ib_cl_data[univ_ib_nn]
res6 = QuantReg(univ_ib_cl_data_, univ_ib_vol_data_).fit(q=0.5).params
if k < 100:
print(
j + "_0" + str(k) + "\t",
np_to_str(res0),
"\t",
np_to_str(res1),
"\t",
np_to_str(res2),
"\t",
np_to_str(res3),
"\t",
np_to_str(res4),
"\t",
np_to_str(res5),
"\t",
np_to_str(res6),
"\t",
)
else:
print(
j + "_" + str(k) + "\t",
np_to_str(res0),
"\t",
np_to_str(res1),
"\t",
np_to_str(res2),
"\t",
np_to_str(res3),
"\t",
np_to_str(res4),
"\t",
np_to_str(res5),
"\t",
np_to_str(res6),
"\t",
)
print("\n")
