def _refresh_old_exposure_matrix2(x):
_d_opca_cl, _d_opca_wgt, _d_opca_bt = x
f_files = _d_opca_cl.listdir()
f_files_dt = crup.str_to_pd_dt(f_files)
PCA_01 = _d_opca_wgt.retrieve('PCA01_Beta')
PCA_02 = _d_opca_wgt.retrieve('PCA02_Beta')
PCA_03 = _d_opca_wgt.retrieve('PCA03_Beta')
PCA_04 = _d_opca_wgt.retrieve('PCA04_Beta')
PCA_05 = _d_opca_wgt.retrieve('PCA05_Beta')
PCA_06 = _d_opca_wgt.retrieve('PCA06_Beta')
PCA_07 = _d_opca_wgt.retrieve('PCA07_Beta')
PCA_08 = _d_opca_wgt.retrieve('PCA08_Beta')
PCA_09 = _d_opca_wgt.retrieve('PCA09_Beta')
PCA_10 = _d_opca_wgt.retrieve('PCA10_Beta')
PCA_11 = _d_opca_wgt.retrieve('PCA11_Beta')
PCA_12 = _d_opca_wgt.retrieve('PCA12_Beta')
PCA_13 = _d_opca_wgt.retrieve('PCA13_Beta')
PCA_14 = _d_opca_wgt.retrieve('PCA14_Beta')
PCA_15 = _d_opca_wgt.retrieve('PCA15_Beta')
PCA_16 = _d_opca_wgt.retrieve('PCA16_Beta')
PCA_17 = _d_opca_wgt.retrieve('PCA17_Beta')
PCA_18 = _d_opca_wgt.retrieve('PCA18_Beta')
PCA_19 = _d_opca_wgt.retrieve('PCA19_Beta')
PCA_20 = _d_opca_wgt.retrieve('PCA20_Beta')
p_dt = crup.pd_dt_to_str(PCA_01['Date', list])
tick_cols = PCA_01.tick_cols()
for i in f_files:
# i = f_files[0]
i_loc = [k for k, k2 in enumerate(p_dt) if k2 == i]
t1 = _d_opca_cl.load(i)['arr_3']
# retain the first 5
t1_ = np.array(t1[0:5], dtype=np.int64)
# first PCA
cons1 = t1_[0]+1
if cons1 <= 9:
pca_test = locals()['PCA_0'+str(cons1)][i_loc, :]
else:
pca_test = locals()['PCA_'+str(cons1)][i_loc, :]
if any(~np.isnan(pca_test[0, tick_cols].values.astype(np.float64))):
beta = pca_test
for j in range(0, 4):
cons1 = t1_[j+1]+1
if cons1 <= 9:
pca_test = locals()['PCA_0'+str(cons1)][i_loc, :]
else:
pca_test = locals()['PCA_'+str(cons1)][i_loc, :]
beta.row_bind(pca_test)
del beta['Date']
else:
beta = None
if beta is not None:
_d_opca_bt.store(i, beta)
return None
def _collect_old_pca_factors2(x):
_d_opca_cl, _d_opca_wgt = x
f_files = _d_opca_cl.listdir()
for i in f_files:
if i == f_files[0]:
t1 = np.array([_d_opca_cl.load(i)['arr_2']])
else:
t1 = np.append(t1, [_d_opca_cl.load(i)['arr_2']], axis=0)
t2 = cruf.DataFrame(t1)
x = ['PCA01', 'PCA02', 'PCA03', 'PCA04', 'PCA05', 'PCA06', 'PCA07', 'PCA08', 'PCA09', 'PCA10', 'PCA11',
'PCA12', 'PCA13', 'PCA14', 'PCA15', 'PCA16', 'PCA17', 'PCA18', 'PCA19', 'PCA20']
t2.set_columns(x)
t2['Date'] = crup.str_to_pd_dt(f_files)
t2 = t2[['Date']+x]
_d_opca_wgt.store('Returns', t2)
for i in x:
t2[i] = np.cumprod(t2[i].values/100+1)
_d_opca_wgt.store('Prices', t2)
return None
def generate_final_tl():
# calculate the latest
last_dt = d_all_tl_id.listdir()[-1]
last_pos = d_all_tl_id.load(last_dt)
# calculate the turnover (for curiosity)
pr = d_all_comb.retrieve('Close')
un = crsf.cr_un.retrieve('US_Universe_Cur')
# get the hedging information
hlev = crsf.cr_comb[6].retrieve('Lev_CC_240D')
hlevi = hlev[hlev.shape[0]-1, hlev.tick_cols()].values.astype('float64')
adv = crsf.cr_comb[6].retrieve('Adv')
advi = adv[adv.shape[0]-1, adv.tick_cols()].values.astype('float64')
htci = hlev.tick_cols()
hlev_can = np.where(hlevi > 10)[0]
hlev_can_tc = [i for j, i in enumerate(htci) if j in hlev_can]
hlev_can_vl = hlevi[hlev_can]
advi_can_vl = advi[hlev_can]
hedge_info = cruf.DataFrame({'Tick': hlev_can_tc, 'Value': hlev_can_vl, 'Turnover': advi_can_vl})
hedge_info.sort('Turnover')
# calculate the actual position files
pos = cruf.DataFrame({'Tick': last_pos['arr_0'], 'Weight': last_pos['arr_1']})
np.sum(pos['Weight'].values)
# get net exposure to allocate
x = last_pos['arr_1']
xp = np.sum(x[x > 0])
xn = -np.sum(x[x < 0])
xnet = xp-xn
# distribute xnet by value
hedge_info['Weight'] = -hlev_can_vl*xnet/np.sum(hlev_can_vl)
hedge_info = hedge_info[['Tick', 'Weight']]
pos.row_bind(hedge_info)
# calculate the dollar values
cap = 15000
pos['Dollar'] = cap*pos['Weight']*1000 # because the hedge will increase the size
# for each ticker in pos, get the last close
pr_tic = pr.tick_cols()
pr_val = pr[pr.shape[0]-1, pr_tic].values.astype('float64')
pr_info = cruf.DataFrame({'Tick': pr_tic, 'Close': pr_val})
pos = cruf.DataFrame.merge(pos, pr_info, how='outer', on='Tick')
pos = pos[~np.isnan(pos['Weight'].values), :]
pos['Shares'] = pos['Dollar'].values/pos['Close'].values
pos['SharesRounded'] = np.round(pos['Shares'].values)
pos['DollarRounded'] = pos['SharesRounded'].values * pos['Close'].values
net_dol = np.sum(pos['DollarRounded'].values)
bil_pr = pr_info[[i for i, j in enumerate(pr_info['Tick'].values) if j == 'BIL'], 'Close'].values[0]
bil_ext = -np.round(net_dol/bil_pr)
pos_bil = [i for i, j in enumerate(pos['Tick'].values) if j == 'BIL'][0]
pos[pos_bil, 'SharesRounded'] += bil_ext
pos[pos_bil, 'DollarRounded'] = pos[pos_bil, 'SharesRounded'] * pos[pos_bil, 'Close']
pos.sort('DollarRounded')
# add universe information
pos_ = cruf.DataFrame.merge(pos, un, how='outer', left_on='Tick', right_on='Ticker')
pos_ = pos_[~np.isnan(pos['Weight'].values), :]
del pos_['Ticker']
pos_['Date'] = crup.str_to_pd_dt([last_dt])[0]
pos_ = pos_[['Date']+list(pos_.columns[:-1])]
crsf.cr_all_tl_ac.store(last_dt+'_FullInfo', pos_)
# aggregate information by sector exposure
pos_['GICS_Sec'] = [(i if len(i)<2 else i[0:2]) for i in pos_['GICS'].values]
sec_info = np.zeros((11, 2))*np.nan
sec_info[0, 0] = np.sum(pos_[(pos_['Weight'].values > 0) & (pos_['GICS_Sec'].values == '10'), 'Weight'].values)
sec_info[0, 1] = np.sum(pos_[(pos_['Weight'].values < 0) & (pos_['GICS_Sec'].values == '10'), 'Weight'].values)
sec_info[1, 0] = np.sum(pos_[(pos_['Weight'].values > 0) & (pos_['GICS_Sec'].values == '15'), 'Weight'].values)
sec_info[1, 1] = np.sum(pos_[(pos_['Weight'].values < 0) & (pos_['GICS_Sec'].values == '15'), 'Weight'].values)
sec_info[2, 0] = np.sum(pos_[(pos_['Weight'].values > 0) & (pos_['GICS_Sec'].values == '20'), 'Weight'].values)
sec_info[2, 1] = np.sum(pos_[(pos_['Weight'].values < 0) & (pos_['GICS_Sec'].values == '20'), 'Weight'].values)
sec_info[3, 0] = np.sum(pos_[(pos_['Weight'].values > 0) & (pos_['GICS_Sec'].values == '25'), 'Weight'].values)
sec_info[3, 1] = np.sum(pos_[(pos_['Weight'].values < 0) & (pos_['GICS_Sec'].values == '25'), 'Weight'].values)
sec_info[4, 0] = np.sum(pos_[(pos_['Weight'].values > 0) & (pos_['GICS_Sec'].values == '30'), 'Weight'].values)
sec_info[4, 1] = np.sum(pos_[(pos_['Weight'].values < 0) & (pos_['GICS_Sec'].values == '30'), 'Weight'].values)
sec_info[5, 0] = np.sum(pos_[(pos_['Weight'].values > 0) & (pos_['GICS_Sec'].values == '35'), 'Weight'].values)
sec_info[5, 1] = np.sum(pos_[(pos_['Weight'].values < 0) & (pos_['GICS_Sec'].values == '35'), 'Weight'].values)
sec_info[6, 0] = np.sum(pos_[(pos_['Weight'].values > 0) & (pos_['GICS_Sec'].values == '40'), 'Weight'].values)
sec_info[6, 1] = np.sum(pos_[(pos_['Weight'].values < 0) & (pos_['GICS_Sec'].values == '40'), 'Weight'].values)
sec_info[7, 0] = np.sum(pos_[(pos_['Weight'].values > 0) & (pos_['GICS_Sec'].values == '45'), 'Weight'].values)
sec_info[7, 1] = np.sum(pos_[(pos_['Weight'].values < 0) & (pos_['GICS_Sec'].values == '45'), 'Weight'].values)
sec_info[8, 0] = np.sum(pos_[(pos_['Weight'].values > 0) & (pos_['GICS_Sec'].values == '50'), 'Weight'].values)
sec_info[8, 1] = np.sum(pos_[(pos_['Weight'].values < 0) & (pos_['GICS_Sec'].values == '50'), 'Weight'].values)
sec_info[9, 0] = np.sum(pos_[(pos_['Weight'].values > 0) & (pos_['GICS_Sec'].values == '55'), 'Weight'].values)
sec_info[9, 1] = np.sum(pos_[(pos_['Weight'].values < 0) & (pos_['GICS_Sec'].values == '55'), 'Weight'].values)
sec_info[10, 0] = np.sum(pos_[(pos_['Weight'].values > 0) & (pos_['GICS_Sec'].values == ''), 'Weight'].values)
sec_info[10, 1] = np.sum(pos_[(pos_['Weight'].values < 0) & (pos_['GICS_Sec'].values == ''), 'Weight'].values)
sec_info_sm = np.sum(sec_info, axis=0)
sec_info[:, 0] = sec_info[:, 0]/sec_info_sm[0]
sec_info[:, 1] = sec_info[:, 1]/sec_info_sm[1]
sec_info = cruf.DataFrame(sec_info)
sec_info.set_columns(['Long', 'Short'])
sec_info['Category'] = ['Energy', 'Materials', 'Industrials', 'Cons Disc', 'Cons Staples', 'Health Care',
'Financials', 'IT', 'Telecom', 'Utilities', 'ETFs']
sec_info = sec_info[['Category', 'Long', 'Short']]
sec_info['Net'] = sec_info['Long'].values - sec_info['Short'].values
sec_info.rename({'Category': 'Sector'})
sec_info.to_csv(crsf.cr_all_tl_ac.name+'\\'+last_dt+'_SecInfo.csv', index_label=None, index=False)
pos_ = pos_[['Date', 'Tick', 'SharesRounded', 'Type', 'Name']]
pos_['SharesRounded'] = pos_['SharesRounded'].astype('int64')
pos_ = pos_[['Date', 'Tick', 'Name', 'Type', 'SharesRounded']]
pos_.rename({'SharesRounded': 'Shares'})
pos_.sort('Shares')
pos_ = pos_[np.abs(pos_['Shares']).values>0, :]
pos_.to_csv(crsf.cr_all_tl_ac.name+'\\'+last_dt+'_Portfolio.csv', index_label=None, index=False)
# test = crsf.cr_all_tl_ac.retrieve(last_dt+'_FullInfo')
# test.sort('Weight')
return None
def _generate_tradelist2(x, prm=4, eiflag=False):
_d_bt_sig, _d_tl_id, _d_comb = x
ret = _d_comb.retrieve('Returns')
lev = _d_comb.retrieve('Lev_CC_120D')
vol = _d_comb.retrieve('Vol_CC_120D')
eq03 = _d_comb.retrieve('EQIDX_Beta_3m')
eq06 = _d_comb.retrieve('EQIDX_Beta_6m')
eq09 = _d_comb.retrieve('EQIDX_Beta_9m')
eq12 = _d_comb.retrieve('EQIDX_Beta_12m')
sp03 = _d_comb.retrieve('SP_Beta_3m')
sp06 = _d_comb.retrieve('SP_Beta_6m')
sp09 = _d_comb.retrieve('SP_Beta_9m')
sp12 = _d_comb.retrieve('SP_Beta_12m')
mom03 = _d_comb.retrieve('MOM_Beta_3m')
mom06 = _d_comb.retrieve('MOM_Beta_6m')
mom09 = _d_comb.retrieve('MOM_Beta_9m')
mom12 = _d_comb.retrieve('MOM_Beta_12m')
f_list = _d_bt_sig.listdir()
p_dt = crup.pd_dt_to_str(lev['Date', list])
p_dt_idx = [i for i, j in enumerate(p_dt) if j in f_list]
# ret = ret[p_dt_idx, :]
lev = lev[p_dt_idx, :]
vol = vol[p_dt_idx, :]
eq03 = eq03[p_dt_idx, :]
eq06 = eq06[p_dt_idx, :]
eq09 = eq09[p_dt_idx, :]
eq12 = eq12[p_dt_idx, :]
sp03 = sp03[p_dt_idx, :]
sp06 = sp06[p_dt_idx, :]
sp09 = sp09[p_dt_idx, :]
sp12 = sp12[p_dt_idx, :]
mom03 = mom03[p_dt_idx, :]
mom06 = mom06[p_dt_idx, :]
mom09 = mom09[p_dt_idx, :]
mom12 = mom12[p_dt_idx, :]
tc = lev.tick_cols()
window = 200
n = len(f_list)
ret_coll = np.zeros(n-2)*np.nan
for i, j in enumerate(f_list):
# i = 0
# j = f_list[0]
# read the return estimates
ret_est = _d_bt_sig.load(j)
reti = ret_est['arr_1']
tci = list(ret_est['arr_0'])
tci_idx = [ii for ii, jj in enumerate(tci) if jj in tc]
levi = lev[i, tci].values.astype('float64')
reti_un = (reti/levi)/1000
voli = vol[i, tci].values.astype('float64')
eq03i = eq03[i, tci].values.astype('float64')
eq06i = eq06[i, tci].values.astype('float64')
eq09i = eq09[i, tci].values.astype('float64')
eq12i = eq12[i, tci].values.astype('float64')
sp03i = sp03[i, tci].values.astype('float64')
sp06i = sp06[i, tci].values.astype('float64')
sp09i = sp09[i, tci].values.astype('float64')
sp12i = sp12[i, tci].values.astype('float64')
mom03i = mom03[i, tci].values.astype('float64')
mom06i = mom06[i, tci].values.astype('float64')
mom09i = mom09[i, tci].values.astype('float64')
mom12i = mom12[i, tci].values.astype('float64')
ret_mini = ret[p_dt_idx[i]-window+1:p_dt_idx[i], tci].values.astype('float64')
ret_mini[np.isnan(ret_mini)] = 0 # hygiene
nn = len(tci)
ei = np.ones(nn)
cori = (np.corrcoef(ret_mini.T)+prm*np.identity(nn))/(1+prm)
dvoli = 16*np.diag(voli)
covi = np.linalg.inv(np.dot(dvoli, np.dot(cori, dvoli)))
if not eiflag:
mat_a = np.vstack((eq03i, eq06i, eq09i, eq12i, # ei
sp03i, sp06i, sp09i, sp12i,
mom03i, mom06i, mom09i, mom12i))
else:
mat_a = np.vstack((ei, eq03i, eq06i, eq09i, eq12i, # ei
sp03i, sp06i, sp09i, sp12i,
mom03i, mom06i, mom09i, mom12i))
wgt_a = 1.5*np.dot(crts.neut_corr_mat(covi, mat_a), reti_un)
wgt_a_sm = np.maximum(np.sum(wgt_a[wgt_a > 0]), -np.sum(wgt_a[wgt_a < 0]))
# wgt_a_sm = np.sum(np.abs(wgt_a))
if wgt_a_sm > 4:
wgt_a = wgt_a*4/wgt_a_sm
if wgt_a_sm < 0.5:
wgt_a = wgt_a*0.5/wgt_a_sm
_d_tl_id.save(j, tci, wgt_a)
if i < n-2:
reti = ret[p_dt_idx[i+2], tci].values.astype('float64')
ret_coll[i] = np.sum(wgt_a*reti)
# print(j, np.array([np.sum(np.abs(wgt_a)), np.max(wgt_a), np.min(wgt_a), ret_coll[i]]))
else:
pass
# print(j, np.array([np.sum(np.abs(wgt_a)), np.max(wgt_a), np.min(wgt_a)]))
# print(16*np.mean(ret_coll)/np.std(ret_coll))
# ret_coll.shape
f_list = f_list[:-2]
f_list_ = crup.str_to_pd_dt(f_list)
ret_coll = cruf.DataFrame({'Date': f_list_, 'Returns': ret_coll})
_d_comb.store('Signal_Returns', ret_coll)
# return ret_coll
return None
def _refresh_combined_betas2(x):
_d_comb, _d_opca_bt = x
ret_data = _d_comb.retrieve('Returns')
tick_cols = ret_data.tick_cols()
f_list = _d_opca_bt.listdir()
for i in f_list:
x = _d_opca_bt.retrieve(i)
if i == f_list[0]:
b1 = np.array([x[0, :]])
b2 = np.array([x[1, :]])
b3 = np.array([x[2, :]])
b4 = np.array([x[3, :]])
# b5 = np.array([x[4, :]])
else:
b1 = np.vstack((b1, x[0, :]))
b2 = np.vstack((b2, x[1, :]))
b3 = np.vstack((b3, x[2, :]))
b4 = np.vstack((b4, x[3, :]))
# b5 = np.vstack((b5, x[4, :]))
f_list_dt = crup.str_to_pd_dt(f_list)
b1df = cruf.DataFrame(b1)
b1df.set_columns(tick_cols)
b2df = cruf.DataFrame(b2)
b2df.set_columns(tick_cols)
b3df = cruf.DataFrame(b3)
b3df.set_columns(tick_cols)
b4df = cruf.DataFrame(b4)
b4df.set_columns(tick_cols)
# b5df = cruf.DataFrame(b5)
# b5df.set_columns(tick_cols)
b1df['Date'] = f_list_dt
b1df = b1df[['Date']+tick_cols]
b2df['Date'] = f_list_dt
b2df = b2df[['Date']+tick_cols]
b3df['Date'] = f_list_dt
b3df = b3df[['Date']+tick_cols]
b4df['Date'] = f_list_dt
b4df = b4df[['Date']+tick_cols]
# b5df['Date'] = f_list_dt
# b5df = b5df[['Date']+tick_cols]
ret_data['Dummy'] = np.nan
ret_data = ret_data[['Date', 'Dummy']]
b1df = cruf.DataFrame.merge(ret_data, b1df, on='Date')
b2df = cruf.DataFrame.merge(ret_data, b2df, on='Date')
b3df = cruf.DataFrame.merge(ret_data, b3df, on='Date')
b4df = cruf.DataFrame.merge(ret_data, b4df, on='Date')
# b5df = cruf.DataFrame.merge(ret_data, b5df, on='Date')
del b1df['Dummy']
del b2df['Dummy']
del b3df['Dummy']
del b4df['Dummy']
# del b5df['Dummy']
_d_comb.store('OPCA0_Beta', b1df)
_d_comb.store('OPCA1_Beta', b2df)
_d_comb.store('OPCA2_Beta', b3df)
_d_comb.store('OPCA3_Beta', b4df)
