def _refresh_old_exposure_matrix2(x): _d_opca_cl, _d_opca_wgt, _d_opca_bt = x f_files = _d_opca_cl.listdir() f_files_dt = crup.str_to_pd_dt(f_files) PCA_01 = _d_opca_wgt.retrieve('PCA01_Beta') PCA_02 = _d_opca_wgt.retrieve('PCA02_Beta') PCA_03 = _d_opca_wgt.retrieve('PCA03_Beta') PCA_04 = _d_opca_wgt.retrieve('PCA04_Beta') PCA_05 = _d_opca_wgt.retrieve('PCA05_Beta') PCA_06 = _d_opca_wgt.retrieve('PCA06_Beta') PCA_07 = _d_opca_wgt.retrieve('PCA07_Beta') PCA_08 = _d_opca_wgt.retrieve('PCA08_Beta') PCA_09 = _d_opca_wgt.retrieve('PCA09_Beta') PCA_10 = _d_opca_wgt.retrieve('PCA10_Beta') PCA_11 = _d_opca_wgt.retrieve('PCA11_Beta') PCA_12 = _d_opca_wgt.retrieve('PCA12_Beta') PCA_13 = _d_opca_wgt.retrieve('PCA13_Beta') PCA_14 = _d_opca_wgt.retrieve('PCA14_Beta') PCA_15 = _d_opca_wgt.retrieve('PCA15_Beta') PCA_16 = _d_opca_wgt.retrieve('PCA16_Beta') PCA_17 = _d_opca_wgt.retrieve('PCA17_Beta') PCA_18 = _d_opca_wgt.retrieve('PCA18_Beta') PCA_19 = _d_opca_wgt.retrieve('PCA19_Beta') PCA_20 = _d_opca_wgt.retrieve('PCA20_Beta') p_dt = crup.pd_dt_to_str(PCA_01['Date', list]) tick_cols = PCA_01.tick_cols() for i in f_files: # i = f_files[0] i_loc = [k for k, k2 in enumerate(p_dt) if k2 == i] t1 = _d_opca_cl.load(i)['arr_3'] # retain the first 5 t1_ = np.array(t1[0:5], dtype=np.int64) # first PCA cons1 = t1_[0]+1 if cons1 <= 9: pca_test = locals()['PCA_0'+str(cons1)][i_loc, :] else: pca_test = locals()['PCA_'+str(cons1)][i_loc, :] if any(~np.isnan(pca_test[0, tick_cols].values.astype(np.float64))): beta = pca_test for j in range(0, 4): cons1 = t1_[j+1]+1 if cons1 <= 9: pca_test = locals()['PCA_0'+str(cons1)][i_loc, :] else: pca_test = locals()['PCA_'+str(cons1)][i_loc, :] beta.row_bind(pca_test) del beta['Date'] else: beta = None if beta is not None: _d_opca_bt.store(i, beta) return None
def _collect_old_pca_factors2(x): _d_opca_cl, _d_opca_wgt = x f_files = _d_opca_cl.listdir() for i in f_files: if i == f_files[0]: t1 = np.array([_d_opca_cl.load(i)['arr_2']]) else: t1 = np.append(t1, [_d_opca_cl.load(i)['arr_2']], axis=0) t2 = cruf.DataFrame(t1) x = ['PCA01', 'PCA02', 'PCA03', 'PCA04', 'PCA05', 'PCA06', 'PCA07', 'PCA08', 'PCA09', 'PCA10', 'PCA11', 'PCA12', 'PCA13', 'PCA14', 'PCA15', 'PCA16', 'PCA17', 'PCA18', 'PCA19', 'PCA20'] t2.set_columns(x) t2['Date'] = crup.str_to_pd_dt(f_files) t2 = t2[['Date']+x] _d_opca_wgt.store('Returns', t2) for i in x: t2[i] = np.cumprod(t2[i].values/100+1) _d_opca_wgt.store('Prices', t2) return None
def generate_final_tl(): # calculate the latest last_dt = d_all_tl_id.listdir()[-1] last_pos = d_all_tl_id.load(last_dt) # calculate the turnover (for curiosity) pr = d_all_comb.retrieve('Close') un = crsf.cr_un.retrieve('US_Universe_Cur') # get the hedging information hlev = crsf.cr_comb[6].retrieve('Lev_CC_240D') hlevi = hlev[hlev.shape[0]-1, hlev.tick_cols()].values.astype('float64') adv = crsf.cr_comb[6].retrieve('Adv') advi = adv[adv.shape[0]-1, adv.tick_cols()].values.astype('float64') htci = hlev.tick_cols() hlev_can = np.where(hlevi > 10)[0] hlev_can_tc = [i for j, i in enumerate(htci) if j in hlev_can] hlev_can_vl = hlevi[hlev_can] advi_can_vl = advi[hlev_can] hedge_info = cruf.DataFrame({'Tick': hlev_can_tc, 'Value': hlev_can_vl, 'Turnover': advi_can_vl}) hedge_info.sort('Turnover') # calculate the actual position files pos = cruf.DataFrame({'Tick': last_pos['arr_0'], 'Weight': last_pos['arr_1']}) np.sum(pos['Weight'].values) # get net exposure to allocate x = last_pos['arr_1'] xp = np.sum(x[x > 0]) xn = -np.sum(x[x < 0]) xnet = xp-xn # distribute xnet by value hedge_info['Weight'] = -hlev_can_vl*xnet/np.sum(hlev_can_vl) hedge_info = hedge_info[['Tick', 'Weight']] pos.row_bind(hedge_info) # calculate the dollar values cap = 15000 pos['Dollar'] = cap*pos['Weight']*1000 # because the hedge will increase the size # for each ticker in pos, get the last close pr_tic = pr.tick_cols() pr_val = pr[pr.shape[0]-1, pr_tic].values.astype('float64') pr_info = cruf.DataFrame({'Tick': pr_tic, 'Close': pr_val}) pos = cruf.DataFrame.merge(pos, pr_info, how='outer', on='Tick') pos = pos[~np.isnan(pos['Weight'].values), :] pos['Shares'] = pos['Dollar'].values/pos['Close'].values pos['SharesRounded'] = np.round(pos['Shares'].values) pos['DollarRounded'] = pos['SharesRounded'].values * pos['Close'].values net_dol = np.sum(pos['DollarRounded'].values) bil_pr = pr_info[[i for i, j in enumerate(pr_info['Tick'].values) if j == 'BIL'], 'Close'].values[0] bil_ext = -np.round(net_dol/bil_pr) pos_bil = [i for i, j in enumerate(pos['Tick'].values) if j == 'BIL'][0] pos[pos_bil, 'SharesRounded'] += bil_ext pos[pos_bil, 'DollarRounded'] = pos[pos_bil, 'SharesRounded'] * pos[pos_bil, 'Close'] pos.sort('DollarRounded') # add universe information pos_ = cruf.DataFrame.merge(pos, un, how='outer', left_on='Tick', right_on='Ticker') pos_ = pos_[~np.isnan(pos['Weight'].values), :] del pos_['Ticker'] pos_['Date'] = crup.str_to_pd_dt([last_dt])[0] pos_ = pos_[['Date']+list(pos_.columns[:-1])] crsf.cr_all_tl_ac.store(last_dt+'_FullInfo', pos_) # aggregate information by sector exposure pos_['GICS_Sec'] = [(i if len(i)<2 else i[0:2]) for i in pos_['GICS'].values] sec_info = np.zeros((11, 2))*np.nan sec_info[0, 0] = np.sum(pos_[(pos_['Weight'].values > 0) & (pos_['GICS_Sec'].values == '10'), 'Weight'].values) sec_info[0, 1] = np.sum(pos_[(pos_['Weight'].values < 0) & (pos_['GICS_Sec'].values == '10'), 'Weight'].values) sec_info[1, 0] = np.sum(pos_[(pos_['Weight'].values > 0) & (pos_['GICS_Sec'].values == '15'), 'Weight'].values) sec_info[1, 1] = np.sum(pos_[(pos_['Weight'].values < 0) & (pos_['GICS_Sec'].values == '15'), 'Weight'].values) sec_info[2, 0] = np.sum(pos_[(pos_['Weight'].values > 0) & (pos_['GICS_Sec'].values == '20'), 'Weight'].values) sec_info[2, 1] = np.sum(pos_[(pos_['Weight'].values < 0) & (pos_['GICS_Sec'].values == '20'), 'Weight'].values) sec_info[3, 0] = np.sum(pos_[(pos_['Weight'].values > 0) & (pos_['GICS_Sec'].values == '25'), 'Weight'].values) sec_info[3, 1] = np.sum(pos_[(pos_['Weight'].values < 0) & (pos_['GICS_Sec'].values == '25'), 'Weight'].values) sec_info[4, 0] = np.sum(pos_[(pos_['Weight'].values > 0) & (pos_['GICS_Sec'].values == '30'), 'Weight'].values) sec_info[4, 1] = np.sum(pos_[(pos_['Weight'].values < 0) & (pos_['GICS_Sec'].values == '30'), 'Weight'].values) sec_info[5, 0] = np.sum(pos_[(pos_['Weight'].values > 0) & (pos_['GICS_Sec'].values == '35'), 'Weight'].values) sec_info[5, 1] = np.sum(pos_[(pos_['Weight'].values < 0) & (pos_['GICS_Sec'].values == '35'), 'Weight'].values) sec_info[6, 0] = np.sum(pos_[(pos_['Weight'].values > 0) & (pos_['GICS_Sec'].values == '40'), 'Weight'].values) sec_info[6, 1] = np.sum(pos_[(pos_['Weight'].values < 0) & (pos_['GICS_Sec'].values == '40'), 'Weight'].values) sec_info[7, 0] = np.sum(pos_[(pos_['Weight'].values > 0) & (pos_['GICS_Sec'].values == '45'), 'Weight'].values) sec_info[7, 1] = np.sum(pos_[(pos_['Weight'].values < 0) & (pos_['GICS_Sec'].values == '45'), 'Weight'].values) sec_info[8, 0] = np.sum(pos_[(pos_['Weight'].values > 0) & (pos_['GICS_Sec'].values == '50'), 'Weight'].values) sec_info[8, 1] = np.sum(pos_[(pos_['Weight'].values < 0) & (pos_['GICS_Sec'].values == '50'), 'Weight'].values) sec_info[9, 0] = np.sum(pos_[(pos_['Weight'].values > 0) & (pos_['GICS_Sec'].values == '55'), 'Weight'].values) sec_info[9, 1] = np.sum(pos_[(pos_['Weight'].values < 0) & (pos_['GICS_Sec'].values == '55'), 'Weight'].values) sec_info[10, 0] = np.sum(pos_[(pos_['Weight'].values > 0) & (pos_['GICS_Sec'].values == ''), 'Weight'].values) sec_info[10, 1] = np.sum(pos_[(pos_['Weight'].values < 0) & (pos_['GICS_Sec'].values == ''), 'Weight'].values) sec_info_sm = np.sum(sec_info, axis=0) sec_info[:, 0] = sec_info[:, 0]/sec_info_sm[0] sec_info[:, 1] = sec_info[:, 1]/sec_info_sm[1] sec_info = cruf.DataFrame(sec_info) sec_info.set_columns(['Long', 'Short']) sec_info['Category'] = ['Energy', 'Materials', 'Industrials', 'Cons Disc', 'Cons Staples', 'Health Care', 'Financials', 'IT', 'Telecom', 'Utilities', 'ETFs'] sec_info = sec_info[['Category', 'Long', 'Short']] sec_info['Net'] = sec_info['Long'].values - sec_info['Short'].values sec_info.rename({'Category': 'Sector'}) sec_info.to_csv(crsf.cr_all_tl_ac.name+'\\'+last_dt+'_SecInfo.csv', index_label=None, index=False) pos_ = pos_[['Date', 'Tick', 'SharesRounded', 'Type', 'Name']] pos_['SharesRounded'] = pos_['SharesRounded'].astype('int64') pos_ = pos_[['Date', 'Tick', 'Name', 'Type', 'SharesRounded']] pos_.rename({'SharesRounded': 'Shares'}) pos_.sort('Shares') pos_ = pos_[np.abs(pos_['Shares']).values>0, :] pos_.to_csv(crsf.cr_all_tl_ac.name+'\\'+last_dt+'_Portfolio.csv', index_label=None, index=False) # test = crsf.cr_all_tl_ac.retrieve(last_dt+'_FullInfo') # test.sort('Weight') return None
def _generate_tradelist2(x, prm=4, eiflag=False): _d_bt_sig, _d_tl_id, _d_comb = x ret = _d_comb.retrieve('Returns') lev = _d_comb.retrieve('Lev_CC_120D') vol = _d_comb.retrieve('Vol_CC_120D') eq03 = _d_comb.retrieve('EQIDX_Beta_3m') eq06 = _d_comb.retrieve('EQIDX_Beta_6m') eq09 = _d_comb.retrieve('EQIDX_Beta_9m') eq12 = _d_comb.retrieve('EQIDX_Beta_12m') sp03 = _d_comb.retrieve('SP_Beta_3m') sp06 = _d_comb.retrieve('SP_Beta_6m') sp09 = _d_comb.retrieve('SP_Beta_9m') sp12 = _d_comb.retrieve('SP_Beta_12m') mom03 = _d_comb.retrieve('MOM_Beta_3m') mom06 = _d_comb.retrieve('MOM_Beta_6m') mom09 = _d_comb.retrieve('MOM_Beta_9m') mom12 = _d_comb.retrieve('MOM_Beta_12m') f_list = _d_bt_sig.listdir() p_dt = crup.pd_dt_to_str(lev['Date', list]) p_dt_idx = [i for i, j in enumerate(p_dt) if j in f_list] # ret = ret[p_dt_idx, :] lev = lev[p_dt_idx, :] vol = vol[p_dt_idx, :] eq03 = eq03[p_dt_idx, :] eq06 = eq06[p_dt_idx, :] eq09 = eq09[p_dt_idx, :] eq12 = eq12[p_dt_idx, :] sp03 = sp03[p_dt_idx, :] sp06 = sp06[p_dt_idx, :] sp09 = sp09[p_dt_idx, :] sp12 = sp12[p_dt_idx, :] mom03 = mom03[p_dt_idx, :] mom06 = mom06[p_dt_idx, :] mom09 = mom09[p_dt_idx, :] mom12 = mom12[p_dt_idx, :] tc = lev.tick_cols() window = 200 n = len(f_list) ret_coll = np.zeros(n-2)*np.nan for i, j in enumerate(f_list): # i = 0 # j = f_list[0] # read the return estimates ret_est = _d_bt_sig.load(j) reti = ret_est['arr_1'] tci = list(ret_est['arr_0']) tci_idx = [ii for ii, jj in enumerate(tci) if jj in tc] levi = lev[i, tci].values.astype('float64') reti_un = (reti/levi)/1000 voli = vol[i, tci].values.astype('float64') eq03i = eq03[i, tci].values.astype('float64') eq06i = eq06[i, tci].values.astype('float64') eq09i = eq09[i, tci].values.astype('float64') eq12i = eq12[i, tci].values.astype('float64') sp03i = sp03[i, tci].values.astype('float64') sp06i = sp06[i, tci].values.astype('float64') sp09i = sp09[i, tci].values.astype('float64') sp12i = sp12[i, tci].values.astype('float64') mom03i = mom03[i, tci].values.astype('float64') mom06i = mom06[i, tci].values.astype('float64') mom09i = mom09[i, tci].values.astype('float64') mom12i = mom12[i, tci].values.astype('float64') ret_mini = ret[p_dt_idx[i]-window+1:p_dt_idx[i], tci].values.astype('float64') ret_mini[np.isnan(ret_mini)] = 0 # hygiene nn = len(tci) ei = np.ones(nn) cori = (np.corrcoef(ret_mini.T)+prm*np.identity(nn))/(1+prm) dvoli = 16*np.diag(voli) covi = np.linalg.inv(np.dot(dvoli, np.dot(cori, dvoli))) if not eiflag: mat_a = np.vstack((eq03i, eq06i, eq09i, eq12i, # ei sp03i, sp06i, sp09i, sp12i, mom03i, mom06i, mom09i, mom12i)) else: mat_a = np.vstack((ei, eq03i, eq06i, eq09i, eq12i, # ei sp03i, sp06i, sp09i, sp12i, mom03i, mom06i, mom09i, mom12i)) wgt_a = 1.5*np.dot(crts.neut_corr_mat(covi, mat_a), reti_un) wgt_a_sm = np.maximum(np.sum(wgt_a[wgt_a > 0]), -np.sum(wgt_a[wgt_a < 0])) # wgt_a_sm = np.sum(np.abs(wgt_a)) if wgt_a_sm > 4: wgt_a = wgt_a*4/wgt_a_sm if wgt_a_sm < 0.5: wgt_a = wgt_a*0.5/wgt_a_sm _d_tl_id.save(j, tci, wgt_a) if i < n-2: reti = ret[p_dt_idx[i+2], tci].values.astype('float64') ret_coll[i] = np.sum(wgt_a*reti) # print(j, np.array([np.sum(np.abs(wgt_a)), np.max(wgt_a), np.min(wgt_a), ret_coll[i]])) else: pass # print(j, np.array([np.sum(np.abs(wgt_a)), np.max(wgt_a), np.min(wgt_a)])) # print(16*np.mean(ret_coll)/np.std(ret_coll)) # ret_coll.shape f_list = f_list[:-2] f_list_ = crup.str_to_pd_dt(f_list) ret_coll = cruf.DataFrame({'Date': f_list_, 'Returns': ret_coll}) _d_comb.store('Signal_Returns', ret_coll) # return ret_coll return None
def _refresh_combined_betas2(x): _d_comb, _d_opca_bt = x ret_data = _d_comb.retrieve('Returns') tick_cols = ret_data.tick_cols() f_list = _d_opca_bt.listdir() for i in f_list: x = _d_opca_bt.retrieve(i) if i == f_list[0]: b1 = np.array([x[0, :]]) b2 = np.array([x[1, :]]) b3 = np.array([x[2, :]]) b4 = np.array([x[3, :]]) # b5 = np.array([x[4, :]]) else: b1 = np.vstack((b1, x[0, :])) b2 = np.vstack((b2, x[1, :])) b3 = np.vstack((b3, x[2, :])) b4 = np.vstack((b4, x[3, :])) # b5 = np.vstack((b5, x[4, :])) f_list_dt = crup.str_to_pd_dt(f_list) b1df = cruf.DataFrame(b1) b1df.set_columns(tick_cols) b2df = cruf.DataFrame(b2) b2df.set_columns(tick_cols) b3df = cruf.DataFrame(b3) b3df.set_columns(tick_cols) b4df = cruf.DataFrame(b4) b4df.set_columns(tick_cols) # b5df = cruf.DataFrame(b5) # b5df.set_columns(tick_cols) b1df['Date'] = f_list_dt b1df = b1df[['Date']+tick_cols] b2df['Date'] = f_list_dt b2df = b2df[['Date']+tick_cols] b3df['Date'] = f_list_dt b3df = b3df[['Date']+tick_cols] b4df['Date'] = f_list_dt b4df = b4df[['Date']+tick_cols] # b5df['Date'] = f_list_dt # b5df = b5df[['Date']+tick_cols] ret_data['Dummy'] = np.nan ret_data = ret_data[['Date', 'Dummy']] b1df = cruf.DataFrame.merge(ret_data, b1df, on='Date') b2df = cruf.DataFrame.merge(ret_data, b2df, on='Date') b3df = cruf.DataFrame.merge(ret_data, b3df, on='Date') b4df = cruf.DataFrame.merge(ret_data, b4df, on='Date') # b5df = cruf.DataFrame.merge(ret_data, b5df, on='Date') del b1df['Dummy'] del b2df['Dummy'] del b3df['Dummy'] del b4df['Dummy'] # del b5df['Dummy'] _d_comb.store('OPCA0_Beta', b1df) _d_comb.store('OPCA1_Beta', b2df) _d_comb.store('OPCA2_Beta', b3df) _d_comb.store('OPCA3_Beta', b4df)