def get_cov_test(): decay = 0.003 Coef = pd.read_csv(r'%s/Coef_%s_%s_%.4f.csv' % (DataFolder, START_TRAIN.date(), END_TRAIN.date(), decay)) Res = pd.read_csv(r'%s/Res_%s_%s_%.4f.csv' % (DataFolder, START_TRAIN.date(), END_TRAIN.date(), decay)) total_cov = functions.get_cov(wTickers, Coef, Res, CarhartDaily, START_TRAIN, END_TRAIN) # plt.pcolor(total_cov) # plt.show() return total_cov
def pfOptimizer6(longTickers, shortTickersHigh, shortTickersLow, Coef, Res, CarhartDaily, StockBeta, BETA_BOUND, WEIGHT_BOUND, back_date, start_date): shortTickers = shortTickersHigh.append(shortTickersLow).sort('ticker').reset_index(drop=True) longTickers['bWeight'] = 1 # binary weights shortTickers['bWeight'] = -1 shortTickersLow['bWeight'] = -1 pfTickers = pd.concat((shortTickers[['ticker']], longTickers[['ticker']]), axis=0).sort('ticker').reset_index(drop=True) sens = Coef[['ticker', 'Mkt-RF', 'SMB', 'HML', 'UMD']].merge(pfTickers) mBeta = matrix(sens['Mkt-RF']) mBeta_smb = matrix(sens['SMB']) mBeta_hml = matrix(sens['HML']) mBeta_umd = matrix(sens['UMD']) mCqaBeta = matrix(StockBeta.merge(pfTickers)['cqaBeta']) longIndex = matrix(pfTickers.merge(longTickers, how='left').fillna(0)['bWeight']) mLongIndex = np.diag(pfTickers.merge(longTickers, how='left').fillna(0)['bWeight']) mLongIndex = matrix(mLongIndex[np.logical_or.reduce([np.sum(mLongIndex, 1) > 0.5])]).trans() shortIndex = -matrix(pfTickers.merge(shortTickers, how='left').fillna(0)['bWeight']) mShortIndex = -np.diag(pfTickers.merge(shortTickers, how='left').fillna(0)['bWeight']) mShortIndex = matrix(mShortIndex[np.logical_or.reduce([np.sum(mShortIndex, 1) > 0.5])]).trans() shortLowIndex = -matrix(pfTickers.merge(shortTickersLow, how='left').fillna(0)['bWeight']) N = pfTickers.shape[0] id = spmatrix(1.0, range(N), range(N)) wBounds = matrix(np.ones((N, 1)) * WEIGHT_BOUND) longBounds = matrix(np.zeros((shortTickers.shape[0], 1))) shortBounds = matrix(np.zeros((longTickers.shape[0], 1))) total_cov = matrix(functions.get_cov(pfTickers, Coef, Res, CarhartDaily, back_date, start_date)) q = matrix(np.zeros((pfTickers.shape[0], 1))) G = matrix([[mBeta], [-mBeta], # [mBeta_smb], [-mBeta_smb], # [mBeta_hml], [-mBeta_hml], # [mBeta_umd], [-mBeta_umd], [mCqaBeta], [-mCqaBeta], [longIndex], [-longIndex], [shortIndex], [-shortIndex], [-shortLowIndex], [id], [-id], [-mLongIndex], [mShortIndex]]).trans() opt_comp = pd.DataFrame() h = matrix([0.1, 0.2, # 0.1, 0.1, # 0.1, 0.1, # 0.1, 0.1, BETA_BOUND, BETA_BOUND, 1, -0.98, -0.98, 1, 0.3, wBounds, wBounds, longBounds, shortBounds]) sol = solvers.qp(total_cov, q, G, h) w_final = sol['x'] wTickers = pfTickers wTickers['weight'] = w_final return (opt_comp, wTickers)
def pfOptimizer5(longTickers, shortTickersHigh, shortTickersLow, Coef, Res, CarhartDaily, StockBeta, BETA_BOUND, WEIGHT_BOUND, back_date, start_date): shortTickers = shortTickersHigh.append(shortTickersLow).sort( 'ticker').reset_index(drop=True) longTickers['bWeight'] = 1 # binary weights shortTickers['bWeight'] = -1 shortTickersLow['bWeight'] = -1 pfTickers = pd.concat((shortTickers[['ticker']], longTickers[['ticker']]), axis=0).sort('ticker').reset_index(drop=True) sens = Coef[['ticker', 'Mkt-RF', 'SMB', 'HML', 'UMD']].merge(pfTickers) mBeta = matrix(sens['Mkt-RF']) mBeta_smb = matrix(sens['SMB']) mBeta_hml = matrix(sens['HML']) mBeta_umd = matrix(sens['UMD']) mCqaBeta = matrix(StockBeta.merge(pfTickers)['cqaBeta']) longIndex = matrix( pfTickers.merge(longTickers, how='left').fillna(0)['bWeight']) mLongIndex = np.diag( pfTickers.merge(longTickers, how='left').fillna(0)['bWeight']) mLongIndex = matrix(mLongIndex[np.logical_or.reduce( [np.sum(mLongIndex, 1) > 0.5])]).trans() shortIndex = -matrix( pfTickers.merge(shortTickers, how='left').fillna(0)['bWeight']) mShortIndex = -np.diag( pfTickers.merge(shortTickers, how='left').fillna(0)['bWeight']) mShortIndex = matrix(mShortIndex[np.logical_or.reduce( [np.sum(mShortIndex, 1) > 0.5])]).trans() shortLowIndex = -matrix( pfTickers.merge(shortTickersLow, how='left').fillna(0)['bWeight']) N = pfTickers.shape[0] id = spmatrix(1.0, range(N), range(N)) wBounds = matrix(np.ones((N, 1)) * WEIGHT_BOUND) longBounds = matrix(np.zeros((shortTickers.shape[0], 1))) shortBounds = matrix(np.zeros((longTickers.shape[0], 1))) total_cov = matrix( functions.get_cov(pfTickers, Coef, Res, CarhartDaily, back_date, start_date)) q = matrix(np.zeros((pfTickers.shape[0], 1))) G = matrix([[mBeta], [-mBeta], [mBeta_smb], [-mBeta_smb], [mBeta_hml], [-mBeta_hml], [mBeta_umd], [-mBeta_umd], [mCqaBeta], [-mCqaBeta], [longIndex], [-longIndex], [shortIndex], [-shortIndex], [-shortLowIndex], [id], [-id], [-mLongIndex], [mShortIndex]]).trans() opt_comp = pd.DataFrame() h = matrix([ 0.1, 0.2, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, BETA_BOUND, BETA_BOUND, 1, -0.98, -0.98, 1, 0.3, wBounds, wBounds, longBounds, shortBounds ]) sol = solvers.qp(total_cov, q, G, h) w_final = sol['x'] wTickers = pfTickers wTickers['weight'] = w_final return (opt_comp, wTickers)
def get_cov_test(): decay = 0.003 Coef = pd.read_csv( r'%s/Coef_%s_%s_%.4f.csv' % (DataFolder, START_TRAIN.date(), END_TRAIN.date(), decay)) Res = pd.read_csv( r'%s/Res_%s_%s_%.4f.csv' % (DataFolder, START_TRAIN.date(), END_TRAIN.date(), decay)) total_cov = functions.get_cov(wTickers, Coef, Res, CarhartDaily, START_TRAIN, END_TRAIN) # plt.pcolor(total_cov) # plt.show() return total_cov
def pf_optimizer(long_tickers, short_tickers, Coef, industry_data, CarhartDaily, StockBeta, beta_bound, position_bound, sector_bound, three_factor_bound, trim, back_date, start_date): long_tickers['bWeight'] = 1 # binary weights short_tickers['bWeight'] = -1 pf_tickers = pd.concat((short_tickers[['ticker']], long_tickers[['ticker']]), axis=0).sort('ticker').reset_index(drop=True).merge(industry_data[['ticker', 'sector']]) sens = Coef[['ticker', 'Mkt-RF', 'SMB', 'HML', 'UMD']].merge(pf_tickers) m_beta = matrix(sens['Mkt-RF']) m_beta_smb = matrix(sens['SMB']) m_beta_hml = matrix(sens['HML']) m_beta_umd = matrix(sens['UMD']) m_cqa_beta = matrix(StockBeta.merge(pf_tickers)['cqaBeta']) longIndex = matrix(pf_tickers.merge(long_tickers, how='left').fillna(0)['bWeight']) mLongIndex = np.diag(pf_tickers.merge(long_tickers, how='left').fillna(0)['bWeight']) mLongIndex = matrix(mLongIndex[np.logical_or.reduce([np.sum(mLongIndex, 1) > 0.5])]).trans() shortIndex = -matrix(pf_tickers.merge(short_tickers, how='left').fillna(0)['bWeight']) mShortIndex = -np.diag(pf_tickers.merge(short_tickers, how='left').fillna(0)['bWeight']) mShortIndex = matrix(mShortIndex[np.logical_or.reduce([np.sum(mShortIndex, 1) > 0.5])]).trans() sector_index = pf_tickers[['ticker', 'sector']] sector_list = pf_tickers['sector'].unique() for sector in sector_list: sector_index.loc[:, sector] = 0.0 sector_index.ix[sector_index['sector'] == sector, sector] = 1.0 mSector_index = matrix(sector_index.iloc[:, 2:].as_matrix()) N = pf_tickers.shape[0] id = spmatrix(1.0, range(N), range(N)) wBounds = matrix(np.ones((N, 1)) * position_bound) longBounds = matrix(np.zeros((short_tickers.shape[0], 1))) shortBounds = matrix(np.zeros((long_tickers.shape[0], 1))) total_cov = matrix(functions.get_cov(pf_tickers, Coef, Res, CarhartDaily, back_date, start_date)) q = matrix(np.zeros((pf_tickers.shape[0], 1))) # three_factor_bound = 0.05 G = matrix([[mSector_index], [-mSector_index], [m_beta], [-m_beta], [m_beta_smb], [-m_beta_smb], [m_beta_hml], [-m_beta_hml], [m_beta_umd], [-m_beta_umd], [m_cqa_beta], [-m_cqa_beta], [longIndex], [-longIndex], [shortIndex], [-shortIndex], [id], [-id], [-mLongIndex], [mShortIndex]]).trans() h = matrix([sector_bound, sector_bound, sector_bound, sector_bound, sector_bound, sector_bound, sector_bound, sector_bound, sector_bound, sector_bound, sector_bound, sector_bound, sector_bound, sector_bound, sector_bound, sector_bound, 0.2, 0.4, three_factor_bound, three_factor_bound, three_factor_bound, three_factor_bound, three_factor_bound, three_factor_bound, beta_bound, beta_bound, 1, -0.98, -0.98, 1, wBounds, wBounds, longBounds, shortBounds]) sol = solvers.qp(total_cov, q, G, h) w_final = sol['x'] wTickers = pf_tickers wTickers['weight'] = w_final wTickers = wTickers[np.abs(wTickers['weight']) > trim] wTickers_properties = wTickers.merge(Coef).merge(StockBeta) stats = pd.DataFrame({'beta': np.dot(wTickers_properties['weight'], wTickers_properties['Mkt-RF']), 'beta_smb': np.dot(wTickers_properties['weight'], wTickers_properties['SMB']), 'beta_hml': np.dot(wTickers_properties['weight'], wTickers_properties['HML']), 'beta_umd': np.dot(wTickers_properties['weight'], wTickers_properties['UMD']), 'beta_cqa': np.dot(wTickers_properties['weight'], wTickers_properties['cqaBeta']), 'long weight': wTickers[wTickers['weight'] > 0]['weight'].sum(), 'short weight': wTickers[wTickers['weight'] < 0]['weight'].sum() }, index=[0]) for sector in sector_list: stats[sector] = wTickers_properties[wTickers_properties['sector'] == sector]['weight'].sum() stats = stats.transpose().reset_index() stats.columns = ['statistics', 'value'] return (stats, wTickers)
def pfOptimizer3(longTickers, shortTickers, Coef, Res, CarhartDaily, StockBeta, BETA_BOUND, WEIGHT_BOUND, back_date, start_date): longTickers['bWeight'] = 1 # binary weights shortTickers['bWeight'] = -1 pfTickers = pd.concat((shortTickers[['ticker']], longTickers[['ticker']]), axis=0).sort('ticker').reset_index(drop=True) sens = Coef[['ticker', 'Mkt-RF', 'SMB', 'HML', 'UMD']].merge(pfTickers) mBeta = matrix(sens['Mkt-RF']) # mBeta_smb = matrix(sens['SMB']) # mBeta_hml = matrix(sens['HML']) # mBeta_umd = matrix(sens['UMD']) mCqaBeta = matrix(StockBeta.merge(pfTickers)['cqaBeta']) longIndex = matrix( pfTickers.merge(longTickers, how='left').fillna(0)['bWeight']) mLongIndex = np.diag( pfTickers.merge(longTickers, how='left').fillna(0)['bWeight']) mLongIndex = matrix(mLongIndex[np.logical_or.reduce( [np.sum(mLongIndex, 1) > 0.5])]).trans() shortIndex = -matrix( pfTickers.merge(shortTickers, how='left').fillna(0)['bWeight']) mShortIndex = -np.diag( pfTickers.merge(shortTickers, how='left').fillna(0)['bWeight']) mShortIndex = matrix(mShortIndex[np.logical_or.reduce( [np.sum(mShortIndex, 1) > 0.5])]).trans() N = pfTickers.shape[0] id = spmatrix(1.0, range(N), range(N)) wBounds = matrix(np.ones((N, 1)) * WEIGHT_BOUND) longBounds = matrix(np.zeros((shortTickers.shape[0], 1))) shortBounds = matrix(np.zeros((longTickers.shape[0], 1))) total_cov = matrix( functions.get_cov(pfTickers, Coef, Res, CarhartDaily, back_date, start_date)) q = matrix(np.zeros((pfTickers.shape[0], 1))) G = matrix([ [mBeta], [-mBeta], # [mBeta_smb], [-mBeta_smb], # [mBeta_hml], [-mBeta_hml], # [mBeta_umd], [-mBeta_umd], [mCqaBeta], [-mCqaBeta], [longIndex], [-longIndex], [shortIndex], [-shortIndex], [id], [-id], [-mLongIndex], [mShortIndex] ]).trans() opt_comp = pd.DataFrame() max_slope = 0 for k in range(11): print 'k = %d' % k h = matrix([ (k + 1) * 0.05, -k * 0.05, # 0.1, 0.1, # 0.1, 0.1, # 0.1, 0.1, BETA_BOUND, BETA_BOUND, 1, -0.98, -0.98, 1, wBounds, wBounds, longBounds, shortBounds ]) # sol = solvers.lp(-mBeta, A, b) try: sol = solvers.qp(total_cov, q, G, h) except ValueError: break w_res = sol['x'] cov_res = sol['primal objective'] beta_res = np.float64(w_res.trans() * mBeta)[0, 0] beta_cqa_res = np.float64(w_res.trans() * mCqaBeta)[0, 0] # beta_smb_res = np.float64(w_res.trans() * mBeta_smb)[0,0] # beta_hml_res = np.float64(w_res.trans() * mBeta_hml)[0,0] # beta_umd_res = np.float64(w_res.trans() * mBeta_umd)[0,0] opt_comp = opt_comp.append( pd.DataFrame([{ 'beta': beta_res, 'beta_cqa': beta_cqa_res, # 'beta_smb': beta_smb_res, # 'beta_hml': beta_hml_res, # 'beta_umd': beta_umd_res, 'vol': math.sqrt(cov_res), }])) slope = beta_res / math.sqrt(cov_res) if slope > max_slope: w_final = w_res wTickers = pfTickers wTickers['weight'] = w_final return (opt_comp, wTickers)
def pf_optimizer(long_tickers, short_tickers, Coef, industry_data, CarhartDaily, StockBeta, beta_bound, position_bound, sector_bound, three_factor_bound, trim, back_date, start_date): long_tickers['bWeight'] = 1 # binary weights short_tickers['bWeight'] = -1 pf_tickers = pd.concat( (short_tickers[['ticker']], long_tickers[['ticker']]), axis=0).sort('ticker').reset_index(drop=True).merge( industry_data[['ticker', 'sector']]) sens = Coef[['ticker', 'Mkt-RF', 'SMB', 'HML', 'UMD']].merge(pf_tickers) m_beta = matrix(sens['Mkt-RF']) m_beta_smb = matrix(sens['SMB']) m_beta_hml = matrix(sens['HML']) m_beta_umd = matrix(sens['UMD']) m_cqa_beta = matrix(StockBeta.merge(pf_tickers)['cqaBeta']) longIndex = matrix( pf_tickers.merge(long_tickers, how='left').fillna(0)['bWeight']) mLongIndex = np.diag( pf_tickers.merge(long_tickers, how='left').fillna(0)['bWeight']) mLongIndex = matrix(mLongIndex[np.logical_or.reduce( [np.sum(mLongIndex, 1) > 0.5])]).trans() shortIndex = -matrix( pf_tickers.merge(short_tickers, how='left').fillna(0)['bWeight']) mShortIndex = -np.diag( pf_tickers.merge(short_tickers, how='left').fillna(0)['bWeight']) mShortIndex = matrix(mShortIndex[np.logical_or.reduce( [np.sum(mShortIndex, 1) > 0.5])]).trans() sector_index = pf_tickers[['ticker', 'sector']] sector_list = pf_tickers['sector'].unique() for sector in sector_list: sector_index.loc[:, sector] = 0.0 sector_index.ix[sector_index['sector'] == sector, sector] = 1.0 mSector_index = matrix(sector_index.iloc[:, 2:].as_matrix()) N = pf_tickers.shape[0] id = spmatrix(1.0, range(N), range(N)) wBounds = matrix(np.ones((N, 1)) * position_bound) longBounds = matrix(np.zeros((short_tickers.shape[0], 1))) shortBounds = matrix(np.zeros((long_tickers.shape[0], 1))) total_cov = matrix( functions.get_cov(pf_tickers, Coef, Res, CarhartDaily, back_date, start_date)) q = matrix(np.zeros((pf_tickers.shape[0], 1))) # three_factor_bound = 0.05 G = matrix([[mSector_index], [-mSector_index], [m_beta], [-m_beta], [m_beta_smb], [-m_beta_smb], [m_beta_hml], [-m_beta_hml], [m_beta_umd], [-m_beta_umd], [m_cqa_beta], [-m_cqa_beta], [longIndex], [-longIndex], [shortIndex], [-shortIndex], [id], [-id], [-mLongIndex], [mShortIndex]]).trans() h = matrix([ sector_bound, sector_bound, sector_bound, sector_bound, sector_bound, sector_bound, sector_bound, sector_bound, sector_bound, sector_bound, sector_bound, sector_bound, sector_bound, sector_bound, sector_bound, sector_bound, 0.2, 0.4, three_factor_bound, three_factor_bound, three_factor_bound, three_factor_bound, three_factor_bound, three_factor_bound, beta_bound, beta_bound, 1, -0.98, -0.98, 1, wBounds, wBounds, longBounds, shortBounds ]) sol = solvers.qp(total_cov, q, G, h) w_final = sol['x'] wTickers = pf_tickers wTickers['weight'] = w_final wTickers = wTickers[np.abs(wTickers['weight']) > trim] wTickers_properties = wTickers.merge(Coef).merge(StockBeta) stats = pd.DataFrame( { 'beta': np.dot(wTickers_properties['weight'], wTickers_properties['Mkt-RF']), 'beta_smb': np.dot(wTickers_properties['weight'], wTickers_properties['SMB']), 'beta_hml': np.dot(wTickers_properties['weight'], wTickers_properties['HML']), 'beta_umd': np.dot(wTickers_properties['weight'], wTickers_properties['UMD']), 'beta_cqa': np.dot(wTickers_properties['weight'], wTickers_properties['cqaBeta']), 'long weight': wTickers[wTickers['weight'] > 0]['weight'].sum(), 'short weight': wTickers[wTickers['weight'] < 0]['weight'].sum() }, index=[0]) for sector in sector_list: stats[sector] = wTickers_properties[wTickers_properties['sector'] == sector]['weight'].sum() stats = stats.transpose().reset_index() stats.columns = ['statistics', 'value'] return (stats, wTickers)
def pfOptimizer3(longTickers, shortTickers, Coef, Res, CarhartDaily, StockBeta, BETA_BOUND, WEIGHT_BOUND, back_date, start_date): longTickers['bWeight'] = 1 # binary weights shortTickers['bWeight'] = -1 pfTickers = pd.concat((shortTickers[['ticker']], longTickers[['ticker']]), axis=0).sort('ticker').reset_index(drop=True) sens = Coef[['ticker', 'Mkt-RF', 'SMB', 'HML', 'UMD']].merge(pfTickers) mBeta = matrix(sens['Mkt-RF']) # mBeta_smb = matrix(sens['SMB']) # mBeta_hml = matrix(sens['HML']) # mBeta_umd = matrix(sens['UMD']) mCqaBeta = matrix(StockBeta.merge(pfTickers)['cqaBeta']) longIndex = matrix(pfTickers.merge(longTickers, how='left').fillna(0)['bWeight']) mLongIndex = np.diag(pfTickers.merge(longTickers, how='left').fillna(0)['bWeight']) mLongIndex = matrix(mLongIndex[np.logical_or.reduce([np.sum(mLongIndex, 1) > 0.5])]).trans() shortIndex = -matrix(pfTickers.merge(shortTickers, how='left').fillna(0)['bWeight']) mShortIndex = -np.diag(pfTickers.merge(shortTickers, how='left').fillna(0)['bWeight']) mShortIndex = matrix(mShortIndex[np.logical_or.reduce([np.sum(mShortIndex, 1) > 0.5])]).trans() N = pfTickers.shape[0] id = spmatrix(1.0, range(N), range(N)) wBounds = matrix(np.ones((N, 1)) * WEIGHT_BOUND) longBounds = matrix(np.zeros((shortTickers.shape[0], 1))) shortBounds = matrix(np.zeros((longTickers.shape[0], 1))) total_cov = matrix(functions.get_cov(pfTickers, Coef, Res, CarhartDaily, back_date, start_date)) q = matrix(np.zeros((pfTickers.shape[0], 1))) G = matrix([[mBeta], [-mBeta], # [mBeta_smb], [-mBeta_smb], # [mBeta_hml], [-mBeta_hml], # [mBeta_umd], [-mBeta_umd], [mCqaBeta], [-mCqaBeta], [longIndex], [-longIndex], [shortIndex], [-shortIndex], [id], [-id], [-mLongIndex], [mShortIndex]]).trans() opt_comp = pd.DataFrame() max_slope = 0 for k in range(11): print 'k = %d' % k h = matrix([(k + 1) * 0.05, -k * 0.05, # 0.1, 0.1, # 0.1, 0.1, # 0.1, 0.1, BETA_BOUND, BETA_BOUND, 1, -0.98, -0.98, 1, wBounds, wBounds, longBounds, shortBounds]) # sol = solvers.lp(-mBeta, A, b) try: sol = solvers.qp(total_cov, q, G, h) except ValueError: break w_res = sol['x'] cov_res = sol['primal objective'] beta_res = np.float64(w_res.trans() * mBeta)[0, 0] beta_cqa_res = np.float64(w_res.trans() * mCqaBeta)[0, 0] # beta_smb_res = np.float64(w_res.trans() * mBeta_smb)[0,0] # beta_hml_res = np.float64(w_res.trans() * mBeta_hml)[0,0] # beta_umd_res = np.float64(w_res.trans() * mBeta_umd)[0,0] opt_comp = opt_comp.append(pd.DataFrame([{'beta': beta_res, 'beta_cqa': beta_cqa_res, # 'beta_smb': beta_smb_res, # 'beta_hml': beta_hml_res, # 'beta_umd': beta_umd_res, 'vol': math.sqrt(cov_res), }])) slope = beta_res / math.sqrt(cov_res) if slope > max_slope: w_final = w_res wTickers = pfTickers wTickers['weight'] = w_final return (opt_comp, wTickers)