def hist_expect_mu(conn,
code_list,
start='2016-01-01',
end='2017-01-01',
backfill=False,
stocks_price_old=None,
business_calendar=None,
industry=None,
**kwargs):
'''
calculate annualized expected return from historical return.
Returns
=======
emu: pandas.Series
expected return
'''
s = Stock(conn,
code_list,
start,
end,
backfill=backfill,
stocks_price_old=stocks_price_old,
business_calendar=business_calendar,
industry=industry)
rets = s.daily_returns
emu = rets.mean() * 252
return emu
def portfolio_construct_by_weight(conn,
start,
code_list,
weights=None,
name_list=None,
capital=1000000,
backfill=True,
stocks_price_old=None,
business_calendar=None,
industry=None):
if weights is None:
weights = np.array(len(code_list) * [1 / len(code_list)])
else:
weights = np.array(weights)
start = first_trading_day(conn, start, business_calendar=business_calendar)
stocks = Stock(conn,
list(code_list),
start=start,
end=start,
backfill=backfill,
stocks_price_old=stocks_price_old,
business_calendar=business_calendar,
industry=industry)
price = pd.melt(stocks.close_price, value_name='price')
shares = np.floor(weights * capital / price['price'] / 100) * 100
db = pd.DataFrame({'code': price['code'], 'shares': shares})
return db
def hist_expect(conn,
code_list,
start='2016-01-01',
end='2017-01-01',
backfill=False,
stocks_price_old=None,
business_calendar=None,
industry=None):
'''
calculate expected return and covariance matrix from historical return.
Returns
=======
emu: pandas.Series
expected return
esigma: pandas.DataFrame
expected covariance matrix
'''
s = Stock(conn,
code_list,
start,
end,
backfill=backfill,
stocks_price_old=stocks_price_old,
business_calendar=business_calendar,
industry=industry)
rets = s.daily_returns
emu = rets.mean() * 252
esigma = rets.cov() * 252
return emu, esigma
def update_balance(balance, conn, stock_list, start, end, backfill,
stocks_price_old, business_calendar, industry, weights, cap,
benchmark_old, rf):
#================== backtest ==========================
# calculate shares of each stock according to weights
if backfill == False:
mport = tool.portfolio_construct_by_weight(conn,
start.strftime('%Y-%m-%d'),
pd.Series(stock_list),
weights=weights,
capital=cap)
else:
stocks = Stock(conn,
stock_list,
start=start.strftime('%Y-%m-%d'),
end=end.strftime('%Y-%m-%d'),
backfill=backfill,
stocks_price_old=stocks_price_old,
business_calendar=business_calendar,
industry=industry)
date = stocks.start
stock_price = stocks.price[date].reset_index()
shares = np.floor(cap * weights / stock_price['close'] / 100) * 100
mport = pd.DataFrame({'code': stock_list, 'shares': shares})
# construct a portfolio according to mport
mk_port = Portfolio(conn,
mport,
start=start.strftime('%Y-%m-%d'),
end=end.strftime('%Y-%m-%d'),
backfill=backfill,
stocks_price_old=stocks_price_old,
business_calendar=business_calendar,
industry=industry,
benchmark_old=benchmark_old)
# update balance
daily_balance = mk_port.port_daily_balance()
if (1 - weights.sum()) > 1e-6:
cash_cap = cap - daily_balance[0]
cash_balance = [
cash_cap * np.exp(rf / 252 * i)
for i in range(1,
len(daily_balance) + 1)
]
cash_balance = pd.Series(cash_balance, index=daily_balance.index)
daily_balance = daily_balance + cash_balance
print(weights.sum())
else:
daily_balance = daily_balance + cap - daily_balance[0]
balance = balance.append(daily_balance)
return balance, mk_port, stocks
def arma_garch_prediction(conn,
code_list,
start1,
start2,
end2,
arma_order=[1, 0, 1],
garch_order=[1, 1]):
stock = Stock(conn, code_list, start=start2, end=end2)
ts_data = stock.daily_returns
correlation = ts_data[-250:].corr()
ts_data = ts_data[start1:]
model = ARMA_GARCH(order1=arma_order, order2=garch_order)
for i in range(len(code_list)):
return_data = ts_data.iloc[:, [i]].dropna()
try:
model.estimation(return_data)
model.prediction(30)
except ValueError:
continue
if i == 0:
pre_return_new = pd.DataFrame(
{ts_data.columns[i]: model.prediction_x})
pre_vol_new = pd.DataFrame(
{ts_data.columns[i]: model.prediction_conditional_vol})
else:
pre_return = pd.DataFrame({ts_data.columns[i]: model.prediction_x})
pre_return_new = pd.concat([pre_return_new, pre_return], axis=1)
pre_return_avg = pre_return_new.mean()
pre_vol = pd.DataFrame(
{ts_data.columns[i]: model.prediction_conditional_vol})
pre_vol_new = pd.concat([pre_vol_new, pre_vol], axis=1)
pre_vol_avg = pre_vol_new.mean()
list1 = list(pre_return_new.columns)
list2 = list(ts_data.columns)
remaining_list = np.setdiff1d(list2, list1)
ts_data_remain = ts_data[remaining_list]
remain_mean = ts_data_remain.mean()
remain_std = ts_data_remain.std()
ereturn = pre_return_avg.append(remain_mean)
evol = pre_vol_avg.append(remain_std)
new_order = np.array(correlation.columns)
evol = evol.reindex(new_order)
ereturn = ereturn.reindex(new_order)
corr = np.asmatrix(correlation)
evol = np.asmatrix(evol)
cov = pd.DataFrame(np.multiply((evol.T * evol), corr))
return ereturn * 252, cov * 252, ts_data_remain
def adding_stock_info():
while True:
print("请输入您需要监测的支股票信息")
code = input("请输入您想要检测的股票代码")
buy_point = float(input("请输入您需要的买点价格"))
sale_point = float(input("请输入您需要的卖点价格"))
#股票对象实例化,并将实例化之后的对象加入列表。
stock = Stock(code, buy_point, sale_point)
stock_list.append(stock)
print("如不需要监测其他的其他的股票请输入：n")
choice = input("请输入您的选择")
if choice == "n":
break
def hist_expect_sigma(conn,
code_list,
start='2016-01-01',
end='2017-01-01',
backfill=False,
stocks_price_old=None,
business_calendar=None,
industry=None,
**kwargs):
'''
calculate annualized covariance matrix from historical return.
Returns
=======
esigma: pandas.DataFrame
expected covariance matrix
'''
s = Stock(conn,
code_list,
start,
end,
backfill=False,
stocks_price_old=stocks_price_old,
business_calendar=business_calendar,
industry=industry)
rets = s.daily_returns
esigma = rets.cov() * 252
# if not enough close price data, covariance should not be calculated
temp = (~pd.isnull(rets)).sum()
idx = (temp[temp < 10].index).tolist()
if idx:
print(start)
print(end)
raise ValueError('Data less than 10 days for stocks {}!'.format(
','.join(idx)))
# deal with 0 correlations
# var = np.diag(esigma)
return esigma
def simu_alpha(conn, code_list, start='2016-01-01', end='2017-01-01',
backfill=True, stocks_price_old=None, business_calendar=None,
industry=None, **kwargs):
'''
calculate annualized expected return from historical return.
Returns
=======
emu: pandas.Series
expected return
'''
s = Stock(conn, code_list, start, end, backfill=backfill,
stocks_price_old=stocks_price_old,
business_calendar=business_calendar,
industry=industry)
rets = s.daily_returns
emu = rets.mean() * 252
if (emu==np.nan).any():
raise ValueError('There is return missing')
z = simu_res_alpha(emu, 0.1)
z = pd.Series(np.reshape(np.array(z), z.shape[1]).tolist(), index=rets.columns)
return z
def construct(self):
price = pd.DataFrame()
returns = pd.DataFrame()
industry = pd.DataFrame()
# for code in self._code_list['code']:
code_list = list(self._code_list['code'])
stock_data = Stock(self._conn,
code_list,
self._start,
self._end,
backfill=self._backfill,
stocks_price_old=self._stocks_price_old,
business_calendar=self._business_calendar,
industry=self._industry)
stock_price = stock_data.close_price
# stock_price.columns = [code]
stock_return = stock_data.daily_returns
stock_industry = stock_data.industry
# stock_return.columns = [code]
price = self.add_stock(price, stock_price)
returns = self.add_stock(returns, stock_return)
industry = self.add_stock(industry, stock_industry)
return price, returns, industry
#mkt_code = 'sh000300'
#stk_list = tool.get_stock_pool(conn,t)
#
#for day in bus_calender['date']:
# coeff=capm_modeling(conn=conn, t=day,frequency='daily',window=252,mkt_code='sh000001')
# betas = pd.concat([betas,coeff['code','end','beta']],axis=1)
c.execute("select date, close, code from index_price where code =='sh000001'")
mkt_price = pd.DataFrame(c.fetchall())
mkt_price.columns = ['Date','Close','Code']
start=mkt_price['Date'][0]
end=mkt_price['Date'].iloc[-1]
date_list=mkt_price['Date'][1:]
#date_list2=['2017-12-29','2017-12-28']
returns = Stock(conn,list(code_list),start,end,all_stocks=True).daily_returns
#date_temp = pd.to_datetime(start)
#date_temp = date_temp+pd.offsets.relativedelta(days=1)*252
#%%
sys.path.append('/Users/yunongwu/Nustore Files/PI/Code/Test/YN_Test/')
from capm import capm_modeling
#from capm import get_date_range
#coeff=capm_modeling(conn=conn, returns=returns,t='2000-01-05',frequency='daily',window=252,mkt_code='sh000001')
coeff_new=pd.DataFrame()
date_list=['2007-03-07']
for t in date_list:
coeff=capm_modeling(conn=conn, returns=returns,t=t,frequency='daily',window=252,mkt_code='sh000001')
coeff['end']=t
from portfolio_class import Portfolio
import matplotlib.pylab as plt
import numpy as np
import scipy.stats as st
conn = sql.connect(
'/Users/tianyulu/Nustore Files/PI/Staff Working File/D.Lu/data.db')
#%%
asofdate = '2017-12-01'
query = 'select code from stocks_price where date="%s"' % (asofdate)
stock_list = pd.read_sql(query, conn)
#%%
code_list = list(stock_list.iloc[0:100, 0])
stock = Stock(conn, code_list, start='2017-01-03')
# select one stock returns
ts_data = stock.daily_returns.iloc[:, 0:100]
# In[186]:
sys.path.append(
'/Users/tianyulu/Nustore Files/PI//code/working_on/Time Series')
import test_ts_dist
# Dickey-Fuller test on stationarity
test_df = test_ts_dist.test_stationarity(ts_data)
# Linjung-Box test on autocorrelation
test_ljq = test_ts_dist.ljungbox_test(ts_data)
sys.path.append(path)
dbpath = os.path.abspath('../../') + '\\Data\\data.db'
conn = sql.connect(dbpath)
from stock_class import Stock
from portfolio_class import Portfolio
from Portfolio_Optimization import mv
import Portfolio_Optimization.rolling as rolling
import stock_screener as sc
import general_tools as tool
import CAPM.capm as capm
from Portfolio_Optimization.methodclass import mu_method
#%% Demo Stock Class
code_list = ['000001', '000002']
s = Stock(conn, code_list, start='2017-01-05', end='2017-02-01')
s.code
s.price
s.close_price
s.daily_returns
s.daily_cum_returns
#%% Demo Portfolio Class
stocks_list = pd.DataFrame({
'code': ['000002', '000001'],
'shares': [1000, 100]
})
p = Portfolio(conn, stocks_list, start='2017-01-05', end='2017-02-01')
p.stock_price()
p.stock_returns()
p.weekly_returns()
def test_negative_stock_change(self):
stock = Stock()
with self.assertRaises(SystemExit):
stock.increment_stock('13', -4)
def capm_modeling(conn,
t,
frequency,
window,
stk_list=None,
riskfree=0.0,
mkt_code='sh000001',
stocks_price_old=None,
business_calendar=None,
industry=None):
"""
Stand at time t,
Run the CAPM model for a rolling period before time t (including t);
Parameters
----------
conn:
sqlite3 connection;
t:
string; a string of time in form of 'yyyy-mm-dd';
frequency:
string; a string of frequency for returns
i.e., 'daily','weekly','monthly','quarterly','annually';
code_list:
list; a list for stock code, default None;
if None, then the entire pool of stock as of time t would be considered
riskfree:
float; a floating number for risk free rate, default 0.0
window:
int; an integer for rolling window input, in the unite of frequency
i.e., if window = 2, frequency = 'weekly', it means 2 weeks
mkt_code:
string; a string for market index, default 'sh000300'
Returns
-------
coeficient matrices: pd.DataFrame
the matrix containing the information of
[code, available beginning date, end date, number of observation, alpha, beta, r_squared]
"""
# Get Business Calender to check if selected date is business day
converter = {
'daily': 1,
'weekly': 5,
'monthly': 21,
'quarterly': 63,
'annually': 252
}
since = time.time()
bus_calender = tool.get_business_calendar(conn)
if (bus_calender[bus_calender['date'] == t].empty):
raise ValueError('date ' + t + ' is not a business day!')
date_range = get_date_range(conn, t, window * converter[frequency],
'daily')
begin = date_range[0]
if stk_list == None:
stk_list = tool.get_stock_pool(conn, t)
data = Stock(conn, [],
begin.strftime('%Y-%m-%d'),
t,
all_stocks=True,
stocks_price_old=stocks_price_old,
business_calendar=business_calendar,
industry=industry).daily_returns
data = data[stk_list['code']]
else:
stk_list = pd.DataFrame(stk_list)
stk_list.columns = ['code']
data = Stock(conn,
list(stk_list['code'][0:500]),
begin.strftime('%Y-%m-%d'),
t,
stocks_price_old=stocks_price_old,
business_calendar=business_calendar,
industry=industry).daily_returns
index = 500
while index < (stk_list.index[~0] + 1):
index_end = np.minimum(index + 500, (stk_list.index[~0] + 1))
data = pd.merge(data,
Stock(conn,
list(stk_list['code'][index:index_end]),
begin.strftime('%Y-%m-%d'), t).daily_returns,
how='outer',
left_index=True,
right_index=True)
index = index_end
# get the time period before and including t
mkt = get_mkt_index(conn, t, window, frequency, mkt_code)[['mkt_ret']]
# Extract all stocks as of time t
# for i in range(1,len(date_range)):
# data.loc[(data.index>date_range[i-1]) & (data.index<=date_range[i]),'group']=date_range[i]
# data2 = pd.DataFrame(data.groupby('group').agg(lambda x:x.sum(skipna = False)))
# data2 = data2.sort_index()
# Calculate coeficients
data2 = convert_return(data, frequency)
results = (map(functools.partial(get_coef, data2, mkt, riskfree),
stk_list['code']))
model_coef = list(results)
time_lag = time.time() - since
cols = ['code', 'begin', 'end', 'number of obs', 'alpha', 'beta', 'r2']
model_coef = pd.DataFrame(model_coef, columns=cols)
print('Coefficient calculation completed in {:.0f}m {:.0f}s'.format(
time_lag // 60, time_lag % 60))
industry_query = 'select code,industry from stock_basics'
industry = pd.read_sql(industry_query, conn)
model_coef = pd.merge(model_coef,
industry,
left_on='code',
right_on='code',
how='left')
# return
model_coef_sub = model_coef[model_coef['number of obs'] > 50]
industry_avg_beta = pd.DataFrame(
model_coef_sub.groupby('industry')['beta'].apply(np.nanmean))
industry_avg_beta.columns = ['industry_avg_beta']
industry_avg_alpha = pd.DataFrame(
model_coef_sub.groupby('industry')['alpha'].apply(np.nanmean))
industry_avg_alpha.columns = ['industry_avg_alpha']
model_coef = pd.merge(model_coef,
industry_avg_beta,
left_on='industry',
right_index=True,
how='left')
model_coef = pd.merge(model_coef,
industry_avg_alpha,
left_on='industry',
right_index=True,
how='left')
model_coef.loc[model_coef['number of obs'] < np.ceil(window * 0.2),
'beta'] = model_coef[model_coef['number of obs'] < np.ceil(
window * 0.2)]['industry_avg_beta']
model_coef.loc[model_coef['number of obs'] < np.ceil(window * 0.2),
'alpha'] = model_coef[model_coef['number of obs'] < np.ceil(
window * 0.2)]['industry_avg_alpha']
return model_coef