def benchmark(symbol, sd, ed, sv):
""" Calculate benchmark strategy
Benchmark strategy:
Buy 1000 shares and hold, starting value 100000
Order DataFrame:
Date, Symbol, Order, Shares
2010-01-04, JPM, BUY, 1000
Params:
@symbol: the stock symbol to act on
@sd: Start date of trading period
@ed: End date of trading period
@sv: Starting value
Return:
@df_benchmark: Dataframe for benchmark strategy
"""
test_sd = sd
test_ed = ed
symbols = [symbol]
# get price data
df_prices = get_data(symbols, pd.date_range(test_sd, test_ed))
# print(df_prices)
start_date = min(df_prices.index)
# print(start_date)
orders = {
'Date': [start_date],
'Symbol': symbols,
'Order': ['BUY'],
'Shares': [1000]
}
df_orders = pd.DataFrame(orders)
# df_orders['Date'] = pd.to_datetime(df_orders['Date'], format="%Y-%m-%d")
df_orders.set_index('Date', inplace=True)
# print("df_orders")
# print(df_orders)
# print(df_orders.info())
# calcualte portfolio value based on trade orders (df_orders)
portvals = compute_portvals(df_orders=df_orders,
start_val=START_VAL,
commission=9.95,
impact=0.005,
start_date=sd,
end_date=ed)
# normalize the portvals
portvals = normalize(portvals)
cr, adr, sddr, sr = get_portfolio_stats(portvals)
return portvals, cr, adr, sddr, sr
def test_code():
start_date = OUT_SAMPLE_DATES[0]
end_date = OUT_SAMPLE_DATES[1]
df_trades = testPolicy(symbol=SYMBOL,
sd=start_date,
ed=end_date,
sv=START_VAL)
# print("df_trades")
# print(df_trades.head())
# print(df_trades.tail())
df_orders = convert_trades_to_order(df_trades)
# print("df_orders")
# print(df_orders)
short_entry_points = df_orders.index[df_orders['Order'] == 'SELL'].tolist()
# print("short_entry_points", short_entry_points)
long_entry_points = df_orders.index[df_orders['Order'] == 'BUY'].tolist()
# print("long_entry_points", long_entry_points)
# calcualte portfolio value based on trade orders (df_orders)
manual_strategy_portvals = compute_portvals(df_orders=df_orders,
start_val=START_VAL,
commission=9.95,
impact=0.005,
start_date=start_date,
end_date=end_date)
# normalize the portvals
manual_strategy_portvals = normalize(manual_strategy_portvals)
(cum_ret_manual_strategy, avg_daily_ret_manual_strategy,
std_daily_ret_manual_strategy, sharpe_ratio_manual_strategy
) = get_portfolio_stats(manual_strategy_portvals)
# Get stats for benchmark performance
(benchmark_portvals, benchmark_cum_ret, benchmark_avg_daily_ret,
benchmark_std_daily_ret,
benchmark_sharpe_ratio) = benchmark(symbol=SYMBOL,
sd=start_date,
ed=end_date,
sv=START_VAL)
print_compare_benchmark(start_date, end_date, benchmark_sharpe_ratio,
sharpe_ratio_manual_strategy, benchmark_cum_ret,
cum_ret_manual_strategy, benchmark_std_daily_ret,
std_daily_ret_manual_strategy,
benchmark_avg_daily_ret,
avg_daily_ret_manual_strategy, benchmark_portvals,
manual_strategy_portvals)
plot_compare(benchmark_portvals, manual_strategy_portvals,
long_entry_points, short_entry_points)
def test_code():
# Test in sample performance
prices = get_prices(symbol=SYMBOL,
sd=IN_SAMPLE_DATES[0],
ed=IN_SAMPLE_DATES[1])
# Strategy Learner
st_learner = sl.StrategyLearner(verbose=False, impact=IMPACT)
st_learner.addEvidence(symbol=SYMBOL,
sd=IN_SAMPLE_DATES[0],
ed=IN_SAMPLE_DATES[1],
sv=START_VAL)
# Test in sample performance
st_trades = st_learner.testPolicy(symbol=SYMBOL,
sd=IN_SAMPLE_DATES[0],
ed=IN_SAMPLE_DATES[1],
sv=START_VAL)
st_orders = st_learner.convert_trades_to_order(st_trades)
st_portval = sim.compute_portvals(st_orders,
start_val=START_VAL,
commission=TRANSACTION_COMMISSION,
impact=IMPACT,
start_date=IN_SAMPLE_DATES[0],
end_date=IN_SAMPLE_DATES[1])
st_portval = st_learner.normalize_df(st_portval)
st_cr, st_adr, st_sddr, st_sr = sim.get_portfolio_stats(st_portval)
print_performance("Strategy Learner", st_cr, st_adr, st_sddr, st_sr,
st_portval)
# Benchmark
ben_portvals, ben_cr, ben_adr, ben_sddr, ben_sr = ms.benchmark(
SYMBOL, IN_SAMPLE_DATES[0], IN_SAMPLE_DATES[1], START_VAL)
print_performance("Benchmark", ben_cr, ben_adr, ben_sddr, ben_sr,
ben_portvals)
# Manual strategy
ms_trades = ms.testPolicy(SYMBOL, IN_SAMPLE_DATES[0], IN_SAMPLE_DATES[1],
START_VAL)
ms_orders = ms.convert_trades_to_order(ms_trades)
ms_portvals = ms.compute_portvals(df_orders=ms_orders,
start_val=START_VAL,
commission=TRANSACTION_COMMISSION,
impact=IMPACT,
start_date=IN_SAMPLE_DATES[0],
end_date=IN_SAMPLE_DATES[1])
ms_portvals = ms.normalize(ms_portvals)
ms_cr, ms_adr, ms_sddr, ms_sr = ms.get_portfolio_stats(ms_portvals)
print_performance("Manual Strategy", ms_cr, ms_adr, ms_sddr, ms_sr,
ms_portvals)
plot_compare(ben_portvals, ms_portvals, st_portval)
def test_code():
start_date = OUT_SAMPLE_DATES[0]
end_date = OUT_SAMPLE_DATES[1]
df_trades = testPolicy(symbol=SYMBOL, sd=start_date, ed=end_date, sv=START_VAL)
# print("df_trades")
# print(df_trades.head())
# print(df_trades.tail())
df_orders = convert_trades_to_order(df_trades)
# print("df_orders")
# print(df_orders.head())
# print(df_orders.tail())
# calcualte portfolio value based on trade orders (df_orders)
optimal_strategy_portvals = compute_portvals(df_orders=df_orders, start_val=START_VAL, commission=0., impact=0.,
start_date=start_date, end_date=end_date)
# normalize the portvals
optimal_strategy_portvals = normalize(optimal_strategy_portvals)
(cum_ret_optimal_strategy, avg_daily_ret_optimal_strategy,
std_daily_ret_optimal_strategy, sharpe_ratio_optimal_strategy) = get_portfolio_stats(optimal_strategy_portvals)
# Get stats for benchmark performance
(benchmark_portvals, benchmark_cum_ret, benchmark_avg_daily_ret,
benchmark_std_daily_ret, benchmark_sharpe_ratio) = benchmark(symbol=SYMBOL,
sd=start_date,
ed=end_date,
sv=START_VAL)
print_compare_benchmark(start_date, end_date, benchmark_sharpe_ratio, sharpe_ratio_optimal_strategy,
benchmark_cum_ret, cum_ret_optimal_strategy, benchmark_std_daily_ret,
std_daily_ret_optimal_strategy, benchmark_avg_daily_ret, avg_daily_ret_optimal_strategy,
benchmark_portvals, optimal_strategy_portvals)
plot_compare(benchmark_portvals, optimal_strategy_portvals)
def check_performance(self, df_indicators, sd, ed, col_name='position'):
df_orders = self.get_orders_from_position(df_indicators,
col_name=col_name)
# print("df_orders")
# print(df_orders)
strategy_portvals = sim.compute_portvals(df_orders=df_orders,
start_val=START_VAL,
commission=0.0,
impact=self.impact,
start_date=sd,
end_date=ed)
# normalize the portvals
strategy_portvals = self.normalize_df(strategy_portvals)
(cum_ret, avg_daily_ret, std_daily_ret,
sharpe_ratio) = sim.get_portfolio_stats(strategy_portvals)
print("final strategy_portvals", strategy_portvals[-1])
print("cum_ret", cum_ret)
print("avg_daily_ret", avg_daily_ret)
print("std_daily_ret", std_daily_ret)
print("sharpe_ratio", sharpe_ratio)
def addEvidence(self, symbol = "IBM", \
sd=dt.datetime(2008,1,1), \
ed=dt.datetime(2009,1,1), \
sv = 10000,n_bins=6):
# this method should create a QLearner, and train it for trading
syms = [symbol]
dates = pd.date_range(sd, ed)
prices, prices_SPY = id.get_price(syms, dates)
if self.verbose: print prices
daily_returns = (prices / prices.shift(1)) - 1
daily_returns = daily_returns[1:]
# get indicators and combine them into as a feature data_frame
lookback = 14
_, PSR = id.get_SMA(prices, lookback)
_, _, bb_indicator = id.get_BB(prices, lookback)
momentum = id.get_momentum(prices, lookback)
_, self.pbins = pd.qcut(PSR, n_bins, labels=False, retbins=True)
_, self.bbins = pd.qcut(bb_indicator,
n_bins,
labels=False,
retbins=True)
_, self.mbins = pd.qcut(momentum, n_bins, labels=False, retbins=True)
self.pbins = self.pbins[1:-1]
self.bbins = self.bbins[1:-1]
self.mbins = self.mbins[1:-1]
# start training
converged = False
df_trades = None
count = 0
old_cum_ret = 0.0
converge_count = 0
converged_prev = False
#print "Total number of states is:", total_states
# Initialize QLearner,
self.learner = ql.QLearner(num_states=100 * n_bins,
num_actions=self.num_actions,
alpha=0.5,
gamma=0.9,
rar=0.0,
radr=0.0,
dyna=0,
verbose=self.verbose)
while (not converged) and (count < 100):
# Set first state to the first data point (first day)
indices = daily_returns.index
holdings = pd.DataFrame(np.nan,
index=indices,
columns=['Holdings'])
#first_state = self.indicators_to_state(PSR.iloc[0], bb_indicator.iloc[0], momentum.iloc[0])
#print("SL 152: holdings.iloc[0] = ", holdings.iloc[0][0], "; daily_rets.iloc[1] = ", daily_returns.iloc[1][0])
holdings.iloc[0] = 0.
#print("SL 153")
#df_prices = prices.copy()
#df_prices['Cash'] = pd.Series(1.0, index=indices)
#df_trades = df_prices.copy()
#df_trades[:] = 0.0
state = self.indicators_to_state(PSR.iloc[0], bb_indicator.iloc[0],
momentum.iloc[0])
action = self.learner.querysetstate(state)
holdings.iloc[0], reward = self.apply_action(
holdings.iloc[0][0], action, daily_returns.iloc[1][0])
#print("SL 171: PSR.shape[0] = ",PSR.shape[0],"; daily_returns.shape[0] = ",daily_returns.shape[0])
# Cycle through dates
for j in range(1, daily_returns.shape[0]):
state = self.indicators_to_state(PSR.iloc[j],
bb_indicator.iloc[j],
momentum.iloc[j])
# Get action by Query learner with current state and reward to get action
action = self.learner.query(state, reward)
# update reward and holdings with the new action.
holdings.iloc[j], reward = self.apply_action(
holdings.iloc[j - 1][0], action, daily_returns.iloc[j][0])
#print("SL 183: holdings.iloc[j][0] = ",holdings.iloc[j][0])
# Implement action returned by learner and update portfolio
#print("SL 206: one learning is done.")
#print("SL 215, holdings.iloc[0]",holdings.iloc[0])
holdings.iloc[-1] = 0
holdings.ffill(inplace=True)
holdings.fillna(0, inplace=True)
#print("SL 216 holdings = ",holdings)
trades = holdings.diff()
trades.iloc[0] = 0
# buy and sell happens when the difference change direction
df_trades = pd.DataFrame(data=trades.values,
index=indices,
columns=['Trades'])
df_orders, _ = generate_orders(df_trades, symbol)
port_vals = compute_portvals(df_orders,
sd=sd,
ed=ed,
impact=self.impact,
start_val=sv,
commission=self.commission)
cum_ret, _, _, _ = get_portfolio_stats(port_vals)
count += 1
old_cum_ret,converged_prev,converge_count,converged = \
check_convergence(old_cum_ret,cum_ret,converged_prev,converge_count)
# check if converge
#if converged:
#print("SL 212: converged at iteration # ",count, "cum_ret is: ", cum_ret)
return df_trades
# example use with new colname
volume_all = ut.get_data(syms, dates,
colname="Volume") # automatically adds SPY
volume = volume_all[syms] # only portfolio symbols
volume_SPY = volume_all['SPY'] # only SPY, for comparison later
if self.verbose: print volume
def plot_optimal_strategy():
tos = TheoreticallyOptimalStrategy()
start_date = dt.datetime(2008, 1, 1)
end_date = dt.datetime(2009, 12, 31)
# dates = pd.date_range(start_date, end_date)
symbol = 'JPM'
df_trades = tos.testPolicy(symbol=symbol,
sd=start_date,
ed=end_date,
sv=100000)
df_orders = df_trades.copy()
df_orders = df_orders.loc[(df_orders.Trades != 0)]
#df_orders = df_trades[['Trades']][df_trades['Trades'] != 0]
df_orders['Symbol'] = symbol
df_orders['Order'] = np.where(df_orders['Trades'] > 0, 'BUY', 'SELL')
df_orders['Shares'] = np.abs(df_orders['Trades'])
port_vals = compute_portvals(df_orders,
start_val=100000,
commission=0.0,
impact=0.0)
benchmark_orders = pd.DataFrame(
data={
'Symbol': ["JPM", "JPM"],
'Order': ["BUY", "BUY"],
'Shares': [1000, 0]
},
index={df_trades.index.min(),
df_trades.index.max()})
#benchmark_orders.loc[benchmark_orders.index[1], 'Shares'] = 0
benchmark_vals = compute_portvals(benchmark_orders,
start_val=100000,
commission=0.0,
impact=0.0)
normed_port = port_vals / port_vals.ix[0]
normed_bench = benchmark_vals / benchmark_vals.ix[0]
"""fig = plt.figure(figsize=(12,6.5))
ax1 = fig.add_subplot(111)
normed_port.plot(ax=ax1, color='red', lw=2)
normed_bench.plot(ax=ax1, color='green', lw=1.2)
ax1.set_ylabel('Normalized Portfolio Value')
ax1.set_xlabel('Date')
plt.grid(True)
plt.legend()
plt.title('Theoretically Optimal Strategy (%s)' % symbol)
#plt.show()
plt.savefig('04_TOS.png')
"""
plt.figure(figsize=(12, 6.5))
plt.plot(normed_port, label="Portfolio", color='red', lw=2)
plt.plot(normed_bench, label="Benchmark", color='green', lw=1.2)
plt.xlabel('Date')
plt.ylabel('Normalized Value')
plt.legend()
plt.grid(True)
plt.title('Theoretically Optimal Strategy (%s)' % symbol)
plt.savefig('04_TOS.png')
plt.close()
port_cr, port_adr, port_stddr, port_sr = get_portfolio_stats(port_vals)
bench_cr, bench_adr, bench_stddr, bench_sr = get_portfolio_stats(
benchmark_vals)
# Compare portfolio against benchmark
print "=== Theoretically Optimal Strategy (TOS) ==="
print "Date Range: {} to {}".format(start_date, end_date)
print
print "Sharpe Ratio of TOS: {}".format(port_sr)
print "Sharpe Ratio of BenchMark : {}".format(bench_sr)
print
print "Cumulative Return of TOS: {}".format(port_cr)
print "Cumulative Return of Benchmark : {}".format(bench_cr)
print
print "Standard Deviation of TOS: {}".format(port_stddr)
print "Standard Deviation of Benchmark : {}".format(bench_stddr)
print
print "Average Daily Return of TOS: {}".format(port_adr)
print "Average Daily Return of BenchMark : {}".format(bench_adr)
print
print "Final TOS Portfolio Value: {}".format(port_vals[-1])
print "Final Benchmark Portfolio Value: {}".format(benchmark_vals[-1])
print
def plot_manual_strategy():
print "MS 86......"
ms = ManualStrategy()
commission = 9.95
impact = 0.005
# in sample
start_date = dt.datetime(2008, 1, 1)
end_date = dt.datetime(2009, 12, 31)
# dates = pd.date_range(start_date, end_date)
symbol = 'JPM'
df_trades = ms.testPolicy(symbol=symbol,
sd=start_date,
ed=end_date,
sv=100000)
# generate orders based on trades
print "MS 99.........."
df_orders, benchmark_orders = generate_orders(df_trades, symbol)
print "MS 101.........."
port_vals = compute_portvals(df_orders,
start_val=100000,
commission=commission,
impact=impact)
#benchmark_orders.loc[benchmark_orders.index[1], 'Shares'] = 0
benchmark_vals = compute_portvals(benchmark_orders,
start_val=100000,
commission=commission,
impact=impact)
normed_port = port_vals / port_vals.ix[0]
normed_bench = benchmark_vals / benchmark_vals.ix[0]
dates = pd.date_range(start_date, end_date)
prices_all = get_data([symbol], dates, addSPY=True, colname='Adj Close')
prices = prices_all[symbol] # only portfolio symbols
# get indicators
lookback = 14
_, PSR = id.get_SMA(prices, lookback)
_, _, bb_indicator = id.get_BB(prices, lookback)
momentum = id.get_momentum(prices, lookback)
# figure 5.
plt.figure(figsize=(12, 6.5))
top = plt.subplot2grid((5, 1), (0, 0), rowspan=3, colspan=1)
bottom = plt.subplot2grid((5, 1), (3, 0), rowspan=2, colspan=1, sharex=top)
# plot the Long or short action
for index, marks in df_trades.iterrows():
if marks['Trades'] > 0:
plt.axvline(x=index, color='blue', linestyle='dashed', alpha=.9)
elif marks['Trades'] < 0:
plt.axvline(x=index, color='black', linestyle='dashed', alpha=.9)
else:
pass
top.xaxis_date()
top.grid(True)
top.plot(normed_port, lw=2, color='red', label='Manual Strategy')
top.plot(normed_bench, lw=1.2, color='green', label='Benchmark')
top.set_title('Portfolio V.S Benchmark - In Sample Analysis')
top.set_ylabel('Normalized Value')
for index, marks in df_trades.iterrows():
if marks['Trades'] > 0:
top.axvline(x=index, color='blue', linestyle='dashed', alpha=.9)
elif marks['Trades'] < 0:
top.axvline(x=index, color='black', linestyle='dashed', alpha=.9)
else:
pass
bottom.plot(momentum, color='olive', lw=1, label="momentum")
bottom.plot(PSR, color='purple', lw=1, label="PSR")
#bottom.plot(bb_indicator, color='blue', lw=1, label="Bollinger")
bottom.set_title('Indicators')
bottom.axhline(y=-0.2, color='grey', linestyle='--', alpha=0.5)
bottom.axhline(y=0, color='grey', linestyle='--', alpha=0.5)
bottom.axhline(y=0.2, color='grey', linestyle='--', alpha=0.5)
bottom.legend()
top.legend()
top.axes.get_xaxis().set_visible(False)
plt.xlim(start_date, end_date)
filename = '05_MS_insample.png'
plt.savefig(filename)
plt.close()
port_cr, port_adr, port_stddr, port_sr = get_portfolio_stats(port_vals)
bench_cr, bench_adr, bench_stddr, bench_sr = get_portfolio_stats(
benchmark_vals)
# Compare portfolio against benchmark
print "=== Manual Strategy (MS) In Sample ==="
print "Date Range: {} to {}".format(start_date, end_date)
print
print "Sharpe Ratio of MS: {}".format(port_sr)
print "Sharpe Ratio of BenchMark : {}".format(bench_sr)
print
print "Cumulative Return of MS: {}".format(port_cr)
print "Cumulative Return of Benchmark : {}".format(bench_cr)
print
print "Standard Deviation of MS: {}".format(port_stddr)
print "Standard Deviation of Benchmark : {}".format(bench_stddr)
print
print "Average Daily Return of MS: {}".format(port_adr)
print "Average Daily Return of BenchMark : {}".format(bench_adr)
print
print "Final MS Portfolio Value: {}".format(port_vals[-1])
print "Final Benchmark Portfolio Value: {}".format(benchmark_vals[-1])
print
# ========================
# OUT OF SAMPLE Analysis
# ========================
start_date = dt.datetime(2010, 1, 1)
end_date = dt.datetime(2011, 12, 31)
# dates = pd.date_range(start_date, end_date)
symbol = 'JPM'
df_trades = ms.testPolicy(symbol=symbol,
sd=start_date,
ed=end_date,
sv=100000)
# generate orders based on trades
df_orders, benchmark_orders = generate_orders(df_trades, symbol)
port_vals = compute_portvals(df_orders,
start_val=100000,
commission=commission,
impact=impact)
#benchmark_orders.loc[benchmark_orders.index[1], 'Shares'] = 0
benchmark_vals = compute_portvals(benchmark_orders,
start_val=100000,
commission=commission,
impact=impact)
normed_port = port_vals / port_vals.ix[0]
normed_bench = benchmark_vals / benchmark_vals.ix[0]
dates = pd.date_range(start_date, end_date)
prices_all = get_data([symbol], dates, addSPY=True, colname='Adj Close')
prices = prices_all[symbol] # only portfolio symbols
# get indicators
lookback = 14
_, PSR = id.get_SMA(prices, lookback)
_, _, bb_indicator = id.get_BB(prices, lookback)
momentum = id.get_momentum(prices, lookback)
# figure 6.
plt.figure(figsize=(12, 6.5))
top = plt.subplot2grid((5, 1), (0, 0), rowspan=3, colspan=1)
bottom = plt.subplot2grid((5, 1), (3, 0), rowspan=2, colspan=1, sharex=top)
# plot the
for index, marks in df_trades.iterrows():
if marks['Trades'] > 0:
plt.axvline(x=index, color='blue', linestyle='dashed', alpha=.9)
elif marks['Trades'] < 0:
plt.axvline(x=index, color='black', linestyle='dashed', alpha=.9)
else:
pass
top.xaxis_date()
top.grid(True)
top.plot(normed_port, lw=2, color='red', label='Manual Strategy')
top.plot(normed_bench, lw=1.2, color='green', label='Benchmark')
# plot the Long or short action
for index, marks in df_trades.iterrows():
if marks['Trades'] > 0:
top.axvline(x=index, color='blue', linestyle='dashed', alpha=.9)
elif marks['Trades'] < 0:
top.axvline(x=index, color='black', linestyle='dashed', alpha=.9)
else:
pass
top.set_title('Portfolio V.S Benchmark - Out Sample Analysis')
top.set_ylabel('Normalized Value')
bottom.plot(momentum, color='olive', lw=1, label="momentum")
bottom.plot(PSR, color='purple', lw=1, label="PSR")
#bottom.plot(bb_indicator, color='blue', lw=1, label="Bollinger")
bottom.set_title('Indicators')
bottom.axhline(y=-0.2, color='grey', linestyle='--', alpha=0.5)
bottom.axhline(y=0, color='grey', linestyle='--', alpha=0.5)
bottom.axhline(y=0.2, color='grey', linestyle='--', alpha=0.5)
bottom.legend()
top.legend()
top.axes.get_xaxis().set_visible(False)
plt.xlim(start_date, end_date)
filename = '06_MS_OutSample.png'
plt.savefig(filename)
plt.close()
port_cr, port_adr, port_stddr, port_sr = get_portfolio_stats(port_vals)
bench_cr, bench_adr, bench_stddr, bench_sr = get_portfolio_stats(
benchmark_vals)
# Compare portfolio against benchmark
print "=== Manual Strategy (MS) Out Sample ==="
print "Date Range: {} to {}".format(start_date, end_date)
print
print "Sharpe Ratio of MS: {}".format(port_sr)
print "Sharpe Ratio of BenchMark : {}".format(bench_sr)
print
print "Cumulative Return of MS: {}".format(port_cr)
print "Cumulative Return of Benchmark : {}".format(bench_cr)
print
print "Standard Deviation of MS: {}".format(port_stddr)
print "Standard Deviation of Benchmark : {}".format(bench_stddr)
print
print "Average Daily Return of MS: {}".format(port_adr)
print "Average Daily Return of BenchMark : {}".format(bench_adr)
print
print "Final MS Portfolio Value: {}".format(port_vals[-1])
print "Final Benchmark Portfolio Value: {}".format(benchmark_vals[-1])
print