def get_indicators(self,
symbol="JPM",
sd=dt.datetime(2008, 1, 1),
ed=dt.datetime(2009, 1, 1)):
df_p = ind.prepare_pricedf(symbol, sd, ed)
ind.rolling_avg(df_p, symbol, df_p)
ind.bollinger_bands(df_p, symbol, df_p)
ind.momentum(df_p, symbol, df_p)
ind.aroon(df_p, symbol, df_p)
df_p.fillna(method='ffill', inplace=True)
df_p.fillna(method='backfill', inplace=True)
return df_p
def _compute_indicators(prices, highs, lows, volumes, lookback):
mtm = momentum(prices, lookback)
sma, sma_ratio = simple_moving_average(prices, lookback)
bbands = bollinger_bands(prices, sma, lookback)
mfi = money_flow_index(prices, highs, lows, volumes, lookback)
return mtm, sma_ratio, bbands, mfi
def _compute_indicators(self):
price = self._stock_data.price
high = self._stock_data.high
low = self._stock_data.low
volume = self._stock_data.volume
mtm = momentum(price, self._lookback)
sma, sma_ratio = simple_moving_average(price, self._lookback)
bbands = bollinger_bands(price, sma, self._lookback)
mfi = money_flow_index(price, high, low, volume, self._lookback)
self._indicators = self._discretize((mtm, sma_ratio, bbands, mfi))
def getIndicators(self, dates, ed, syms, window=14, lookback=3):
# I can look back at historical data before
# the actual start date of the portfolio as
# long as I only use that for technical analysis
lookback_dates = pd.date_range(dates[0] - (window * 2), ed)
prices_w_lookback = ut.get_data(syms,
lookback_dates,
addSPY=True,
colname='Adj Close')[syms]
bbp = ind.bollinger_percentage(prices_w_lookback, window)
sma = ind.sma(prices_w_lookback, window)
momentum = ind.momentum(prices_w_lookback, lookback)
return bbp, sma, momentum
def __init__(self, sym='SPY', sd=dt.datetime(2008,1, 1), \
ed=dt.datetime(2009,12,31),sv=100000,verbose=False):
self.sym = sym
self.verbose = verbose
self.shares = 0
self.position = 0
self.sv = sv
self.cash = sv
self.pp = 0
#period
n = 28
chart = False
#take in, normalize data
df = ut.get_data([sym], pd.date_range(sd - dt.timedelta(100), ed))[sym]
normed = df / df.ix[0]
#get features
sma = ind.SMA(normed, n)
smap = ind.standardize(normed / sma)
bbp = ind.standardize(ind.bband(normed, n, chart)[0])
stdev = ind.standardize(normed.rolling(window=n, min_periods=n).std())
mom = ind.standardize(ind.momentum(normed, n, chart))
momvol = ind.standardize(mom / stdev)
#daily returns & combined features
rets = pd.DataFrame(df)
daily_rets = rets[sym].copy()
daily_rets[1:] = (rets[sym].ix[1:] / rets[sym].ix[:-1].values) - 1
daily_rets.ix[0] = 0
#print df
#df = pd.DataFrame(df).assign(normed = normed).assign(smap=smap).\
# assign(stdev=stdev).assign(bbp=bbp).assign(mom=mom)[sd:]
df = pd.DataFrame(df).assign(normed = normed).assign(smap=smap).\
assign(bbp=bbp).assign(mom=mom)[sd:]
#print df
daily_rets.ix[0] = 0
df = df.assign(dr=daily_rets)
#print df
self.df = df
self.market = df.iterrows()
self.curr = self.market.next()
self.action = self.Action()
#self.features = ['smap', 'stdev', 'bbp', 'mom']
self.features = ['smap', 'bbp', 'mom']
def testPolicy(symbol='AAPL',
sd=dt.datetime(2010, 1, 1),
ed=dt.datetime(2011, 12, 31),
sv=100000):
df_prices = get_data([symbol], pd.date_range(sd, ed))
df_prices = df_prices[[symbol]]
df_trades = pd.DataFrame(0.0, index=df_prices.index, columns=[symbol])
#Calculate indicator dataframes
upper_band, lower_band = ind.BollingerBands(df_prices)
momentum = ind.momentum(df_prices)
midpoint = ind.midpoints(df_prices)
pos = 0.0
for i in range(1, df_trades.shape[0] - 1):
price = df_prices[symbol].iloc[i]
lastprice = df_prices[symbol].iloc[i - 1]
#Signal for entering short position
if price < upper_band[symbol].iloc[i] and lastprice > upper_band[
symbol].iloc[i]:
df_trades[symbol].iloc[i] = -1000.0 - pos
#Signal for entering long position
elif price > lower_band[symbol].iloc[i] and lastprice < lower_band[
symbol].iloc[i]:
df_trades[symbol].iloc[i] = 1000.0 - pos
#Signals for closing out positions
if pos == 1000.0:
if price < midpoint[symbol].iloc[i] and lastprice > midpoint[
symbol].iloc[i] and momentum[symbol].iloc[i] <= 0:
df_trades[symbol].iloc[i] = -1 * pos
elif pos == -1000.0:
if price > midpoint[symbol].iloc[i] and lastprice < midpoint[
symbol].iloc[i] and momentum[symbol].iloc[i] >= 0:
df_trades[symbol].iloc[i] = -1 * pos
pos += df_trades[symbol].iloc[i]
return df_trades
def testPolicy(symbol = "AAPL", sd=dt.datetime(2010,1,1), \
ed = dt.datetime(2011,12,31), sv=100000):
chart = False
n = 14
#symbol = [symbol]
prices = get_data(symbol, pd.date_range(sd, ed))
#print prices
#ffill and drop SPY
prices.fillna(method='ffill', inplace=True)
prices.fillna(method='bfill', inplace=True)
prices = prices[symbol]
#Generate indicators
sma = ind.SMA(prices, n)
smap = ind.standardize(prices / sma)
bbp = ind.standardize(ind.bband(prices, n, chart)[0])
stdev = ind.standardize(prices.rolling(window=n, min_periods=n).std())
mom = ind.standardize(ind.momentum(prices, n, chart))
momvol = ind.standardize(mom / stdev)
orders = prices.copy()
orders.ix[:, :] = np.nan
smap_cross = pd.DataFrame(0, index=smap.index, columns=smap.columns)
smap_cross[smap >= 1] = 1
smap_cross[1:] = smap_cross.diff()
smap_cross.ix[0] = 0
orders[(smap < 0.95) & (bbp < 0) & (stdev < 0.1) & (mom < 0)] = 1000
orders[(smap > 1.05) & (bbp > 1) & (stdev > 0.3) & (mom > 0)] = -1000
orders[(smap_cross != 0)] = 0
orders.ffill(inplace=True)
orders.fillna(0, inplace=True)
orders[1:] = orders.diff()
#orders.ix[0] = 1000
#orders.ix[-1] = -1000
#orders = orders.loc[(orders!=0).any(axis=1)]
#orders.ix[0] = 0
#orders.ix[-1] = orders.ix[-2]*-1
#print orders
return orders
def testPolicy(self,
symbol="JPM",
sd=dt.datetime(2008, 1, 1),
ed=dt.datetime(2009, 12, 31),
sv=100000):
dates = pd.date_range(sd, ed)
syms = []
syms.append(symbol)
prices_SPY = get_data(syms, dates, addSPY=True, colname='Adj Close')
prices_sym = prices_SPY[syms]
init_trade = prices_sym.copy()
init_trade[:] = 0
# benchmark - buy and hold 1000 shares
init_trade.iloc[0, 0] = 1000
benchmark_portfolio = compute_portvals(init_trade,
start_val=100000,
commission=0.0,
impact=0.0)
# manual strategy
man_trade = prices_sym.copy()
man_trade[:] = 0
window = 14
lookback = 3
# I can look back at historical data before
# the actual start date of the portfolio as
# long as I only use that for technical analysis
lookback_dates = pd.date_range(dates[0] - (window * 2), ed)
prices_w_lookback = get_data(syms,
lookback_dates,
addSPY=True,
colname='Adj Close')[syms]
bbp = ind.bollinger_percentage(prices_w_lookback, window)
sma = ind.sma(prices_w_lookback, window)
momentum = ind.momentum(prices_w_lookback, lookback)
# One DF for iterating
indicators = pd.concat([bbp, sma, momentum], axis=1)
indicators.columns = ['BBP', 'SMA', 'Momentum']
indicators = indicators.dropna() # Just drop any day where we have NAN
man_trade = init_trade.copy()
man_trade.iloc[:, :] = 0
man_trade.iloc[:, -1] = 0.0
# track long and short positions
longPos = []
shortPos = []
position = 0
for i in range(indicators.shape[0] - 1):
# dropna removes some dates from indicators, can't rely on index
date = indicators.iloc[i].name.strftime('%Y-%m-%d')
if (indicators.iloc[i]['BBP'] <
.03) and (indicators.iloc[i]['Momentum'] >
-0.2) and (indicators.iloc[i]['SMA'] < .97):
#BUY
if position != 1000:
man_trade.loc[date][syms] = 1000 - position
position = 1000
longPos.append(date)
elif (indicators.iloc[i]['BBP'] >
.97) and (indicators.iloc[i]['Momentum'] <
0.2) and (indicators.iloc[i]['SMA'] > 1.03):
#SELL
if position != -1000:
man_trade.loc[date][syms] = -1000 - position
position = -1000
shortPos.append(date)
man_portfolio = compute_portvals(man_trade,
start_val=100000,
commission=self.commission,
impact=self.impact)
# Strategy vs Benchmark
fig = plt.figure()
plt.title('Manual Strategy')
x = benchmark_portfolio.index.values # dates
y = benchmark_portfolio.values
y = y / y[0] # normalize
plt.plot(x, y, color='green', label='Benchmark')
x2 = man_portfolio.index.values
y2 = man_portfolio.values
y2 = y2 / y2[0]
plt.plot(x2, y2, color='red', label='Manual Strategy')
plt.tight_layout()
plt.legend()
plt.savefig('MS_' + str(sd.year) + '.png')
# Entry points
plt.clf()
fig = plt.figure
plt.title('Entry Points')
x = prices_sym.index.values
y = prices_sym.values / prices_sym.values[0]
plt.plot(x, y, color='orange', label='Prices')
# plot longs
for l in longPos:
y = prices_sym.loc[l][0] / prices_sym.iloc[0, 0]
plt.plot([l, l], [y + .1, y - .1], color='blue')
for s in shortPos:
y = prices_sym.loc[s][0] / prices_sym.iloc[0, 0]
plt.plot([s, s], [y + .1, y - .1], color='black')
plt.grid()
# custom legend entries for long and short
ax = plt.gca()
handles, labels = ax.get_legend_handles_labels()
long_line = mlines.Line2D([], [], color='blue', label='Long')
short_line = mlines.Line2D([], [], color='black', label='Short')
handles.append(long_line)
handles.append(short_line)
plt.legend(handles=handles)
plt.tight_layout()
plt.savefig('entrypoints_MS_' + str(sd.year) + '.png')
# Stats
daily_returns_bench = benchmark_portfolio / benchmark_portfolio.shift(
1) - 1
c_return_bench = benchmark_portfolio.iloc[
-1] / benchmark_portfolio.iloc[0] - 1
stdev_daily_bench = daily_returns_bench.std()
mean_daily_bench = daily_returns_bench.mean()
daily_returns_opt = man_portfolio / man_portfolio.shift(1) - 1
c_return_opt = man_portfolio.iloc[-1] / man_portfolio.iloc[0] - 1
stdev_daily_opt = daily_returns_opt.std()
mean_daily_opt = daily_returns_opt.mean()
print('Benchmark Stats')
print('Cumulative Return: ' + str(c_return_bench))
print('StDev Daily: ' + str(stdev_daily_bench))
print('Mean Daily: ' + str(mean_daily_bench))
print('Manual Stats')
print('Cumulative Return: ' + str(c_return_opt))
print('StDev Daily: ' + str(stdev_daily_opt))
print('Mean Daily: ' + str(mean_daily_opt))
def addEvidence(self, symbol = "IBM", \
sd=dt.datetime(2008,1,1), \
ed=dt.datetime(2009,1,1), \
sv = 10000):
# add your code to do learning here
# example usage of the old backward compatible util function
syms = [symbol]
dates = pd.date_range(sd, ed)
prices_all = ut.get_data(syms, dates) # automatically adds SPY
prices = prices_all[syms] # only portfolio symbols
prices_SPY = prices_all['SPY'] # only SPY, for comparison later
#if self.verbose: print prices
#print prices
# example use with new colname
volume_all = ut.get_data(syms, dates,
colname="Volume") # automatically adds SPY
volume = volume_all[syms] # only portfolio symbols
volume_SPY = volume_all['SPY'] # only SPY, for comparison later
if self.verbose: print volume
#print volume
S = sma(prices, 20)
upper, lower, bb, r_m, r_std = bollinger(prices)
M = momentum(prices)
SMA = pd.DataFrame({'SMA': S})
bb_val = pd.DataFrame({'Bollinger': bb})
Upper = pd.DataFrame({'Upper': upper})
Lower = pd.DataFrame({'Lower': lower})
Momentum = pd.DataFrame({'Momentum': M})
R_M = pd.DataFrame({'Rolling Mean': r_m})
R_STD = pd.DataFrame({"Rolling STD": r_std})
ind = pd.concat((SMA, bb_val, Upper, Lower, R_M, R_STD, Momentum),
axis=1)
ind.fillna(0, inplace=True)
ind = ind[:-self.Days]
x_train = ind.values
'''
for i in range(0,len(prices)-self.Days):
if i<20:
x_train[i][0] = 0
x_train[i][1] = 0
x_train[i][2] = 0
x_train[i][3] = prices.iloc[i]
x_train[i][4] = prices.iloc[i + self.Days]
else:
x_train[i][0] = SMA.iloc[i]
x_train[i][1] = bb_val.iloc[i]
x_train[i][2] = Momentum.iloc[i]
x_train[i][3]=prices.iloc[i]
x_train[i][4]=prices.iloc[i+self.Days]
'''
#print x_train
y_temp = []
for i in range(0, len(prices) - self.Days):
if prices.ix[i + self.Days,
symbol] / prices.ix[i, symbol] > 1.008 + self.impact:
y_temp.append(1)
elif prices.ix[i + self.Days, symbol] / prices.ix[
i, symbol] < 0.992 - self.impact:
y_temp.append(-1)
else:
y_temp.append(0)
y_train = np.array(y_temp)
self.learner.addEvidence(x_train, y_train)
def testPolicy(self, symbol = "IBM", \
sd=dt.datetime(2009,1,1), \
ed=dt.datetime(2010,1,1), \
sv = 10000):
# here we build a fake set of trades
# your code should return the same sort of data
dates = pd.date_range(sd, ed)
prices_all = ut.get_data([symbol], dates) # automatically adds SPY
prices = prices_all[[
symbol,
]] # only portfolio symbols
trades_SPY = prices_all['SPY'] # only SPY, for comparison later
#print prices
S = sma(prices, 20)
upper, lower, bb, r_m, r_std = bollinger(prices)
M = momentum(prices)
SMA = pd.DataFrame({'SMA': S})
bb_val = pd.DataFrame({'Bollinger': bb})
Upper = pd.DataFrame({'Upper': upper})
Lower = pd.DataFrame({'Lower': lower})
Momentum = pd.DataFrame({'Momentum': M})
R_M = pd.DataFrame({'Rolling Mean': r_m})
R_STD = pd.DataFrame({"Rolling STD": r_std})
ind = pd.concat((SMA, bb_val, Upper, Lower, R_M, R_STD, Momentum),
axis=1)
ind.fillna(0, inplace=True)
ind = ind[:-self.Days]
x_test = ind.values
'''
x_test = np.zeros(shape=(len(prices) - self.Days, 3))
for i in range(0, len(prices) - self.Days):
if i<20:
x_test[i][0] = 0
x_test[i][1] = 0
x_test[i][2] = 0
else:
x_test[i][0] = SMA.iloc[i]
x_test[i][1] = bb_val.iloc[i]
x_test[i][2] = Momentum.iloc[i]
'''
#print x_test
y_ans = self.learner.query(x_test)
#print(y_ans)
trades = pd.DataFrame(0, columns=prices.columns, index=prices.index)
shares = 0
for i in range(0, len(prices) - self.Days):
if y_ans[i] == 1:
trades[symbol].iloc[i] = 1000 - shares
shares = 1000
elif y_ans[i] == -1:
trades[symbol].iloc[i] = -shares - 1000
shares = -1000
#print trades
'''
trades.values[:,:] = 0 # set them all to nothing
trades.values[0,:] = 1000 # add a BUY at the start
trades.values[40,:] = -1000 # add a SELL
trades.values[41,:] = 1000 # add a BUY
trades.values[60,:] = -2000 # go short from long
trades.values[61,:] = 2000 # go long from short
trades.values[-1,:] = -1000 #exit on the last day
if self.verbose: print type(trades) # it better be a DataFrame!
if self.verbose: print trades
if self.verbose: print prices_all
'''
return trades
def __init__(self, sym='SPY', sd=dt.datetime(2008,1, 1), \
ed=dt.datetime(2009,12,31),sv=100000,verbose=False, experiment=False, impact=0.0):
self.sym = sym
self.verbose = verbose
self.shares = 0
self.position = 2
self.sv = sv
self.cash = sv
self.pp = 0
self.impact = impact
self.experiment = experiment
#n = 3 #period
chart = False
#take in, normalize data
df = ut.get_data([sym], pd.date_range(sd - dt.timedelta(100), ed))[sym]
#print df
#print sd, ed
normed = df / df.ix[0]
#print normed
#daily returns & combined features
rets = pd.DataFrame(df)
daily_rets = rets[sym].copy()
daily_rets[1:] = (rets[sym].ix[1:] / rets[sym].ix[:-1].values) - 1
daily_rets.ix[0] = 0
#Simple moving average
sma5 = normed.rolling(5).mean()
sma15 = normed.rolling(15).mean()
sma20 = normed.rolling(20).mean()
sma25 = normed.rolling(25).mean()
sma30 = normed.rolling(30).mean()
sma40 = normed.rolling(40).mean()
#print sma5
# Volatility
vol5 = normed.rolling(5).std()
vol10 = normed.rolling(10).std()
vol20 = normed.rolling(20).std()
vol30 = normed.rolling(30).std()
# Bollinger bands
sma_bb = normed.rolling(5).mean()
sma_bb_std = normed.rolling(5).std()
bb = (normed -
(sma_bb - 2 * sma_bb_std)) / ((sma_bb + 2 * sma_bb_std) -
(sma_bb - 2 * sma_bb_std))
# Moving average convergence/divergence
#ema12 = pd.ewma(np.asarray(normed), span=12)
#ema26 = pd.ewma(np.asarray(normed), span=26)
#macd = ema12 - ema26
# Momentum
momentum2 = normed / normed.shift(2) - 1
momentum5 = normed / normed.shift(5) - 1
momentum10 = normed / normed.shift(10) - 1
# Combine into new dataframe
df = pd.DataFrame(df).assign(sma5=normed / sma5 - 1).assign(\
bb=bb).assign(momentum2=momentum2).assign(normed=normed).\
assign(vol10=vol10).assign(vol20=vol20).assign(vol30=vol30).assign(vol10=vol10)\
.assign(vol5=vol5).assign(\
momentum5=momentum5).assign(momentum10=momentum10)[sd:]
df = df.assign(dr=daily_rets)
#print df
# Determine optimal features for states
corr_df = df.corr().abs()
corr = corr_df['dr'][:]
corr.ix[0] = 0
corr['normed'] = 0
icorr = np.asarray(corr)
scorr = icorr.argsort(
)[-4:][::-1] # select top 3 features and daily returns
scorr = scorr[1:] # remove daily returns from possible features
optimal_ftrs = []
for i in scorr:
optimal_ftrs.append(corr_df.columns[i])
if experiment:
#print "Experiment 1 ACTIVATED"
n = 14
sma = ind.SMA(normed, n)
smap = ind.standardize(normed / sma)
bbp = ind.standardize(ind.bband(normed, n, chart)[0])
stdev = ind.standardize(
normed.rolling(window=n, min_periods=n).std())
mom = ind.standardize(ind.momentum(normed, n, chart))
momvol = ind.standardize(mom / stdev)
df = pd.DataFrame(df).assign(normed=normed).assign(
smap=smap).assign(stdev=stdev).assign(bbp=bbp).assign(
mom=mom)[sd:]
optimal_ftrs = ['smap', 'stdev', 'bbp', 'mom']
#df = pd.DataFrame(df).assign(normed = normed).assign(smap=smap).assign(bbp=bbp).assign(mom=mom)[sd:]
#df = pd.DataFrame(df).assign(normed = normed)[sd:]
self.df = df
self.market = df.iterrows()
self.curr = self.market.next()
self.action = self.Action()
self.features = optimal_ftrs
def combined_ind(df,
symbol="JPM",
sd=dt.datetime(2008, 1, 1),
ed=dt.datetime(2009, 12, 31),
window=10):
mome10 = momentum(df, symbol, window=10)[1]
SMA20, div20 = SMA(df, symbol, window=20)[1:]
bb_ind20 = bol_bands(df, symbol, window=20)[3]
#print mome10
mome10 = mome10 / mome10.iloc[10]
div20 = div20 / div20.iloc[19]
bb_ind20 = bb_ind20 / bb_ind20.iloc[19]
#print momentum1
short_entries = []
short_exits = []
long_entries = []
long_exits = []
signals = []
entry = False
for i in range(len(df.index)):
if mome10[i - 1] > 3.0 and mome10[i] < 2.8 and div20[
i] < 1.5 and bb_ind20[i] < 1.0 and not entry:
short_entries.append(df.index[i])
entry = True
signals.append([str(df.index[i]).split()[0], 'BUY'])
if mome10[
i -
1] < -2.0 and div20[i] > 0.5 and bb_ind20[i] > -1.0 and entry:
short_exits.append(df.index[i])
entry = False
signals.append([str(df.index[i]).split()[0], 'SELL'])
if mome10[i - 1] < -6.0 and mome10[i] > -6.0 and div20[
i] < 1.5 and bb_ind20[i] < 1.0 and not entry:
long_entries.append(df.index[i])
entry = True
signals.append([str(df.index[i]).split()[0], 'BUY'])
if mome10[i - 1] > 5.0 and mome10[
i -
1] < 5.0 and div20[i] > 0.5 and bb_ind20[i] > -1.0 and entry:
long_exits.append(df.index[i])
entry = False
signals.append([str(df.index[i]).split()[0], 'SELL'])
ordersfile = open(
'combined_indicator_order_{}_{}.csv'.format(
str(sd).split()[0],
str(ed).split()[0]), 'w')
ordersfile.write("Date,Symbol,Order,Shares\n")
for signal in signals:
ordersfile.write("%s,%s,%s,1000\n" % (signal[0], symbol, signal[1]))
ordersfile.close()
df[symbol] = df[symbol] / df[symbol].iloc[0]
ax = df[symbol].plot(title='Combined_ind', label=symbol)
mome10.plot(label='Mome10', ax=ax)
div20.plot(label='Div20', ax=ax)
bb_ind20.plot(label='bb_ind20', ax=ax)
ymin, ymax = ax.get_ylim()
plt.vlines(long_entries, ymin, ymax, color='g')
plt.vlines(long_exits, ymin, ymax)
plt.vlines(short_entries, ymin, ymax, color='r')
plt.vlines(short_exits, ymin, ymax)
ax.legend(loc='lower right')
plt.savefig('combined_ind_{}_{}.png'.format(
str(sd).split()[0],
str(ed).split()[0]))
plt.close('all')
#plt.show()
return long_entries, long_exits, short_entries, short_exits
