def testPolicy(self,symbol = "JPM", sd=dt.datetime(2008,1,1), ed=dt.datetime(2009,12,31), sv = 100000):
# Lookback window
lookback = 14
# Get the date range
dates = pd.date_range(sd, ed)
# Get the symbol in a list
symbols = []
symbols.append(symbol)
symbol_SPY = []
symbol_SPY.append("SPY")
# Get all the prices of the symbols
price = get_data(symbols, dates) # automatically adds SPY
# Forward fill and Backward fill
price = price.fillna(method='ffill')
price = price.fillna(method='bfill')
price_symbol = price[symbols] # only portfolio symbols
# Calculate the Technical Indicators (Price/SMA ratio, Bollinger Band Percentage, Stochastic Oscillator)
sma = ind.calculate_price_SMA_ratio(price_symbol, symbols, lookback)
bbp = ind.calculate_bbp(price_symbol, symbols, lookback)
momentum = ind.calculate_momentum(price, symbols, lookback)
# Create a binnary array showing when price is above SMA
sma_cross = pd.DataFrame(0, index=sma.index, columns=sma.columns)
sma_cross[sma >= 1] = 1
# Turn that array into one that only shows the crossings (-1 is cross down, +1 is cross up)
sma_cross[1:] = sma_cross.diff()
sma_cross.ix[0] = 0
# df_trades starts as a NAN array
df_trades = price_symbol.copy()
df_trades.ix[:,:] = np.NaN
# Now calculate the df_trades
# Stock may be oversold, BUY
df_trades[(sma < 0.95) & (bbp < 0) & (momentum < -0.05) ] = 1000
# Stock may be overbought, SELL
df_trades[(sma > 1.05) & (bbp > 1) & (momentum > 0.05) ] = -1000
# Apply exit order conditions
df_trades[(sma_cross != 0)] = 0
# All other days with NaN mean hold whatever you have, do nothing.
# Forward fill NaNs with previous values, then fill remaining NaNs with 0
df_trades.ffill(inplace=True)
df_trades.fillna(0, inplace=True)
# Now take the diff, which will give us an order to place only when the target shares changed.
df_trades[1:] = df_trades.diff()
df_trades.ix[0] = 0
# Drop all rows with non-zero values (no orders)
df_trades = df_trades.loc[(df_trades != 0).any(axis=1)]
# Now we have only the days that have orders.
return df_trades
def addEvidence(self, symbol = "IBM", \
sd=dt.datetime(2008,1,1), \
ed=dt.datetime(2009,1,1), \
sv = 10000):
# Get the past data to lookback and calculate the indicators
sd_lookback = sd - dt.timedelta(days=self.lookback+15)
ed_nday = ed + dt.timedelta(days=self.ndays+10)
# Create training data
syms=[symbol]
dates = pd.date_range(sd_lookback, ed_nday)
prices_all = ut.get_data(syms, dates) # automatically adds SPY
prices = prices_all[syms] # only portfolio symbols
prices_SPY = prices_all['SPY'] # only SPY, for comparison later
# Forward fill and Backward fill
prices = prices.fillna(method='ffill')
prices = prices.fillna(method='bfill')
# Calculate the Technical Indicators (Price/SMA ratio, Bollinger Band Percentage, Stochastic Oscillator)
sma = ind.calculate_price_SMA_ratio(prices, syms, self.lookback)
bbp = ind.calculate_bbp(prices, syms, self.lookback)
momentum = ind.calculate_momentum(prices, syms, self.lookback)
#Slice the indicators between the actual start date & end date
sma = sma.loc[sd:ed]
bbp = bbp.loc[sd:ed]
momentum = momentum.loc[sd:ed]
# Join all the indicators to form train X data
trainX = np.concatenate((sma,bbp,momentum),axis=1)
# Calculate N day returns as N day returns = Price[today+Ndays]/Price[today] - 1.0
nday_ret = (prices.shift(-self.ndays)/prices) - 1.0
# Slice the nday returns between the actual start & end dates
nday_ret = nday_ret.loc[sd:ed]
# Let us get the prices of the actual start date & end date
prices_actual = prices.loc[sd:ed]
YBUY = 0.02
YSELL = -0.02
# Construct train Y data
trainY = prices_actual.copy()
trainY = trainY.values * 0
# Go LONG if returns > threshold
trainY[nday_ret > YBUY+self.impact] = +1
# Go SHORT if returns < threshold
trainY[nday_ret < YSELL-self.impact] = -1
# Convert trainY into a 1D numpy array
trainY = np.hstack(trainY)
# Train the BagLearner with data
self.learner.addEvidence(trainX, trainY)
def addEvidence(self, symbol="IBM", \
sd=dt.datetime(2008, 1, 1), \
ed=dt.datetime(2009, 1, 1), \
sv=10000):
# add your code to do learning here
# example usage of the old backward compatible util function
syms = [symbol]
dates = pd.date_range(sd, ed)
prices_all = ut.get_data(syms, dates) # automatically adds SPY
prices = prices_all[syms] # only portfolio symbols
prices_SPY = prices_all['SPY'] # only SPY, for comparison later
if self.verbose: print(prices)
# Get Indicators used from Project 6 - Manual Strategy (indicators.py)
sma = ind.calculate_sma(prices)
bb_upper, bb_lower, bb_ratio = ind.calculate_bb(prices)
window = 10
momentum = ind.calculate_momentum(prices, window)
indicators = pd.concat((sma, bb_ratio, momentum), axis=1)
indicators.columns = ['SMA', 'BBR', 'MOM']
indicators.fillna(0, inplace=True)
indicators = indicators[:-5]
trainX = indicators.values
# Create the Training Set for Y Values based on the indicators
# We use market variance as 2% and use N = 5 days
N = 5
YBUY = 0.02 + self.impact
YSELL = -0.02 - self.impact
prices = prices / prices.iloc[0]
trainY = np.zeros(prices.shape[0] - N) # Normalize Prices
for t in range(prices.shape[0] - N):
# ret = (price[t+N]/price[t]) - 1.0
ret = (prices.ix[t + N, symbol] / prices.ix[t, symbol]) - 1.0
if ret > YBUY:
trainY[t] = 1 # LONG
elif ret < YSELL:
trainY[t] = -1 # SHORT
else:
trainY[t] = 0 # CASH
# Feed our bag learner the training data to learn a strategy
self.learner.addEvidence(trainX, trainY)
def get_indicators_df(self, prices):
N = self.N
# Calculate indicators
sma, sma_ratio = calculate_sma(prices, N)
mtm = calculate_momentum(prices, N)
top_band, bot_band, bbp = calculate_bbp(prices, N)
x_train = np.zeros((len(prices) - N, 3))
bbp_flat = bbp.as_matrix().flatten()[N:]
sma_flat = sma_ratio.as_matrix().flatten()[N:]
mtm_flat = mtm.as_matrix().flatten()[N:]
x_train[:, 0] = bbp_flat
x_train[:, 1] = sma_flat
x_train[:, 2] = mtm_flat
# Change Nans to zeros
where_nan = np.isnan(x_train)
x_train[where_nan] = 0
return x_train
def testPolicy(self,
sym='JPM',
sd=dt.datetime(2008, 1, 1),
ed=dt.datetime(2009, 12, 31),
sv=100000):
lookback = 20
shares = 1000
dates = pd.date_range(sd, ed)
price = get_data([sym], dates).drop(['SPY'], axis=1)
price = price.fillna(method='ffill')
price = price.fillna(method='bfill')
# debugg
pd.set_option('display.max_rows', 1000)
sma, sma_ratio = calculate_sma(price, lookback)
mtm = calculate_momentum(price, lookback)
top_band, bot_band, bbp = calculate_bbp(price, lookback)
orders1 = []
for day in range(lookback + 1, price.shape[0]):
if (sma_ratio.iloc[day][sym] < 0.95) and (
bbp.iloc[day][sym] < 0) and (mtm.iloc[day][sym] < -0.095):
orders1.append([price.index[day].date(), sym, 'BUY', shares])
elif sma_ratio.iloc[day][sym] > 1.05 and bbp.iloc[day][
sym] > 1 and (mtm.iloc[day][sym] > 0.095):
orders1.append([price.index[day].date(), sym, 'SELL', shares])
elif (mtm.iloc[day][sym] > 0.095) & (mtm.iloc[day][sym] > -0.095):
orders1.append([price.index[day].date(), sym, 'BUY', 0])
df_trades = pd.DataFrame(orders1,
columns=['Date', 'Symbol', 'Order', 'Shares'])
return df_trades
def testPolicy(self, symbol="IBM", \
sd=dt.datetime(2009, 1, 1), \
ed=dt.datetime(2010, 1, 1), \
sv=10000):
syms = [symbol]
dates = pd.date_range(sd, ed)
prices_all = ut.get_data(syms, dates) # automatically adds SPY
prices = prices_all[syms] # only portfolio symbols
# prices_SPY = prices_all['SPY'] # only SPY, for comparison later
# Get Indicators used from Project 6 - Manual Strategy (indicators.py)
sma = ind.calculate_sma(prices)
bb_upper, bb_lower, bb_ratio = ind.calculate_bb(prices)
window = 10
momentum = ind.calculate_momentum(prices, window)
indicators = pd.concat((sma, bb_ratio, momentum), axis=1)
indicators.columns = ['SMA', 'BBR', 'MOM']
indicators.fillna(0, inplace=True)
indicators = indicators[:-5]
testX = indicators.values
# Based on the trained learner, get the predicted Y values
testY = self.learner.query(testX)
# Make the dataframe for trades
df_trades = pd.DataFrame(index=prices_all.index, columns=[symbol])
df_trades.loc[:] = 0
current_holding = 0
for i in range(testY.shape[0]):
# Buy the stock i.e. LONG
if testY[i] > 0:
if current_holding == 0:
df_trades.values[i, :] = 1000
current_holding += 1000
elif current_holding == -1000:
df_trades.values[i, :] = 2000
current_holding += 2000
elif current_holding == 1000:
df_trades.values[i, :] = 0
# Sell the stock i.e. SHORT
elif testY[i] < 0:
if current_holding == 0:
df_trades.values[i, :] = -1000
current_holding -= 1000
elif current_holding == 1000:
df_trades.values[i, :] = -2000
current_holding -= 2000
elif current_holding == -1000:
df_trades.values[i, :] = 0
# HOLD the Stock
elif testY[i] == 0:
if current_holding == 1000:
df_trades.values[i, :] = -1000
current_holding -= 1000
elif current_holding == -1000:
df_trades.values[i, :] = 1000
current_holding += 1000
elif current_holding == 0:
df_trades.values[i, :] = 0
if current_holding == -1000:
df_trades.values[prices.shape[0] - 1, :] = 1000
elif current_holding == 1000:
df_trades.values[prices.shape[0] - 1, :] = -1000
if self.verbose: print(type(df_trades)) # it better be a DataFrame!
if self.verbose: print(df_trades)
if self.verbose: print(prices_all)
return df_trades
def testPolicy(symbol="JPM", sd=dt.datetime(2008, 1, 1), ed=dt.datetime(2009, 12, 31), sv=100000):
# Get the prices for SPY and JPM
dates = pd.date_range(sd, ed)
window = 10
prices_all = util.get_data([symbol], dates)
prices_all = prices_all / prices_all.iloc[0] # Normalize the prices
prices_SPY = prices_all['SPY']
prices_JPM = prices_all['JPM']
# Get the indicators
sma = ind.calculate_sma(prices_JPM)
upper_band, lower_band, bb_ratio = ind.calculate_bb(prices_JPM)
momentum = ind.calculate_momentum(prices_JPM, window=10)
# Set thresholds for the indicators
sma_threshold = (0.93, 1.08) # (Low, High)
bbratio_threshold = (-1.0, 1.0)
momentum_threshold = (-0.3, 0.3)
# Make the dataframe for trades
df_trades = pd.DataFrame(index=prices_SPY.index, columns=[symbol])
current_holding = 0
date_last = None
for date in prices_SPY.index:
# For SMA, BB and Momentum, our window is 10 days
# so we need to make no trades until we have the indicators established
if date_last is None or date <= sd + dt.timedelta(days=window):
df_trades.loc[date] = 0
date_last = date
continue
# Sell the stock i.e. SHORT
if (sma.loc[date] > sma_threshold[1] and momentum.loc[date] > momentum_threshold[1])\
or (bb_ratio.loc[date] > bbratio_threshold[1]):
if current_holding == 0:
df_trades.loc[date] = -1000
current_holding -= 1000
elif current_holding == 1000:
df_trades.loc[date] = -2000
current_holding -= 2000
elif current_holding == -1000:
df_trades.loc[date] = 0
# Buy the stock i.e. LONG
elif (sma.loc[date] < sma_threshold[0] and momentum.loc[date] < momentum_threshold[0])\
or (bb_ratio.loc[date] < bbratio_threshold[0]):
if current_holding == 0:
df_trades.loc[date] = 1000
current_holding += 1000
elif current_holding == -1000:
df_trades.loc[date] = 2000
current_holding += 2000
elif current_holding == 1000:
df_trades.loc[date] = 0
else:
df_trades.loc[date] = 0
date_last = date
return df_trades
def testPolicy(self, symbol = "IBM", \
sd=dt.datetime(2009,1,1), \
ed=dt.datetime(2010,1,1), \
sv = 10000):
# Get data for previous days to lookback and calculate the indicators
sd_lookback = sd - dt.timedelta(days=self.lookback+15)
# Get data for next days to calculate N day returns
ed_nday = ed + dt.timedelta(days=self.ndays+10)
# Get the prices
syms = [symbol]
dates = pd.date_range(sd_lookback, ed_nday)
prices_all = ut.get_data(syms, dates) # automatically adds SPY
prices = prices_all[syms] # only portfolio symbols
# Forward fill and Backward fill
prices = prices.fillna(method='ffill')
prices = prices.fillna(method='bfill')
# Calculate the Technical Indicators (Price/SMA ratio, Bollinger Band Percentage, Stochastic Oscillator)
sma = ind.calculate_price_SMA_ratio(prices, syms, self.lookback)
bbp = ind.calculate_bbp(prices, syms, self.lookback)
momentum = ind.calculate_momentum(prices, syms, self.lookback)
#Slice the indicators between the actual start date & end date
sma = sma.loc[sd:ed]
bbp = bbp.loc[sd:ed]
momentum = momentum.loc[sd:ed]
# Join all the indicators to form test X data
testX = np.concatenate((sma,bbp,momentum),axis=1)
# Query the Baglearner to get the trade predictions
predY = self.learner.query(testX) # get the predictions
# Let us get the prices of the actual start date & end date
prices_actual = prices.loc[sd:ed]
# Create the trades dataframe
trades = prices_actual.copy() # only portfolio symbols
trades_SPY = prices_all['SPY'] # only SPY, for comparison later
# Clear out the Dataframe so we can accumulate values into it
for day in range(trades.shape[0]):
trades.ix[day] = 0.0
# Holdings (At any given day, the holdings should be only be either -1000, 0 or 1000)
holdings = 0
# Iterate through the dataframe to add trades
for day in range(trades.shape[0]):
# Go LONG if +1
if (predY[day] > 0) and (holdings < 1000):
# BUY 1000 shares first
trades.ix[day] = 1000.0
holdings = holdings + 1000
# Even then, if the holdings sums upto only 0, BUY a 1000 more shares
if (holdings == 0):
trades.ix[day] = trades.ix[day] + 1000
holdings = holdings + 1000
# Go SHORT if -1
elif (predY[day] < 0) and (holdings > -1000):
# SELL 1000 shares first
trades.ix[day] = -1000.0
holdings = holdings - 1000
# Even then, if the holdings comes to only 0, SELL a 1000 more shares
if (holdings == 0):
trades.ix[day] = trades.ix[day] - 1000
holdings = holdings - 1000
"""
trades.values[:,:] = 0 # set them all to nothing
trades.values[0,:] = 1000 # add a BUY at the start
trades.values[40,:] = -1000 # add a SELL
trades.values[41,:] = 1000 # add a BUY
trades.values[60,:] = -2000 # go short from long
trades.values[61,:] = 2000 # go long from short
trades.values[-1,:] = -1000 #exit on the last day
"""
if self.verbose: print type(trades) # it better be a DataFrame!
if self.verbose: print trades
if self.verbose: print prices_all
return trades