def testPolicy(self,
symbol="JPM",
sd=dt.datetime(2008, 1, 1),
ed=dt.datetime(2009, 12, 31),
sv=100000):
symbols = [symbol]
stock_price, sma, p_to_sma = indicators.price_sma(start_date=sd,
end_date=ed,
symbols=symbols)
rsi = indicators.rsi(start_date=sd, end_date=ed, symbols=symbols)
mfi = indicators.mfi(start_date=sd, end_date=ed, symbols=symbols)
vix = indicators.vix(start_date=sd, end_date=ed, symbols=symbols)
portfolio = stock_price.copy()
portfolio.iloc[:, :] = np.nan
# print '======begining portfolio====\n', portfolio
# print portfolio
curr_state = 'neutral'
long_entry = []
long_exit = []
short_entry = []
short_exit = []
for i in range(len(portfolio)):
# print curr_state
if (rsi.iloc[i, 0] < 30) & (p_to_sma.iloc[i, 0] < 1) & (
curr_state != 'long') & (vix.iloc[i, 0] < 50):
portfolio.iloc[i, 0] = 1000
curr_state = 'long'
long_entry.append(portfolio.iloc[i:].index[0])
elif (mfi.iloc[i, 0] > 70) & (p_to_sma.iloc[i, 0] > 1) & (
curr_state != 'short') & (vix.iloc[i, 0] > 20):
portfolio.iloc[i, 0] = -1000
curr_state = 'short'
short_entry.append(portfolio.iloc[i:].index[0])
if (rsi.iloc[i, 0] >= 55) & (curr_state == 'long'):
portfolio.iloc[i, 0] = 0
curr_state = 'neutral'
long_exit.append(portfolio.iloc[i:].index[0])
elif (mfi.iloc[i, 0] <= 45) & (curr_state == 'short'):
portfolio.iloc[i, 0] = 0
curr_state = 'neutral'
short_exit.append(portfolio.iloc[i:].index[0])
portfolio.fillna(method='ffill', inplace=True)
portfolio.fillna(0, inplace=True)
# print '======end portfolio====\n', portfolio
df_trade = portfolio - portfolio.shift(1)
df_trade.iloc[0] = portfolio.iloc[0]
# print '=====df_ttrade=====\n', df_trade
return df_trade, long_entry, long_exit, short_entry, short_exit
def addEvidence(self, symbol="IBM", \
sd=dt.datetime(2008, 1, 1), \
ed=dt.datetime(2009, 1, 1), \
sv=10000):
# stock_older_days = 30
symbols = [symbol]
prices, sma, p_to_sma = indicators.price_sma(start_date=sd, end_date=ed, symbols=symbols)
prices.columns = ['Price']
sma.columns = ['sma']
p_to_sma.columns = ['p/sma']
rsi = indicators.rsi(start_date=sd, end_date=ed, symbols=symbols)
mfi = indicators.mfi(start_date=sd, end_date=ed, symbols=symbols)
vix = indicators.vix(start_date=sd, end_date=ed, symbols=symbols)
rsi.columns = ['RSI']
mfi.columns = ['MFI']
vix.columns = ['VIX']
portfolio = prices.copy()
portfolio.iloc[:, :] = np.nan
Xtrain = pd.concat([p_to_sma, rsi, mfi, vix], axis=1)
Xtrain.fillna(value=0, inplace=True)
# print '=========Xtrain=======\n'
# print Xtrain
date_index = Xtrain.index
trainX = Xtrain.values[:, 1:]
# print '=========trainX=======\n'
# print trainX
self.N = 7
nday_rt = prices.shift(-self.N) / prices - 1.0
pd.set_option('display.max_rows', 1000)
# print 'n days return: '
# print nday_rt
# print
nday_rt[nday_rt > 0] = prices.shift(-self.N) / (prices * (1.0 + self.impact)) - 1.0
nday_rt[nday_rt < 0] = prices.shift(-self.N) / (prices * (1.0 - self.impact)) - 1.0
# print 'n days return: '
# print nday_rt
# print
yvalues = nday_rt.copy()
yvalues.iloc[:, :] = 0
# go long when return is larger than 3%, go short when return is lower than -3%
threshold = 0.03
yvalues[nday_rt >= threshold] = 1
yvalues[nday_rt <= -threshold] = -1
# print xvalues == 0
yvalues[trainX == 0] = 0
trainY = np.array(yvalues)
self.learner.addEvidence(trainX, trainY)
def addEvidence(self, symbol="IBM", \
sd=dt.datetime(2008, 1, 1), \
ed=dt.datetime(2009, 1, 1), \
sv=10000):
# stock_older_days = 30
symbols = [symbol]
prices, sma, p_to_sma = indicators.price_sma(start_date=sd,
end_date=ed,
symbols=symbols)
prices.columns = ['Price']
sma.columns = ['sma']
p_to_sma.columns = ['p/sma']
rsi = indicators.rsi(start_date=sd, end_date=ed, symbols=symbols)
mfi = indicators.mfi(start_date=sd, end_date=ed, symbols=symbols)
vix = indicators.vix(start_date=sd, end_date=ed, symbols=symbols)
rsi.columns = ['RSI']
mfi.columns = ['MFI']
vix.columns = ['VIX']
portfolio = prices.copy()
portfolio.iloc[:, :] = np.nan
Xtrain = pd.concat([p_to_sma, rsi, mfi, vix], axis=1)
Xtrain.fillna(value=0, inplace=True)
# print '=========Xtrain=======\n'
# print Xtrain
date_index = Xtrain.index
Xtrain = Xtrain.values[:, 1:]
# print '=========trainX=======\n'
# print trainX
Ytrain = (prices.shift(-self.lookback) /
prices) - 1 # Ytrain is 14 day return
# print '=========Ytrain=======\n', Ytrain
Ytrain = Ytrain.values
# Ytrain.nan_to_num(0, copy=True)shift
# print '=========Ytrain=======\n', Ytrain
YBUY = 0.03
YSELL = -0.03
for i in range(len(Ytrain)):
if Ytrain[i] > YBUY:
Ytrain[i] = 1
elif Ytrain[i] < YSELL:
Ytrain[i] = -1
else:
Ytrain[i] = 0
# print '=========Ytrain=======\n', Ytrain
# print 'self.learner_1=======\n', self.learner_1
self.learner.addEvidence(Xtrain, Ytrain)
self.Y_in = self.learner.query(Xtrain)
def testPolicy(self, symbol="IBM", \
sd=dt.datetime(2009, 1, 1), \
ed=dt.datetime(2010, 1, 1), \
sv=10000):
# stock_older_days = 30
symbols = [symbol]
prices, sma, p_to_sma = indicators.price_sma(start_date=sd,
end_date=ed,
symbols=symbols)
prices.columns = ['Price']
sma.columns = ['sma']
p_to_sma.columns = ['p/sma']
rsi = indicators.rsi(start_date=sd, end_date=ed, symbols=symbols)
mfi = indicators.mfi(start_date=sd, end_date=ed, symbols=symbols)
vix = indicators.vix(start_date=sd, end_date=ed, symbols=symbols)
rsi.columns = ['RSI']
mfi.columns = ['MFI']
vix.columns = ['VIX']
portfolio = prices.copy()
portfolio.iloc[:, :] = np.nan
Xtrain = pd.concat([p_to_sma, rsi, mfi, vix], axis=1)
Xtrain.fillna(value=0, inplace=True)
# print '=========Xtrain=======\n', Xtrain
trading_days = Xtrain.index
Xtest = Xtrain.values[:, 1:]
# print '=========Xtest=======\n', Xtest
triggers = self.learner.query(Xtest)
# print '==========triggers====\n', triggers
triggers = pd.DataFrame(triggers[0], index=trading_days)
# use this for bag learner: triggers = pd.DataFrame(triggers[0], index=trading_days)
status = 'cash'
trades = prices.copy()
trades.values[:, :] = 0 # set them all to nothing
total_triggers = triggers.shape[0]
trading_days = list(prices.index)
j = 0
for i in range(total_triggers):
# option = triggers.ix[i, symbol][0]
option = triggers.ix[i, 0]
gap = trading_days.index(triggers.index[i]) - trading_days.index(
triggers.index[j])
if status == 'cash':
if option == 1.0:
trades.ix[triggers.index[i], :] = 1000
status = 'long'
j = i
elif option == -1.0:
trades.ix[triggers.index[i], :] = -1000
status = 'short'
j = i
elif status == 'long':
if (option == 0.0) & (gap >= self.lookback):
trades.ix[triggers.index[i], :] = -1000
status = 'cash'
j = i
elif (option == -1.0) & (gap >= self.lookback):
trades.ix[triggers.index[i], :] = -2000
status = 'short'
j = i
elif status == 'short':
if (option == 0.0) & (gap >= self.lookback):
trades.ix[triggers.index[i], :] = 1000
status = 'cash'
j = i
elif (option == 1.0) & (gap >= self.lookback):
trades.ix[triggers.index[i], :] = 2000
status = 'long'
j = i
if self.verbose: print type(trades) # it better be a DataFrame!
if self.verbose: print trades
return trades
def testPolicy(self, symbol="IBM", \
sd=dt.datetime(2009, 1, 1), \
ed=dt.datetime(2010, 1, 1), \
sv=10000):
# stock_older_days = 30
symbols = [symbol]
prices, sma, p_to_sma = indicators.price_sma(start_date=sd,
end_date=ed,
symbols=symbols)
prices.columns = ['Price']
sma.columns = ['sma']
p_to_sma.columns = ['p/sma']
rsi = indicators.rsi(start_date=sd, end_date=ed, symbols=symbols)
mfi = indicators.mfi(start_date=sd, end_date=ed, symbols=symbols)
vix = indicators.vix(start_date=sd, end_date=ed, symbols=symbols)
rsi.columns = ['RSI']
mfi.columns = ['MFI']
vix.columns = ['VIX']
portfolio = prices.copy()
portfolio.iloc[:, :] = np.nan
Xtrain = pd.concat([p_to_sma, rsi, mfi, vix], axis=1)
Xtrain.fillna(value=0, inplace=True)
# print '=========Xtrain=======\n', Xtrain
trading_days = Xtrain.index
Xtest = Xtrain.values[:, 1:]
# print '=========Xtest=======\n', Xtest
y_nparray = self.learner.query(Xtest)
# print list(self.y_nparray[0])
# print '=========Y_in=======\n', self.y_nparray
y_df = pd.DataFrame(data=list(y_nparray[0]),
index=trading_days,
columns=['y_nparray'])
# note: use (data=list(self.y_nparray[0]) for bagged
# print '====y_df====\n', y_df
# print y_df
curr_state = 'neutral'
long_entry = []
long_exit = []
short_entry = []
short_exit = []
entry_point = 0
for i in range(len(y_df)):
if ((int(y_df.iloc[i])) == 1) & (curr_state == 'neutral'):
# if (int(y_df.iloc[i])) == 1:
curr_state = 'long'
portfolio.iloc[i, 0] = 1000
entry_point = i
long_entry.append(portfolio.iloc[i:].index[0])
elif ((int(y_df.iloc[i])) == -1) & (curr_state == 'neutral'):
# elif (int(y_df.iloc[i])) == -1:
curr_state = 'short'
portfolio.iloc[i, 0] = -1000
entry_point = i
short_entry.append(portfolio.iloc[i:].index[0])
# if (i >= entry_point + self.lookback):
# curr_state = 'neutral'
# entry_point = 0
# long_exit.append(portfolio.iloc[i:].index[0])
if (i >= entry_point + self.lookback) & (curr_state == 'long'):
portfolio.iloc[i, 0] = 0
curr_state = 'neutral'
entry_point = 0
long_exit.append(portfolio.iloc[i:].index[0])
elif (i >= entry_point + self.lookback) & (curr_state == 'short'):
portfolio.iloc[i, 0] = 0
curr_state = 'neutral'
entry_point = 0
short_exit.append(portfolio.iloc[i:].index[0])
portfolio.fillna(method='ffill', inplace=True)
portfolio.fillna(0, inplace=True)
# print '======end portfolio====\n', portfolio
df_trade = portfolio - portfolio.shift(1)
df_trade.iloc[0] = portfolio.iloc[0]
# print '=====df_ttrade=====\n', df_trade
return df_trade
def testPolicy(self, symbol="IBM", \
sd=dt.datetime(2009, 1, 1), \
ed=dt.datetime(2010, 1, 1), \
sv=10000):
'''
# # here we build a fake set of trades
# # your code should return the same sort of data
# dates = pd.date_range(sd, ed)
# prices_all = ut.get_data([symbol], dates) # automatically adds SPY
# prices = prices_all[[symbol, ]]
# trading_days = prices.index
#
# price_sma, rsi, macd_hist = get_indicators(prices)
#
# xvalues = pd.concat([price_sma, rsi, macd_hist], ignore_index=True, axis=1)
# # print '=====xvalues=====\n', xvalues
# xvalues.fillna(value=0, inplace=True)
# testX = np.array(xvalues)
'''
# stock_older_days = 30
symbols = [symbol]
prices, sma, p_to_sma = indicators.price_sma(start_date=sd, end_date=ed, symbols=symbols)
prices.columns = ['Price']
sma.columns = ['sma']
p_to_sma.columns = ['p/sma']
rsi = indicators.rsi(start_date=sd, end_date=ed, symbols=symbols)
mfi = indicators.mfi(start_date=sd, end_date=ed, symbols=symbols)
vix = indicators.vix(start_date=sd, end_date=ed, symbols=symbols)
rsi.columns = ['RSI']
mfi.columns = ['MFI']
vix.columns = ['VIX']
portfolio = prices.copy()
portfolio.iloc[:, :] = np.nan
Xtrain = pd.concat([p_to_sma, rsi, mfi, vix], axis=1)
Xtrain.fillna(value=0, inplace=True)
# print '=========Xtrain=======\n', Xtrain
trading_days = Xtrain.index
testX = Xtrain.values[:, 1:]
# print '=========trainX=======\n', testX
# print '==========textX====\n', testX
triggers = self.learner.query(testX)
# print '==========triggers====\n', triggers
triggers = pd.DataFrame(triggers[0], index=trading_days)
# use this for bag learner: triggers = pd.DataFrame(triggers[0], index=trading_days)
status = 'cash'
trades = prices.copy()
trades.values[:, :] = 0 # set them all to nothing
total_triggers = triggers.shape[0]
trading_days = list(prices.index)
j = 0
for i in range(total_triggers):
# option = triggers.ix[i, symbol][0]
option = triggers.ix[i, 0]
gap = trading_days.index(triggers.index[i]) - trading_days.index(triggers.index[j])
if status == 'cash':
if option == 1.0:
trades.ix[triggers.index[i], :] = 1000
status = 'long'
j = i
elif option == -1.0:
trades.ix[triggers.index[i], :] = -1000
status = 'short'
j = i
elif status == 'long':
if (option == 0.0) & (gap >= self.N):
trades.ix[triggers.index[i], :] = -1000
status = 'cash'
j = i
elif (option == -1.0) & (gap >= self.N):
trades.ix[triggers.index[i], :] = -2000
status = 'short'
j = i
elif status == 'short':
if (option == 0.0) & (gap >= self.N):
trades.ix[triggers.index[i], :] = 1000
status = 'cash'
j = i
elif (option == 1.0) & (gap >= self.N):
trades.ix[triggers.index[i], :] = 2000
status = 'long'
j = i
if self.verbose: print type(trades) # it better be a DataFrame!
if self.verbose: print trades
return trades
def addEvidence(self, symbol="IBM", \
sd=dt.datetime(2008, 1, 1), \
ed=dt.datetime(2009, 1, 1), \
sv=10000):
'''Firstly assemble the indicators into the trainning Dataframe (Xtrain and Ytrain), then train the RTLearner'''
'''calculate indicators'''
stock_older_days = 45
symbols = [symbol]
prices, unused_a, unused_b = indicators.price_sma(
start_date=(sd - dt.timedelta(days=stock_older_days)),
end_date=ed,
symbols=symbols)
first_trading_day = sd
while first_trading_day not in prices.index:
first_trading_day += dt.timedelta(days=1)
prices = prices / prices.loc[first_trading_day]
sma = prices.copy().rolling(14).mean()
p_to_sma = prices / sma
prices.columns = ['Price']
sma.columns = ['sma']
p_to_sma.columns = ['p/sma']
rsi = indicators.rsi(start_date=(sd -
dt.timedelta(days=stock_older_days)),
end_date=ed,
symbols=symbols)
mfi = indicators.mfi(start_date=(sd -
dt.timedelta(days=stock_older_days)),
end_date=ed,
symbols=symbols)
vix = indicators.vix(start_date=sd, end_date=ed, symbols=symbols)
rsi.columns = ['RSI']
mfi.columns = ['MFI']
vix.columns = ['VIX']
'''cut indicators into proper size. i.e. betyween start_date and end_date'''
prices = prices.loc[first_trading_day:]
sma = sma.loc[first_trading_day:]
p_to_sma = p_to_sma.loc[first_trading_day:]
rsi = rsi.loc[first_trading_day:]
mfi = mfi.loc[first_trading_day:]
vix = vix.loc[first_trading_day:]
'''assemble Xtrain'''
Xtrain = pd.concat([p_to_sma, rsi, mfi, vix], axis=1)
# Xtrain.fillna(value=0, inplace=True)
# print '=========Xtrain=======\n'
# print Xtrain
Xtrain = Xtrain.values[:, 1:]
# print '=========trainX=======\n'
# print trainX
'''assemble Ytrain'''
Ytrain = (prices.shift(-self.lookback) /
prices) - 1 # Ytrain is 14 day return
# print '=========Ytrain=======\n', Ytrain
Ytrain[Ytrain > 0] = prices.shift(-self.lookback) / (
prices * (1.0 + 2 * self.impact)) - 1.0
Ytrain[Ytrain < 0] = prices.shift(-self.lookback) / (
prices * (1.0 - 2 * self.impact)) - 1.0
Ytrain = Ytrain.values
# Ytrain.nan_to_num(0, copy=True)shift
# print '=========Ytrain=======\n', Ytrain
YBUY = 0.08
YSELL = -0.08
for i in range(len(Ytrain)):
if Ytrain[i] > YBUY:
Ytrain[i] = 1
elif Ytrain[i] < YSELL:
Ytrain[i] = -1
else:
Ytrain[i] = 0
# print '=========Ytrain=======\n', Ytrain
# print 'self.learner_1=======\n', self.learner_1
'''use Xtain and Ytrain to train the random table'''
self.learner.addEvidence(Xtrain, Ytrain)
self.Y_in = self.learner.query(Xtrain)
def testPolicy(self, symbol="IBM", \
sd=dt.datetime(2009, 1, 1), \
ed=dt.datetime(2010, 1, 1), \
sv=10000):
'''Firstly assemble the indicators into the testing Dataframe (Xtest), then query the RTLearner for Ytrain'''
'''calculate indicators'''
stock_older_days = 45
symbols = [symbol]
prices, unused_a, unused_b = indicators.price_sma(
start_date=(sd - dt.timedelta(days=stock_older_days)),
end_date=ed,
symbols=symbols)
first_trading_day = sd
while first_trading_day not in prices.index:
first_trading_day += dt.timedelta(days=1)
prices = prices / prices.loc[first_trading_day]
sma = prices.copy().rolling(14).mean()
p_to_sma = prices / sma
prices.columns = ['Price']
sma.columns = ['sma']
p_to_sma.columns = ['p/sma']
rsi = indicators.rsi(start_date=(sd -
dt.timedelta(days=stock_older_days)),
end_date=ed,
symbols=symbols)
mfi = indicators.mfi(start_date=(sd -
dt.timedelta(days=stock_older_days)),
end_date=ed,
symbols=symbols)
vix = indicators.vix(start_date=sd, end_date=ed, symbols=symbols)
rsi.columns = ['RSI']
mfi.columns = ['MFI']
vix.columns = ['VIX']
'''cut indicators into proper size. i.e. betyween start_date and end_date'''
prices = prices.loc[first_trading_day:]
sma = sma.loc[first_trading_day:]
p_to_sma = p_to_sma.loc[first_trading_day:]
rsi = rsi.loc[first_trading_day:]
mfi = mfi.loc[first_trading_day:]
vix = vix.loc[first_trading_day:]
'''' make empty portfolio for future use'''
portfolio = prices.copy()
portfolio.iloc[:, :] = np.nan
'''assemble Xtest'''
Xtest = pd.concat([p_to_sma, rsi, mfi, vix],
axis=1,
join_axes=[vix.index])
# Xtrain.fillna(value=0, inplace=True)
# print '=========Xtrain=======\n', Xtrain
date_index = Xtest.index
Xtest = Xtest.values[:, 1:]
# print '=========Xtest=======\n', Xtest
'''query for Ytest, then assemble it back to a dataframe for the market simulator'''
Ytest = self.learner.query(Xtest)
# print list(self.y_nparray[0])
# print '=========Y_in=======\n', self.y_nparray
y_df = pd.DataFrame(data=list(Ytest[0]),
index=date_index,
columns=['Ytest'])
# note: use (data=list(self.y_nparray[0]) for bagged
# print '====y_df====\n', y_df
# print y_df
'''simulating market, enter a position, then exit the position until lookback days passed '''
curr_state = 'neutral'
long_entry = []
long_exit = []
short_entry = []
short_exit = []
entry_point = 0
for i in range(len(y_df)):
if ((int(y_df.iloc[i])) == 1) & (curr_state == 'neutral'):
# if (int(y_df.iloc[i])) == 1:
curr_state = 'long'
portfolio.iloc[i, 0] = 1000
entry_point = i
long_entry.append(portfolio.iloc[i:].index[0])
elif ((int(y_df.iloc[i])) == -1) & (curr_state == 'neutral'):
# elif (int(y_df.iloc[i])) == -1:
curr_state = 'short'
portfolio.iloc[i, 0] = -1000
entry_point = i
short_entry.append(portfolio.iloc[i:].index[0])
if (i >= entry_point + self.lookback) & (curr_state == 'long'):
portfolio.iloc[i, 0] = 0
curr_state = 'neutral'
entry_point = 0
long_exit.append(portfolio.iloc[i:].index[0])
elif (i >= entry_point + self.lookback) & (curr_state == 'short'):
portfolio.iloc[i, 0] = 0
curr_state = 'neutral'
entry_point = 0
short_exit.append(portfolio.iloc[i:].index[0])
'''good old market simulator, generating the df_trade'''
portfolio.fillna(method='ffill', inplace=True)
portfolio.fillna(0, inplace=True)
# print '======end portfolio====\n', portfolio
df_trade = portfolio - portfolio.shift(1)
df_trade.iloc[0] = portfolio.iloc[0]
# print '=====df_ttrade=====\n', type(df_trade)
return df_trade