def testPolicy(self, symbol="JPM", sd=dt.datetime(2008, 1, 1), ed=dt.datetime(2009, 12, 31), sv=100000): symbols = [symbol] stock_price, sma, p_to_sma = indicators.price_sma(start_date=sd, end_date=ed, symbols=symbols) rsi = indicators.rsi(start_date=sd, end_date=ed, symbols=symbols) mfi = indicators.mfi(start_date=sd, end_date=ed, symbols=symbols) vix = indicators.vix(start_date=sd, end_date=ed, symbols=symbols) portfolio = stock_price.copy() portfolio.iloc[:, :] = np.nan # print '======begining portfolio====\n', portfolio # print portfolio curr_state = 'neutral' long_entry = [] long_exit = [] short_entry = [] short_exit = [] for i in range(len(portfolio)): # print curr_state if (rsi.iloc[i, 0] < 30) & (p_to_sma.iloc[i, 0] < 1) & ( curr_state != 'long') & (vix.iloc[i, 0] < 50): portfolio.iloc[i, 0] = 1000 curr_state = 'long' long_entry.append(portfolio.iloc[i:].index[0]) elif (mfi.iloc[i, 0] > 70) & (p_to_sma.iloc[i, 0] > 1) & ( curr_state != 'short') & (vix.iloc[i, 0] > 20): portfolio.iloc[i, 0] = -1000 curr_state = 'short' short_entry.append(portfolio.iloc[i:].index[0]) if (rsi.iloc[i, 0] >= 55) & (curr_state == 'long'): portfolio.iloc[i, 0] = 0 curr_state = 'neutral' long_exit.append(portfolio.iloc[i:].index[0]) elif (mfi.iloc[i, 0] <= 45) & (curr_state == 'short'): portfolio.iloc[i, 0] = 0 curr_state = 'neutral' short_exit.append(portfolio.iloc[i:].index[0]) portfolio.fillna(method='ffill', inplace=True) portfolio.fillna(0, inplace=True) # print '======end portfolio====\n', portfolio df_trade = portfolio - portfolio.shift(1) df_trade.iloc[0] = portfolio.iloc[0] # print '=====df_ttrade=====\n', df_trade return df_trade, long_entry, long_exit, short_entry, short_exit
def addEvidence(self, symbol="IBM", \ sd=dt.datetime(2008, 1, 1), \ ed=dt.datetime(2009, 1, 1), \ sv=10000): # stock_older_days = 30 symbols = [symbol] prices, sma, p_to_sma = indicators.price_sma(start_date=sd, end_date=ed, symbols=symbols) prices.columns = ['Price'] sma.columns = ['sma'] p_to_sma.columns = ['p/sma'] rsi = indicators.rsi(start_date=sd, end_date=ed, symbols=symbols) mfi = indicators.mfi(start_date=sd, end_date=ed, symbols=symbols) vix = indicators.vix(start_date=sd, end_date=ed, symbols=symbols) rsi.columns = ['RSI'] mfi.columns = ['MFI'] vix.columns = ['VIX'] portfolio = prices.copy() portfolio.iloc[:, :] = np.nan Xtrain = pd.concat([p_to_sma, rsi, mfi, vix], axis=1) Xtrain.fillna(value=0, inplace=True) # print '=========Xtrain=======\n' # print Xtrain date_index = Xtrain.index trainX = Xtrain.values[:, 1:] # print '=========trainX=======\n' # print trainX self.N = 7 nday_rt = prices.shift(-self.N) / prices - 1.0 pd.set_option('display.max_rows', 1000) # print 'n days return: ' # print nday_rt # print nday_rt[nday_rt > 0] = prices.shift(-self.N) / (prices * (1.0 + self.impact)) - 1.0 nday_rt[nday_rt < 0] = prices.shift(-self.N) / (prices * (1.0 - self.impact)) - 1.0 # print 'n days return: ' # print nday_rt # print yvalues = nday_rt.copy() yvalues.iloc[:, :] = 0 # go long when return is larger than 3%, go short when return is lower than -3% threshold = 0.03 yvalues[nday_rt >= threshold] = 1 yvalues[nday_rt <= -threshold] = -1 # print xvalues == 0 yvalues[trainX == 0] = 0 trainY = np.array(yvalues) self.learner.addEvidence(trainX, trainY)
def addEvidence(self, symbol="IBM", \ sd=dt.datetime(2008, 1, 1), \ ed=dt.datetime(2009, 1, 1), \ sv=10000): # stock_older_days = 30 symbols = [symbol] prices, sma, p_to_sma = indicators.price_sma(start_date=sd, end_date=ed, symbols=symbols) prices.columns = ['Price'] sma.columns = ['sma'] p_to_sma.columns = ['p/sma'] rsi = indicators.rsi(start_date=sd, end_date=ed, symbols=symbols) mfi = indicators.mfi(start_date=sd, end_date=ed, symbols=symbols) vix = indicators.vix(start_date=sd, end_date=ed, symbols=symbols) rsi.columns = ['RSI'] mfi.columns = ['MFI'] vix.columns = ['VIX'] portfolio = prices.copy() portfolio.iloc[:, :] = np.nan Xtrain = pd.concat([p_to_sma, rsi, mfi, vix], axis=1) Xtrain.fillna(value=0, inplace=True) # print '=========Xtrain=======\n' # print Xtrain date_index = Xtrain.index Xtrain = Xtrain.values[:, 1:] # print '=========trainX=======\n' # print trainX Ytrain = (prices.shift(-self.lookback) / prices) - 1 # Ytrain is 14 day return # print '=========Ytrain=======\n', Ytrain Ytrain = Ytrain.values # Ytrain.nan_to_num(0, copy=True)shift # print '=========Ytrain=======\n', Ytrain YBUY = 0.03 YSELL = -0.03 for i in range(len(Ytrain)): if Ytrain[i] > YBUY: Ytrain[i] = 1 elif Ytrain[i] < YSELL: Ytrain[i] = -1 else: Ytrain[i] = 0 # print '=========Ytrain=======\n', Ytrain # print 'self.learner_1=======\n', self.learner_1 self.learner.addEvidence(Xtrain, Ytrain) self.Y_in = self.learner.query(Xtrain)
def testPolicy(self, symbol="IBM", \ sd=dt.datetime(2009, 1, 1), \ ed=dt.datetime(2010, 1, 1), \ sv=10000): # stock_older_days = 30 symbols = [symbol] prices, sma, p_to_sma = indicators.price_sma(start_date=sd, end_date=ed, symbols=symbols) prices.columns = ['Price'] sma.columns = ['sma'] p_to_sma.columns = ['p/sma'] rsi = indicators.rsi(start_date=sd, end_date=ed, symbols=symbols) mfi = indicators.mfi(start_date=sd, end_date=ed, symbols=symbols) vix = indicators.vix(start_date=sd, end_date=ed, symbols=symbols) rsi.columns = ['RSI'] mfi.columns = ['MFI'] vix.columns = ['VIX'] portfolio = prices.copy() portfolio.iloc[:, :] = np.nan Xtrain = pd.concat([p_to_sma, rsi, mfi, vix], axis=1) Xtrain.fillna(value=0, inplace=True) # print '=========Xtrain=======\n', Xtrain trading_days = Xtrain.index Xtest = Xtrain.values[:, 1:] # print '=========Xtest=======\n', Xtest triggers = self.learner.query(Xtest) # print '==========triggers====\n', triggers triggers = pd.DataFrame(triggers[0], index=trading_days) # use this for bag learner: triggers = pd.DataFrame(triggers[0], index=trading_days) status = 'cash' trades = prices.copy() trades.values[:, :] = 0 # set them all to nothing total_triggers = triggers.shape[0] trading_days = list(prices.index) j = 0 for i in range(total_triggers): # option = triggers.ix[i, symbol][0] option = triggers.ix[i, 0] gap = trading_days.index(triggers.index[i]) - trading_days.index( triggers.index[j]) if status == 'cash': if option == 1.0: trades.ix[triggers.index[i], :] = 1000 status = 'long' j = i elif option == -1.0: trades.ix[triggers.index[i], :] = -1000 status = 'short' j = i elif status == 'long': if (option == 0.0) & (gap >= self.lookback): trades.ix[triggers.index[i], :] = -1000 status = 'cash' j = i elif (option == -1.0) & (gap >= self.lookback): trades.ix[triggers.index[i], :] = -2000 status = 'short' j = i elif status == 'short': if (option == 0.0) & (gap >= self.lookback): trades.ix[triggers.index[i], :] = 1000 status = 'cash' j = i elif (option == 1.0) & (gap >= self.lookback): trades.ix[triggers.index[i], :] = 2000 status = 'long' j = i if self.verbose: print type(trades) # it better be a DataFrame! if self.verbose: print trades return trades
def testPolicy(self, symbol="IBM", \ sd=dt.datetime(2009, 1, 1), \ ed=dt.datetime(2010, 1, 1), \ sv=10000): # stock_older_days = 30 symbols = [symbol] prices, sma, p_to_sma = indicators.price_sma(start_date=sd, end_date=ed, symbols=symbols) prices.columns = ['Price'] sma.columns = ['sma'] p_to_sma.columns = ['p/sma'] rsi = indicators.rsi(start_date=sd, end_date=ed, symbols=symbols) mfi = indicators.mfi(start_date=sd, end_date=ed, symbols=symbols) vix = indicators.vix(start_date=sd, end_date=ed, symbols=symbols) rsi.columns = ['RSI'] mfi.columns = ['MFI'] vix.columns = ['VIX'] portfolio = prices.copy() portfolio.iloc[:, :] = np.nan Xtrain = pd.concat([p_to_sma, rsi, mfi, vix], axis=1) Xtrain.fillna(value=0, inplace=True) # print '=========Xtrain=======\n', Xtrain trading_days = Xtrain.index Xtest = Xtrain.values[:, 1:] # print '=========Xtest=======\n', Xtest y_nparray = self.learner.query(Xtest) # print list(self.y_nparray[0]) # print '=========Y_in=======\n', self.y_nparray y_df = pd.DataFrame(data=list(y_nparray[0]), index=trading_days, columns=['y_nparray']) # note: use (data=list(self.y_nparray[0]) for bagged # print '====y_df====\n', y_df # print y_df curr_state = 'neutral' long_entry = [] long_exit = [] short_entry = [] short_exit = [] entry_point = 0 for i in range(len(y_df)): if ((int(y_df.iloc[i])) == 1) & (curr_state == 'neutral'): # if (int(y_df.iloc[i])) == 1: curr_state = 'long' portfolio.iloc[i, 0] = 1000 entry_point = i long_entry.append(portfolio.iloc[i:].index[0]) elif ((int(y_df.iloc[i])) == -1) & (curr_state == 'neutral'): # elif (int(y_df.iloc[i])) == -1: curr_state = 'short' portfolio.iloc[i, 0] = -1000 entry_point = i short_entry.append(portfolio.iloc[i:].index[0]) # if (i >= entry_point + self.lookback): # curr_state = 'neutral' # entry_point = 0 # long_exit.append(portfolio.iloc[i:].index[0]) if (i >= entry_point + self.lookback) & (curr_state == 'long'): portfolio.iloc[i, 0] = 0 curr_state = 'neutral' entry_point = 0 long_exit.append(portfolio.iloc[i:].index[0]) elif (i >= entry_point + self.lookback) & (curr_state == 'short'): portfolio.iloc[i, 0] = 0 curr_state = 'neutral' entry_point = 0 short_exit.append(portfolio.iloc[i:].index[0]) portfolio.fillna(method='ffill', inplace=True) portfolio.fillna(0, inplace=True) # print '======end portfolio====\n', portfolio df_trade = portfolio - portfolio.shift(1) df_trade.iloc[0] = portfolio.iloc[0] # print '=====df_ttrade=====\n', df_trade return df_trade
def testPolicy(self, symbol="IBM", \ sd=dt.datetime(2009, 1, 1), \ ed=dt.datetime(2010, 1, 1), \ sv=10000): ''' # # here we build a fake set of trades # # your code should return the same sort of data # dates = pd.date_range(sd, ed) # prices_all = ut.get_data([symbol], dates) # automatically adds SPY # prices = prices_all[[symbol, ]] # trading_days = prices.index # # price_sma, rsi, macd_hist = get_indicators(prices) # # xvalues = pd.concat([price_sma, rsi, macd_hist], ignore_index=True, axis=1) # # print '=====xvalues=====\n', xvalues # xvalues.fillna(value=0, inplace=True) # testX = np.array(xvalues) ''' # stock_older_days = 30 symbols = [symbol] prices, sma, p_to_sma = indicators.price_sma(start_date=sd, end_date=ed, symbols=symbols) prices.columns = ['Price'] sma.columns = ['sma'] p_to_sma.columns = ['p/sma'] rsi = indicators.rsi(start_date=sd, end_date=ed, symbols=symbols) mfi = indicators.mfi(start_date=sd, end_date=ed, symbols=symbols) vix = indicators.vix(start_date=sd, end_date=ed, symbols=symbols) rsi.columns = ['RSI'] mfi.columns = ['MFI'] vix.columns = ['VIX'] portfolio = prices.copy() portfolio.iloc[:, :] = np.nan Xtrain = pd.concat([p_to_sma, rsi, mfi, vix], axis=1) Xtrain.fillna(value=0, inplace=True) # print '=========Xtrain=======\n', Xtrain trading_days = Xtrain.index testX = Xtrain.values[:, 1:] # print '=========trainX=======\n', testX # print '==========textX====\n', testX triggers = self.learner.query(testX) # print '==========triggers====\n', triggers triggers = pd.DataFrame(triggers[0], index=trading_days) # use this for bag learner: triggers = pd.DataFrame(triggers[0], index=trading_days) status = 'cash' trades = prices.copy() trades.values[:, :] = 0 # set them all to nothing total_triggers = triggers.shape[0] trading_days = list(prices.index) j = 0 for i in range(total_triggers): # option = triggers.ix[i, symbol][0] option = triggers.ix[i, 0] gap = trading_days.index(triggers.index[i]) - trading_days.index(triggers.index[j]) if status == 'cash': if option == 1.0: trades.ix[triggers.index[i], :] = 1000 status = 'long' j = i elif option == -1.0: trades.ix[triggers.index[i], :] = -1000 status = 'short' j = i elif status == 'long': if (option == 0.0) & (gap >= self.N): trades.ix[triggers.index[i], :] = -1000 status = 'cash' j = i elif (option == -1.0) & (gap >= self.N): trades.ix[triggers.index[i], :] = -2000 status = 'short' j = i elif status == 'short': if (option == 0.0) & (gap >= self.N): trades.ix[triggers.index[i], :] = 1000 status = 'cash' j = i elif (option == 1.0) & (gap >= self.N): trades.ix[triggers.index[i], :] = 2000 status = 'long' j = i if self.verbose: print type(trades) # it better be a DataFrame! if self.verbose: print trades return trades
def addEvidence(self, symbol = "AAPL", \ sd=dt.datetime(2008,1,1), \ ed=dt.datetime(2009,12,31), \ sv = 100000): # need to determine a lookback and pull data for before hand self.learner = ql.QLearner(num_actions=3, num_states=3000) # example usage of the old backward compatible util function syms = [symbol] lookback = 14 indicator_start = sd.date() - dt.timedelta(days=(lookback * 2)) dates = pd.date_range(indicator_start, ed) prices_all = ut.get_data(syms, dates) # automatically adds SPY prices = prices_all[syms] # only portfolio symbols prices_SPY = prices_all['SPY'] # only SPY, for comparison later sma, psr = ind.price_sma(prices, lookback) _, _, bbp = ind.bollinger_bands(prices, sma, lookback) wpr = ind.williams_pr(prices, lookback) psr = psr.ix[sd:, :].dropna() wpr = wpr.ix[sd:, :].dropna() bbp = bbp.ix[sd:, :].dropna() prices = prices.ix[sd:, :] self.indicators = pd.concat([ self.discretize_index(wpr, sym=symbol), self.discretize_index(bbp, sym=symbol), self.discretize_index(psr, sym=symbol) ], axis=1) iterations = 5 convergence_count = 0 last_return = 0 daily_returns = (prices.shift(-1) / prices) - 1 while iterations: actions = [] holdings = [0] port_val = [sv] state = int( str(1) + "".join([str(int(i)) for i in self.indicators.iloc[0]])) actions.append(self.learner.querysetstate(state)) #yday_return = for day in prices.index: # reward based on yesterday's action # change in price bewteen first and second day if len(holdings) == 1: reward = 0 elif actions[-1] == 0: if daily_returns.ix[day, 0] > 0 and holdings[-1] > 0: reward = 1000 * daily_returns.ix[day, 0] elif daily_returns.ix[day, 0] > 0 and holdings[-1] < 0: reward = -1000 * daily_returns.ix[day, 0] elif daily_returns.ix[day, 0] < 0 and holdings[-1] < 0: reward = -1000 * daily_returns.ix[day, 0] elif daily_returns.ix[day, 0] < 0 and holdings[-1] > 0: reward = 1000 * daily_returns.ix[day, 0] elif actions[-1] == 1: # if we correctly picked a buy reward = 1000 * daily_returns.ix[day, 0] elif actions[-1] == 2: reward = -1000 * daily_returns.ix[day, 0] #we're calculating the new state for the second day hold_ind = 1 if holdings[-1] != 0: hold_ind = 2 state = int( str(hold_ind) + "".join([str(int(i)) for i in self.indicators.ix[day, :]])) # this will be "yesterday's action" and the price change for second and third day actions.append(self.learner.query(state, reward)) shares, holdings = self.simulate_trade(actions[-1], holdings) iterations -= 1 if self.verbose: print port_val[-1]
def addEvidence(self, symbol="IBM", \ sd=dt.datetime(2008, 1, 1), \ ed=dt.datetime(2009, 1, 1), \ sv=10000): '''Firstly assemble the indicators into the trainning Dataframe (Xtrain and Ytrain), then train the RTLearner''' '''calculate indicators''' stock_older_days = 45 symbols = [symbol] prices, unused_a, unused_b = indicators.price_sma( start_date=(sd - dt.timedelta(days=stock_older_days)), end_date=ed, symbols=symbols) first_trading_day = sd while first_trading_day not in prices.index: first_trading_day += dt.timedelta(days=1) prices = prices / prices.loc[first_trading_day] sma = prices.copy().rolling(14).mean() p_to_sma = prices / sma prices.columns = ['Price'] sma.columns = ['sma'] p_to_sma.columns = ['p/sma'] rsi = indicators.rsi(start_date=(sd - dt.timedelta(days=stock_older_days)), end_date=ed, symbols=symbols) mfi = indicators.mfi(start_date=(sd - dt.timedelta(days=stock_older_days)), end_date=ed, symbols=symbols) vix = indicators.vix(start_date=sd, end_date=ed, symbols=symbols) rsi.columns = ['RSI'] mfi.columns = ['MFI'] vix.columns = ['VIX'] '''cut indicators into proper size. i.e. betyween start_date and end_date''' prices = prices.loc[first_trading_day:] sma = sma.loc[first_trading_day:] p_to_sma = p_to_sma.loc[first_trading_day:] rsi = rsi.loc[first_trading_day:] mfi = mfi.loc[first_trading_day:] vix = vix.loc[first_trading_day:] '''assemble Xtrain''' Xtrain = pd.concat([p_to_sma, rsi, mfi, vix], axis=1) # Xtrain.fillna(value=0, inplace=True) # print '=========Xtrain=======\n' # print Xtrain Xtrain = Xtrain.values[:, 1:] # print '=========trainX=======\n' # print trainX '''assemble Ytrain''' Ytrain = (prices.shift(-self.lookback) / prices) - 1 # Ytrain is 14 day return # print '=========Ytrain=======\n', Ytrain Ytrain[Ytrain > 0] = prices.shift(-self.lookback) / ( prices * (1.0 + 2 * self.impact)) - 1.0 Ytrain[Ytrain < 0] = prices.shift(-self.lookback) / ( prices * (1.0 - 2 * self.impact)) - 1.0 Ytrain = Ytrain.values # Ytrain.nan_to_num(0, copy=True)shift # print '=========Ytrain=======\n', Ytrain YBUY = 0.08 YSELL = -0.08 for i in range(len(Ytrain)): if Ytrain[i] > YBUY: Ytrain[i] = 1 elif Ytrain[i] < YSELL: Ytrain[i] = -1 else: Ytrain[i] = 0 # print '=========Ytrain=======\n', Ytrain # print 'self.learner_1=======\n', self.learner_1 '''use Xtain and Ytrain to train the random table''' self.learner.addEvidence(Xtrain, Ytrain) self.Y_in = self.learner.query(Xtrain)
def testPolicy(self, symbol="IBM", \ sd=dt.datetime(2009, 1, 1), \ ed=dt.datetime(2010, 1, 1), \ sv=10000): '''Firstly assemble the indicators into the testing Dataframe (Xtest), then query the RTLearner for Ytrain''' '''calculate indicators''' stock_older_days = 45 symbols = [symbol] prices, unused_a, unused_b = indicators.price_sma( start_date=(sd - dt.timedelta(days=stock_older_days)), end_date=ed, symbols=symbols) first_trading_day = sd while first_trading_day not in prices.index: first_trading_day += dt.timedelta(days=1) prices = prices / prices.loc[first_trading_day] sma = prices.copy().rolling(14).mean() p_to_sma = prices / sma prices.columns = ['Price'] sma.columns = ['sma'] p_to_sma.columns = ['p/sma'] rsi = indicators.rsi(start_date=(sd - dt.timedelta(days=stock_older_days)), end_date=ed, symbols=symbols) mfi = indicators.mfi(start_date=(sd - dt.timedelta(days=stock_older_days)), end_date=ed, symbols=symbols) vix = indicators.vix(start_date=sd, end_date=ed, symbols=symbols) rsi.columns = ['RSI'] mfi.columns = ['MFI'] vix.columns = ['VIX'] '''cut indicators into proper size. i.e. betyween start_date and end_date''' prices = prices.loc[first_trading_day:] sma = sma.loc[first_trading_day:] p_to_sma = p_to_sma.loc[first_trading_day:] rsi = rsi.loc[first_trading_day:] mfi = mfi.loc[first_trading_day:] vix = vix.loc[first_trading_day:] '''' make empty portfolio for future use''' portfolio = prices.copy() portfolio.iloc[:, :] = np.nan '''assemble Xtest''' Xtest = pd.concat([p_to_sma, rsi, mfi, vix], axis=1, join_axes=[vix.index]) # Xtrain.fillna(value=0, inplace=True) # print '=========Xtrain=======\n', Xtrain date_index = Xtest.index Xtest = Xtest.values[:, 1:] # print '=========Xtest=======\n', Xtest '''query for Ytest, then assemble it back to a dataframe for the market simulator''' Ytest = self.learner.query(Xtest) # print list(self.y_nparray[0]) # print '=========Y_in=======\n', self.y_nparray y_df = pd.DataFrame(data=list(Ytest[0]), index=date_index, columns=['Ytest']) # note: use (data=list(self.y_nparray[0]) for bagged # print '====y_df====\n', y_df # print y_df '''simulating market, enter a position, then exit the position until lookback days passed ''' curr_state = 'neutral' long_entry = [] long_exit = [] short_entry = [] short_exit = [] entry_point = 0 for i in range(len(y_df)): if ((int(y_df.iloc[i])) == 1) & (curr_state == 'neutral'): # if (int(y_df.iloc[i])) == 1: curr_state = 'long' portfolio.iloc[i, 0] = 1000 entry_point = i long_entry.append(portfolio.iloc[i:].index[0]) elif ((int(y_df.iloc[i])) == -1) & (curr_state == 'neutral'): # elif (int(y_df.iloc[i])) == -1: curr_state = 'short' portfolio.iloc[i, 0] = -1000 entry_point = i short_entry.append(portfolio.iloc[i:].index[0]) if (i >= entry_point + self.lookback) & (curr_state == 'long'): portfolio.iloc[i, 0] = 0 curr_state = 'neutral' entry_point = 0 long_exit.append(portfolio.iloc[i:].index[0]) elif (i >= entry_point + self.lookback) & (curr_state == 'short'): portfolio.iloc[i, 0] = 0 curr_state = 'neutral' entry_point = 0 short_exit.append(portfolio.iloc[i:].index[0]) '''good old market simulator, generating the df_trade''' portfolio.fillna(method='ffill', inplace=True) portfolio.fillna(0, inplace=True) # print '======end portfolio====\n', portfolio df_trade = portfolio - portfolio.shift(1) df_trade.iloc[0] = portfolio.iloc[0] # print '=====df_ttrade=====\n', type(df_trade) return df_trade