def addEvidence(self, symbol="IBM",
sd=dt.datetime(2008, 1, 1),
ed=dt.datetime(2009, 1, 1),
sv=10000, n=1):
# add your code to do learning here
# Get price info
syms = [symbol]
adj_sd = sd - timedelta(days=30)
dates = pd.date_range(adj_sd, ed)
prices_all = ut.get_data(syms, dates) # automatically adds SPY
prices = prices_all[syms] # only portfolio symbols
# Calculate indicators
df = add_all_indicators(prices, syms[0], add_helper_data=False)
# Filter to time range
df = df.loc[sd:ed, :].copy()
# Get state df
state_df = self.fit_transform_state(df, symbol, n_days=n)
# Initialize Q-learner
self._learner = QLearner(num_states=len(self._state_dict), num_actions=len(self._actions), dyna=self._dyna)
# Fit learner
for _ in range(self._epochs):
self.fit_learner(state_df)
return
def __init__(self, verbose=False, impact=0.0):
self.verbose = verbose
self.impact = impact
self.learner = QLearner(num_states=90000, num_actions=2, dyna=0)
self.mom_mean = 0
self.mom_std = 0
self.long_mom_mean = 0
self.long_mom_std = 0
self.bollinger_bands = None
self.rsi = None
self.momentum = None
self.cr = -100000
def __init__(self, verbose=False, impact=0.0):
self.verbose = verbose
self.impact = impact
self.learner = QLearner(num_states=1000, \
num_actions = 3, \
alpha = 0.2, \
gamma = 0.9, \
rar = 0.5, \
radr = 0.99, \
dyna = 0, \
verbose = False)
def train_lunar_lander(env,
framework="pytorch",
hidden_layer_dimensions: List[int] = [128, 64],
use_dropout=False,
training_episode_count=2000,
alpha=1e-4,
gamma=0.99,
epsilon_start=1.0,
epsilon_decay=0.998,
epsilon_min=0.0,
replay_memory_size=2**16,
replay_sample_size=32,
training_start_memory_size=64,
mean_reward_recency=100,
model_saving_folder=join('.', "models")):
dimensions = _get_dimensions(env, hidden_layer_dimensions)
q_fun = _get_qfun(framework, dimensions, alpha, use_dropout=use_dropout)
lunar_lander = QLearner(env=env,
q_fun=q_fun,
epsilon_decay=epsilon_decay,
epsilon_min=epsilon_min,
gamma=gamma)
mean_reward, logs = lunar_lander.train(
episode_count=training_episode_count,
epsilon_start=epsilon_start,
replay_memory=replay_memory_size,
replay_sample_size=replay_sample_size,
training_start_memory_size=training_start_memory_size,
mean_reward_recency=mean_reward_recency)
file_path_no_exname, _ = _get_file_path(
folder_path=model_saving_folder,
framework=framework,
extension_name='.mod',
hidden_layer_dimensions=hidden_layer_dimensions,
mean_reward=mean_reward,
alpha=alpha,
gamma=gamma,
epsilon_start=epsilon_start,
epsilon_decay=epsilon_decay,
epsilon_min=epsilon_min,
replay_memory_size=replay_memory_size,
replay_sample_size=replay_sample_size,
use_dropout=use_dropout,
timestamp=int(time()))
q_fun.model.save(file_path_no_exname + '.mod')
logs.to_csv(file_path_no_exname + '.csv', index=False)
def __init__(self, max_position=100, starting_cash=100000):
self.indutil = IndUtil()
self.epochMax = 200
self.epochMin = 20
self.converged = False
self.learner = QLearner(
num_states=self.indutil.get_state_size(),
num_actions=3,
alpha=0.2,
gamma=0.9,
rar=0.5,
radr=0.99,
verbose=False
)
self.convergedata = collections.deque(maxlen=10)
self.numshares = max_position
self.starting_cash = float(starting_cash)
def __init__(self, learners=None):
self.cards = [i for i in range(52)]
# fold, check, call, they can raise any tenth of the pot, or up to 10x the pot
num_actions = 22
# pre-flop, flop, turn, river
if learners is None:
self.learners = [
QLearner(num_actions=num_actions),
QLearner(num_actions=num_actions),
QLearner(num_actions=num_actions),
QLearner(num_actions=num_actions)
]
else:
self.learners = learners
self.state = hand_state()
self.num_players = 0
self.big_blind = -1
self.choices = [[], [], [], []]
self.pot = 0
self.invested = 0
def Qlearning_Ltest():
''' Testing function for Q learning on L-track'''
q = QLearner(0.5, 0.9, 0.9, "L")
#q.track.show()
q.train((32, 2, 0, 1))
q.train((32, 3, 0, 1))
q.train()
q.train()
for i in range(10):
print(q.trial_run())
def addEvidence(self, symbol = "IBM", \
sd=dt.datetime(2008,1,1), \
ed=dt.datetime(2009,1,1), \
sv = 10000):
stock_data = StockData(symbol, sd, ed)
trading_state_factory = TradingStateFactory(
stock_data, self._indicator_discretizer)
self._learner = QLearner(num_states=trading_state_factory.num_states,
num_actions=3,
alpha=0.2,
gamma=0.9,
rar=0.5,
radr=0.99,
dyna=0)
self._trading_environment.qlearner = self._learner
self._trading_environment.trading_state_factory = trading_state_factory
self._trading_environment.stock_data = stock_data
self._trading_environment.trading_options = {
'trading_dates': stock_data.trading_dates,
'impact': self.impact
}
latest_cumulative_return = -999
current_cumulative_return = 0
episodes = 0
# Run learning episodes until the cumulative return of the strategy has converged
while np.abs(latest_cumulative_return -
current_cumulative_return) > 0.001:
latest_cumulative_return = current_cumulative_return
trades = self._trading_environment.run_learning_episode()
orders = self._convert_trades_to_marketisim_orders(symbol, trades)
portfolio_values = compute_portvals(
orders,
start_val=sv,
commission=0.,
impact=self.impact,
prices=stock_data.price.copy(),
)
current_cumulative_return = self._compute_cumulative_return(
portfolio_values)
episodes += 1
# Keep track of the number of training episodes
self._metadata['training_episodes'] = episodes
def crash_test():
''' Testing function for crash detection in simulator '''
q = QLearner(0.5, 0.9, "O")
start = (3, 21)
end = (7, 21)
c = q.track.check_for_crash(start, end)
if c:
print("Crashed!")
q.agent.set_state(c[0], c[1], 0, 0)
q.track.show()
else:
print("Safe!")
q.agent.set_state(end[0], end[1], 0, 0)
q.track.show()
def __init__(self, verbose=False, impact=0.0):
self.verbose = verbose
self.impact = impact
self.symbol = None
self.states = None
num_states = 500
num_actions = 3
alpha = 0.5 # 0.2
gamma = 0.75 # 0.8
rar = 0.65 # 0.5
radr = 0.99
dyna = 0
verbose = False
qleaner = QLearner(num_states, num_actions, alpha, gamma, rar, radr,
dyna, verbose)
self.learner = qleaner
plt.figure()
plot_df.plot(x='Iterations', y='Mean-V', title=title)
plt.show()
if __name__ == '__main__':
fl = FrozenLake(size=20)
vi, vi_policy, it, t = fl.value_iteration()
pi, pi_policy, _, _ = fl.policy_iteration(discount=0.9)
print('Eval : Value Iteration Policy')
fl.eval_policy(vi_policy, iterations=1000)
print('Eval : Policy Iteration Policy')
fl.eval_policy(pi_policy, iterations=1000)
ql = QLearner(fl.name, fl.prob, fl.rewards)
ql.q_learning_trials(trials=20, vi=vi, pi=pi)
run_stats, ql_policy = ql.q_learning(gamma=0.999,
alpha=0.45,
alpha_decay=0.999907088,
alpha_min=0.082682664,
epsilon=0.968587999,
epsilon_min=0.148113184,
epsilon_decay=0.996218224,
n_iter=60000,
returnStats=True)
fl.plotQlearn(run_stats, 'Frozen Lake - Q-Learning')
# fl.eval_policy(ql_policy,iterations=1000)
sys.exit()
from QLearner import QLearner
from QCharacter import QCharacter
from features1 import *
features = [
distanceToExit, distanceToBomb, distanceToMonster, inBombExplosionRange,
anyDroppedBombs
]
weights = [
173.17464200348178, -1.8397384409939697, -28.549911042490006,
-22.392729217308883, 0.050295801861828845
]
qlearner = QLearner(weights, features)
# Create the game
# TODO Change this if you want different random choices
g = Game.fromfile('map.txt')
g.add_monster(StupidMonster(
"stupid", # name
"S", # avatar
3,
9 # position
))
# TODO Add your character
q_character = QCharacter(
"me", # name
"C", # avatar
return p1_stats
if __name__ == "__main__":
# Example Usage
# battle(Board(show_board=True, show_result=True), RandomPlayer(), RandomPlayer(), 1, learn=False, show_result=True)
# battle(Board(), RandomPlayer(), RandomPlayer(), 100, learn=False, show_result=True)
# battle(Board(), RandomPlayer(), SmartPlayer(), 100, learn=False, show_result=True)
# battle(Board(), RandomPlayer(), PerfectPlayer(), 100, learn=False, show_result=True)
# battle(Board(), SmartPlayer(), PerfectPlayer(), 100, learn=False, show_result=True)
# ========================================================================
# ** Initialization **
# ========================================================================
qlearner = QLearner()
NUM = qlearner.GAME_NUM
# ========================================================================
# ** TRAIN: play 2*NUM games against players who only make random moves **
# ========================================================================
board = Board()
battle(board, RandomPlayer(), qlearner, NUM, learn=True, show_result=False)
battle(board, qlearner, RandomPlayer(), NUM, learn=True, show_result=False)
# ========================================================================
# ** TEST: play 1000 games against each opponent
# ========================================================================
q_rand = battle(board, qlearner, RandomPlayer(), 500)
rand_q = battle(board, RandomPlayer(), qlearner, 500)
q_smart = battle(board, qlearner, SmartPlayer(), 500)
class Agent:
def __init__(self, max_position=100, starting_cash=100000):
self.indutil = IndUtil()
self.epochMax = 200
self.epochMin = 20
self.converged = False
self.learner = QLearner(
num_states=self.indutil.get_state_size(),
num_actions=3,
alpha=0.2,
gamma=0.9,
rar=0.5,
radr=0.99,
verbose=False
)
self.convergedata = collections.deque(maxlen=10)
self.numshares = max_position
self.starting_cash = float(starting_cash)
"""
This will determine if the algorithm has converged (internal use only)
It will look for less than a 5% change in the average of the last 10 port values
"""
def has_converged(self, reward, epochs):
self.convergedata.append(reward)
if self.epochMin > epochs:
return False
elif abs(100*(np.average(self.convergedata)-reward)/reward) < 1:
return True
elif self.epochMax >= epochs:
return True
else:
return False
"""
This method is an internal method used by training to keep track of action effects
"""
def take_action(self, df, symbol, index, action, cash, position, last_port_value):
# 0 for hold, 1 for buy, 2 for sell
price = df.loc[index][symbol]
if action == 1 and position < self.numshares and cash >= self.numshares * price:
position += self.numshares
cash -= self.numshares * price
elif action == 2 and position > 0:
cash += position * price
position = 0
reward = ((cash + position*price)/last_port_value)-1
port_value = cash + position*price
return cash, position, reward, port_value
"""
This will train the agents q-learner
"""
def train(self, df, symbol):
states = self.indutil.compute_indicators(df, symbol)
epoch = 0
last_epoch_value = self.starting_cash
while not self.has_converged(last_epoch_value, epoch):
position = 0
cash = self.starting_cash
last_port_value = cash
action = self.learner.init_state(states.iloc[0].values[0])
cash, position, reward, port_value \
= self.take_action(df, symbol, states.index[0], action, cash, position, last_port_value)
for index, state in states[1:].iterrows():
action = self.learner.step(state.values[0], reward)
last_port_value = port_value
cash, position, reward, port_value \
= self.take_action(df, symbol, index, action, cash, position, last_port_value)
price = df.iloc[-1][symbol]
last_epoch_value = position * price + cash
epoch += 1
"""
This method will be called to test the learned policy.
"""
def test(self, df, symbol):
states = self.indutil.compute_indicators(df, symbol)
trades = df[[symbol, ]]
trades.values[:, :] = 0
position = 0
cash = self.starting_cash
for index, state in states.iterrows():
action = self.learner.query(state.values[0])
price = df.loc[index][symbol]
if action == 1 and cash >= price*self.numshares and position < self.numshares:
trades.loc[index][symbol] = self.numshares
position = self.numshares
cash -= price*self.numshares
elif action == 2 and position > 0:
trades.loc[index][symbol] = -position
cash += price * position
position = 0
return trades
class StrategyLearner(object):
# constructor
def __init__(self, verbose=False, impact=0.0):
self.verbose = verbose
self.impact = impact
self.learner = QLearner(num_states=1000, \
num_actions = 3, \
alpha = 0.2, \
gamma = 0.9, \
rar = 0.5, \
radr = 0.99, \
dyna = 0, \
verbose = False)
# this method should create a QLearner, and train it for trading
def addEvidence(self, symbol = "JPM", \
sd=dt.datetime(2008,1,1), \
ed=dt.datetime(2009,12,31), \
sv = 100000):
dates = pd.date_range(sd, ed)
prices = ut.get_data([symbol], dates)[[symbol]]
prices['Cash'] = 1.0
high = ut.get_data([symbol], dates, colname='High')[[symbol]]
low = ut.get_data([symbol], dates, colname='Low')[[symbol]]
orig_close = ut.get_data([symbol], dates, colname='Close')[[symbol]]
adj_high = high * prices[[symbol]] / orig_close
adj_low = low * prices[[symbol]] / orig_close
orders = pd.DataFrame().reindex_like(prices)
orders = orders.rename(index=str,
columns={
'Cash': 'Order',
symbol: 'Shares'
})
orders['Shares'] = 0
orders['Order'] = 'CASH'
orders.index.name = 'Date'
orders.index = pd.to_datetime(orders.index, format="%Y/%m/%d")
positions = pd.DataFrame().reindex_like(prices)
positions.fillna(0, inplace=True)
positions.iloc[0, -1] = sv
action = self.learner.querysetstate(0)
sma_range = indicators.sma(prices[[symbol]], 10).iloc[:, 0]
sma_bins = pd.qcut(sma_range, 10, labels=False)
bb_range = indicators.bb(prices[[symbol]])
bb_range['value'] = bb_range.High - bb_range.Low
bb_range.fillna(method='bfill', inplace=True)
bb_range = bb_range['value']
bb_bins = pd.qcut(bb_range, 10, labels=False)
comparisons = [
adj_high[symbol] - adj_low[symbol],
abs(adj_low[symbol] - prices[symbol].shift(1)),
abs(adj_high[symbol] - prices[symbol].shift(1))
]
tr = pd.concat(comparisons, axis=1).max(axis=1)
tr.fillna(method='bfill', inplace=True)
atr_range = tr.rolling(14, min_periods=1).mean()
atr_bins = pd.qcut(atr_range, 10, labels=False)
states = sma_bins * 100 + bb_bins * 10 + atr_bins
states = atr_bins * 100 + bb_bins * 10 + sma_bins
#states = pd.qcut(sma_range, 500, labels=False) # for experiment 1 to compare with manual strategy
pre_shares = 0
normalized_close = prices[symbol] / prices.iloc[0, 0]
daily_return = normalized_close - normalized_close.shift(1)
daily_return.fillna(method='bfill', inplace=True)
pre_orders = orders.copy()
pre_orders.iloc[0, 0] = 1000
while not orders.equals(pre_orders): # check if converges
pre_cash = sv
pre_holdings = 0
pre_orders = orders.copy()
for date, row in orders.iterrows():
cur_state = states[date]
reward = daily_return[date] * pre_holdings * (1 - self.impact)
action = self.learner.query(cur_state, reward)
orders.loc[date, 'Order'] = action
if action == 0:
orders.loc[date, 'Shares'] = -1000 - pre_shares
positions.loc[date, symbol] = -1000
if action == 1:
orders.loc[date, 'Shares'] = 0 - pre_shares
positions.loc[date, symbol] = 0
if action == 2:
orders.loc[date, 'Shares'] = 1000 - pre_shares
positions.loc[date, symbol] = 1000
positions.loc[date, 'Cash'] = pre_cash - orders.loc[
date, 'Shares'] * prices.loc[date, symbol]
pre_cash = positions.loc[date, 'Cash']
pre_holdings = positions.loc[date, symbol]
#cur_return = ((positions * prices).sum(axis=1).iloc[-1] - sv )/sv
# add your code to do learning here
# example usage of the old backward compatible util function
# syms=[symbol]
# dates = pd.date_range(sd, ed)
# prices_all = ut.get_data(syms, dates) # automatically adds SPY
# prices = prices_all[syms] # only portfolio symbols
# prices_SPY = prices_all['SPY'] # only SPY, for comparison later
# if self.verbose: print prices
# example use with new colname
# volume_all = ut.get_data(syms, dates, colname = "Volume") # automatically adds SPY
# volume = volume_all[syms] # only portfolio symbols
# volume_SPY = volume_all['SPY'] # only SPY, for comparison later
# if self.verbose: print volume
# this method should use the existing policy and test it against new data
def testPolicy(self, symbol = "JPM", \
sd=dt.datetime(2010,1,1), \
ed=dt.datetime(2011,12,31), \
sv = 100000):
dates = pd.date_range(sd, ed)
prices = ut.get_data([symbol], dates)[[symbol]]
high = ut.get_data([symbol], dates, colname='High')[[symbol]]
low = ut.get_data([symbol], dates, colname='Low')[[symbol]]
orig_close = ut.get_data([symbol], dates, colname='Close')[[symbol]]
adj_high = high * prices[[symbol]] / orig_close
adj_low = low * prices[[symbol]] / orig_close
sma_range = indicators.sma(prices[[symbol]], 10).iloc[:, 0]
sma_bins = pd.qcut(sma_range, 10, labels=False)
bb_range = indicators.bb(prices[[symbol]])
bb_range['value'] = bb_range.High - bb_range.Low
bb_range.fillna(method='bfill', inplace=True)
bb_range = bb_range['value']
bb_bins = pd.qcut(bb_range, 10, labels=False)
comparisons = [
adj_high[symbol] - adj_low[symbol],
abs(adj_low[symbol] - prices[symbol].shift(1)),
abs(adj_high[symbol] - prices[symbol].shift(1))
]
tr = pd.concat(comparisons, axis=1).max(axis=1)
tr.fillna(method='bfill', inplace=True)
atr_range = tr.rolling(14, min_periods=1).mean()
atr_bins = pd.qcut(atr_range, 10, labels=False)
states = sma_bins * 100 + bb_bins * 10 + atr_bins
states = atr_bins * 100 + bb_bins * 10 + sma_bins
#states = pd.qcut(sma_range, 1000, labels=False) # for experiment 1 to compare with manual strategy
trades = prices.copy()
pre_position = 0
for date, row in trades.iterrows():
cur_state = states[date] # compute current state
action = self.learner.querysetstate(cur_state)
if action == 0:
trades.loc[date, symbol] = -1000 - pre_position
pre_position = -1000
elif action == 1:
trades.loc[date, symbol] = 0 - pre_position
pre_position = 0
elif action == 2:
trades.loc[date, symbol] = 1000 - pre_position
pre_position = 1000
# here we build a fake set of trades
# your code should return the same sort of data
# dates = pd.date_range(sd, ed)
# prices_all = ut.get_data([symbol], dates) # automatically adds SPY
# trades = prices_all[[symbol,]] # only portfolio symbols
# trades_SPY = prices_all['SPY'] # only SPY, for comparison later
# trades.values[:,:] = 0 # set them all to nothing
# trades.values[0,:] = 1000 # add a BUY at the start
# trades.values[40,:] = -1000 # add a SELL
# trades.values[41,:] = 1000 # add a BUY
# trades.values[60,:] = -2000 # go short from long
# trades.values[61,:] = 2000 # go long from short
# trades.values[-1,:] = -1000 #exit on the last day
if self.verbose: print(type(trades)) # it better be a DataFrame!
if self.verbose: print(trades)
if self.verbose: print(prices_all)
return trades
def author(self):
return 'hwang404'
def addEvidence(self, symbol = "IBM", \
sd=dt.datetime(2008,1,1), \
ed=dt.datetime(2009,1,1), \
sv = 10000):
# add your code to do learning here
# get data
data = self.getData(sd, ed, symbol)
# get indicator data
indicator_m = MomentumIndicator(data.iloc[:, 0],
n=8).calculate_helper_data()
indicator_bb = BollingerBandIndicator(data.iloc[:, 0], 20,
2).calculate_helper_data()
indicator_s = StochasticIndicator(data, 14).calculate_helper_data()
# Get bins
self.bin_momentum = pd.qcut(indicator_m.iloc[:, 1],
4,
retbins=True,
duplicates="drop")[1]
self.bin_bollinger = pd.qcut(indicator_bb.iloc[:, 1] -
indicator_bb.iloc[:, 3],
4,
retbins=True,
duplicates="drop")[1]
self.bin_stochastic = pd.qcut(indicator_s.iloc[:, 1],
4,
retbins=True,
duplicates="drop")[1]
self.bin_price = pd.qcut(data.iloc[:, 0].diff(periods=1),
3,
retbins=True,
duplicates="drop")[1]
price_diff = data.iloc[:, 0].diff()
momentums = np.digitize(indicator_m.iloc[:, 1], self.bin_momentum)
bb_diffs = np.digitize(
indicator_bb.iloc[:, 1] - indicator_bb.iloc[:, 3],
self.bin_bollinger)
stochastics = np.digitize(indicator_s.iloc[:, 1], self.bin_stochastic)
price_features = np.digitize(price_diff, self.bin_price)
# Set first 20 input to -1 as the algo requires 20 days to get started due to rolling window for bollinger band.
momentums[:20] = -1
bb_diffs[:20] = -1
stochastics[:20] = -1
price_features[:20] = -1
# initialize learner
num_states = self.getTotalNumberOfStates()
self.learner = QLearner(num_states=num_states, \
num_actions = 3, \
alpha = 0.05, \
gamma = 0.9, \
rar = 0.99, \
radr = 0.999, \
dyna = 0, \
verbose = False)
# loop day by day
# create variables for loop
date_list = data.index
iter = 0
max_iter = 25
while iter <= max_iter:
portfolio = pd.DataFrame(np.zeros((len(date_list), 1)),
index=date_list)
stock_shares = 0
balance = sv
total_value = sv
commission = 0
holdings = 0 # -1 for short, 0 for cash, 1 for long
current_num_stocks = 0
order = 0
for i, day in enumerate(date_list):
current_strtime = day.strftime('%Y-%m-%d')
# TODO: update state and reward accordingly
if i == 0:
price_feature = 0
else:
price_feature = price_features[i]
holdings = current_num_stocks // 1000
#state = self.convertFeaturesToState(price_feature, indicator_m.loc[current_strtime], indicator_bb.loc[current_strtime], indicator_s.loc[current_strtime], holdings)
state = self.convertFeaturesToState(price_feature,
momentums[i], bb_diffs[i],
stochastics[i], holdings)
reward = self.calculateReward(total_value, order)
# Get action
action = self.learner.query(state, reward) - 1
order = 0
if holdings != action:
# long
if action == 1:
order = 1000 - current_num_stocks
current_num_stocks = 1000
# short
elif action == -1:
order = -1000 - current_num_stocks
current_num_stocks = -1000
# do nothing
elif action == 0:
order = 0 - current_num_stocks
current_num_stocks = 0
# If there are order
if order != 0:
# get order details
order_num_of_shares = order
current_symbol_price = data.loc[current_strtime].iloc[0]
balance += -1 * order_num_of_shares * current_symbol_price
stock_shares += order_num_of_shares
# Minus commission
if order_num_of_shares != 0:
balance -= commission
# Minus impact
balance -= abs(order_num_of_shares
) * current_symbol_price * self.impact
# Update portfolio with balance and current stock worth
stock_value = data.loc[current_strtime].iloc[0] * stock_shares
total_value = balance + stock_value
portfolio.loc[current_strtime] = total_value
# current_ret = (portfolio.iloc[-1] / portfolio.iloc[0]).iloc[0]
iter += 1
class StrategyLearner(object):
# constructor
def __init__(self, verbose=False, impact=0.0):
self.verbose = verbose
self.impact = impact
# this method should create a QLearner, and train it for trading
def addEvidence(self, symbol = "IBM", \
sd=dt.datetime(2008,1,1), \
ed=dt.datetime(2009,1,1), \
sv = 10000):
# add your code to do learning here
# get data
data = self.getData(sd, ed, symbol)
# get indicator data
indicator_m = MomentumIndicator(data.iloc[:, 0],
n=8).calculate_helper_data()
indicator_bb = BollingerBandIndicator(data.iloc[:, 0], 20,
2).calculate_helper_data()
indicator_s = StochasticIndicator(data, 14).calculate_helper_data()
# Get bins
self.bin_momentum = pd.qcut(indicator_m.iloc[:, 1],
4,
retbins=True,
duplicates="drop")[1]
self.bin_bollinger = pd.qcut(indicator_bb.iloc[:, 1] -
indicator_bb.iloc[:, 3],
4,
retbins=True,
duplicates="drop")[1]
self.bin_stochastic = pd.qcut(indicator_s.iloc[:, 1],
4,
retbins=True,
duplicates="drop")[1]
self.bin_price = pd.qcut(data.iloc[:, 0].diff(periods=1),
3,
retbins=True,
duplicates="drop")[1]
price_diff = data.iloc[:, 0].diff()
momentums = np.digitize(indicator_m.iloc[:, 1], self.bin_momentum)
bb_diffs = np.digitize(
indicator_bb.iloc[:, 1] - indicator_bb.iloc[:, 3],
self.bin_bollinger)
stochastics = np.digitize(indicator_s.iloc[:, 1], self.bin_stochastic)
price_features = np.digitize(price_diff, self.bin_price)
# Set first 20 input to -1 as the algo requires 20 days to get started due to rolling window for bollinger band.
momentums[:20] = -1
bb_diffs[:20] = -1
stochastics[:20] = -1
price_features[:20] = -1
# initialize learner
num_states = self.getTotalNumberOfStates()
self.learner = QLearner(num_states=num_states, \
num_actions = 3, \
alpha = 0.05, \
gamma = 0.9, \
rar = 0.99, \
radr = 0.999, \
dyna = 0, \
verbose = False)
# loop day by day
# create variables for loop
date_list = data.index
iter = 0
max_iter = 25
while iter <= max_iter:
portfolio = pd.DataFrame(np.zeros((len(date_list), 1)),
index=date_list)
stock_shares = 0
balance = sv
total_value = sv
commission = 0
holdings = 0 # -1 for short, 0 for cash, 1 for long
current_num_stocks = 0
order = 0
for i, day in enumerate(date_list):
current_strtime = day.strftime('%Y-%m-%d')
# TODO: update state and reward accordingly
if i == 0:
price_feature = 0
else:
price_feature = price_features[i]
holdings = current_num_stocks // 1000
#state = self.convertFeaturesToState(price_feature, indicator_m.loc[current_strtime], indicator_bb.loc[current_strtime], indicator_s.loc[current_strtime], holdings)
state = self.convertFeaturesToState(price_feature,
momentums[i], bb_diffs[i],
stochastics[i], holdings)
reward = self.calculateReward(total_value, order)
# Get action
action = self.learner.query(state, reward) - 1
order = 0
if holdings != action:
# long
if action == 1:
order = 1000 - current_num_stocks
current_num_stocks = 1000
# short
elif action == -1:
order = -1000 - current_num_stocks
current_num_stocks = -1000
# do nothing
elif action == 0:
order = 0 - current_num_stocks
current_num_stocks = 0
# If there are order
if order != 0:
# get order details
order_num_of_shares = order
current_symbol_price = data.loc[current_strtime].iloc[0]
balance += -1 * order_num_of_shares * current_symbol_price
stock_shares += order_num_of_shares
# Minus commission
if order_num_of_shares != 0:
balance -= commission
# Minus impact
balance -= abs(order_num_of_shares
) * current_symbol_price * self.impact
# Update portfolio with balance and current stock worth
stock_value = data.loc[current_strtime].iloc[0] * stock_shares
total_value = balance + stock_value
portfolio.loc[current_strtime] = total_value
# current_ret = (portfolio.iloc[-1] / portfolio.iloc[0]).iloc[0]
iter += 1
# this method should use the existing policy and test it against new data
def testPolicy(self, symbol = "IBM", \
sd=dt.datetime(2009,1,1), \
ed=dt.datetime(2010,1,1), \
sv = 10000):
data = self.getData(sd, ed, symbol)
# get indicator data
indicator_m = MomentumIndicator(data.iloc[:, 0],
n=8).calculate_helper_data()
indicator_bb = BollingerBandIndicator(data.iloc[:, 0], 20,
2).calculate_helper_data()
indicator_s = StochasticIndicator(data, 14).calculate_helper_data()
momentums = np.digitize(indicator_m.iloc[:, 1], self.bin_momentum)
bb_diffs = np.digitize(
indicator_bb.iloc[:, 1] - indicator_bb.iloc[:, 3],
self.bin_bollinger)
stochastics = np.digitize(indicator_s.iloc[:, 1], self.bin_stochastic)
price_features = np.digitize(data.iloc[:, 0].diff(), self.bin_price)
# Set first 20 input to -1 as the algo requires 20 days to get started due to rolling window for bollinger band.
momentums[:20] = -1
bb_diffs[:20] = -1
stochastics[:20] = -1
price_features[:20] = -1
trade_df = pd.DataFrame(index=data.index, columns=[symbol])
current_holdings = 0
action = 0
for i in range(data.shape[0]):
if i == 0:
price_feature = 0
else:
price_feature = price_features[i]
#state = self.convertFeaturesToState(price_feature, indicator_m.iloc[i],indicator_bb.iloc[i],indicator_s.iloc[i],current_holdings // 1000)
state = self.convertFeaturesToState(price_feature, momentums[i],
bb_diffs[i], stochastics[i],
current_holdings // 1000)
action = self.learner.querysetstate(state) - 1
# buy
if action == 1:
trade_df.iloc[i] = 1000 - current_holdings
current_holdings = 1000
# sell
elif action == -1:
trade_df.iloc[i] = -1000 - current_holdings
current_holdings = -1000
# do nothing
elif action == 0:
trade_df.iloc[i] = 0 - current_holdings
current_holdings = 0
# return trade_df
if self.verbose: print(type(trade_df)) # it better be a DataFrame!
if self.verbose: print(trade_df)
#if self.verbose: print(prices_all)
return trade_df
def getTotalNumberOfStates(self):
# + 1 is for the state when some values are not yet calculated
return 5 * 5 * 5 * 3 * 4 + 1
# stochastic -> Value from 0-100
# bollinger -> 3 Values. SMA + 2*STD, SMA and SMA - 2*STD
# momentum -> value around -0.5 to 0.5
# holding -> -1, 0 or 1
# Col 0 is always adjusted close
# Features starts from col 1
def convertFeaturesToState(self, price, momentum, bollinger, stochastic,
holding):
if np.isnan(momentum) or np.isnan(bollinger) or np.isnan(
stochastic
) or momentum == -1 or bollinger == -1 or stochastic == -1 or price == -1:
return self.getTotalNumberOfStates() - 1
# These values starts from 1. -1 to make it start from 0
momentum -= 1
bollinger -= 1
stochastic -= 1
price -= 1
# Holding starts from -1, +1 to make it start from 0
holding += 1
return momentum + stochastic * 5 + bollinger * 25 + holding * 125 + price * 375
def calculateReward(self, total_value, prev_day_order):
try:
return_val = (total_value - self.prev_total_value)
self.prev_total_value = total_value
return return_val
except AttributeError:
self.prev_total_value = total_value
return 0
def getData(self, start_date, end_date, symbol):
date_list = pd.date_range(start_date, end_date)
d = ut.get_data([symbol], date_list)[symbol]
close = ut.get_data([symbol], date_list, colname="Close")[symbol]
high = ut.get_data([symbol], date_list, colname="High")[symbol]
low = ut.get_data([symbol], date_list, colname="Low")[symbol]
data = pd.DataFrame(index=d.index,
columns=["Adj Close", "Close", "High", "Low"])
data.iloc[:, 0] = d
data.iloc[:, 1] = close
data.iloc[:, 2] = high
data.iloc[:, 3] = low
data = data.fillna(method="ffill")
data = data.fillna(method="bfill")
return data
# This method is to fulfill inheritance requirement for strategy
def getStrategyName(self):
return "Q Learning Strategy"
def author(self):
return "jkok7"
print('_' * 60)
print()
return p1_stats
if __name__ == "__main__":
# Example Usage
# battle(Board(show_board=True, show_result=True), RandomPlayer(), RandomPlayer(), 1, learn=False, show_result=True)
# battle(Board(), RandomPlayer(), RandomPlayer(), 100, learn=False, show_result=True)
# battle(Board(), RandomPlayer(), SmartPlayer(), 100, learn=False, show_result=True)
# battle(Board(), RandomPlayer(), PerfectPlayer(), 100, learn=False, show_result=True)
# battle(Board(), SmartPlayer(), PerfectPlayer(), 100, learn=False, show_result=True)
qlearner = QLearner()
qlearner = QLearnerXO()
NUM = qlearner.GAME_NUM
# NUM = 10
# train: play NUM games against players who only make random moves
print('Training QLearner against RandomPlayer for {} times......'.format(
NUM))
board = Board()
qlearner.epsilon = 0.5
qlearner.alpha = 0.5
qlearner.varyA_E = True
# battle(board, RandomPlayer(), qlearner, NUM, learn=True, show_result=False)
# print(qlearner.epsilon, qlearner.alpha)
# qlearner.epsilon = 0
from poker_learner import poker_learner
from QLearner import QLearner
from poker_learner import poker_learner
learners = []
for i in range(4):
f = open("./learners/" + str(i), 'r')
learners.append(QLearner())
learners[i].Q = eval(f.read())
f.close()
p = poker_learner(learners=learners)
def Qlearning_Rtest():
''' Testing function for Q learning on R-track'''
q = QLearner(0.5, 0.9, 0.9, "R", True)
q.track.show()
q.train((23, 11, 1, -1))
q.train((12, 17, 1, -1))
q.train((13, 17, 1, -1))
q.train((21, 25, 0, -1))
q.train((21, 23, -1, -1))
q.train((5, 24, 1, 0))
q.train((4, 23, 1, 1))
q.train()
q.train()
for i in range(10):
print(q.trial_run())
class StrategyLearner(object):
# constructor
def __init__(self, verbose=False, impact=0.0):
self.verbose = verbose
self.impact = impact
self.learner = QLearner(num_states=90000, num_actions=2, dyna=0)
self.mom_mean = 0
self.mom_std = 0
self.long_mom_mean = 0
self.long_mom_std = 0
self.bollinger_bands = None
self.rsi = None
self.momentum = None
self.cr = -100000
def author(self):
return 'nlerner3'
# this method should create a QLearner, and train it for trading
def addEvidence(self, symbol = "IBM", \
sd=dt.datetime(2008,1,1), \
ed=dt.datetime(2009,1,1), \
sv = 10000):
self.cash = sv
start = sd - dt.timedelta(days=60)
dates = pd.date_range(start, ed)
data = ut.get_data([symbol], dates)
data.fillna(method="ffill", inplace=True)
data.fillna(method="bfill", inplace=True)
prices = indicators.normalized_prices(data[symbol])
self.bollinger_bands = None
self.momentum = None
self.rsi = indicators.rolling_rsi(prices, 5)
self.bollinger_bands = indicators.bollinger_bands(prices, 15)
self.momentum = indicators.rolling_momentum(prices, 2)
self.mom_mean = self.momentum.mean()
self.mom_std = self.momentum.std()
last_action = 2
current_state = self.create_state(sd, prices[:sd], last_action)
action = self.learner.querysetstate(current_state)
trades = pd.DataFrame(index=prices[sd:].index, columns=[symbol])
keepgoing = True
iteration = 0
while keepgoing:
for i in trades.index:
reward = self.get_reward(action, last_action, prices[:i])
last_action = action
current_state = self.create_state(i, prices[:i], last_action)
action = self.learner.query(current_state, reward)
cr = self.cash / sv - 1
if cr == self.cr or iteration == 20:
keepgoing = False
self.cr = cr
iteration += 1
def get_reward(self, action, last_action, prices_to_date):
delta = prices_to_date.ix[-1] - prices_to_date.ix[-2]
gains = 0
if action == 0:
gains = -1000 * delta
if action == 1:
gains += 1000 * delta
if action != last_action:
gains = gains - (1000 * prices_to_date.ix[-1] * self.impact)
self.cash = self.cash + gains
return gains
def create_state(self, date, prices_to_date, prev_action):
rsi = self.rsi_state(date)
mom = self.momentum_state(date)
bollinger = self.bollinger_state(date, prices_to_date)
return int("{}{}{}{}{}".format(bollinger, rsi[0], rsi[1], mom,
prev_action))
def rsi_state(self, date):
rsi = self.rsi[:date]
current_rsi = rsi[-1]
last_rsi = rsi[-2]
rsi_0 = int(current_rsi / 10)
if rsi_0 > 9:
rsi_0 = 9
rsi_1 = int(last_rsi / 10)
if rsi_1 > 9:
rsi_1 = 9
return (rsi_0, rsi_1)
def momentum_state(self, date):
rolling_momentum = self.momentum[:date][-1]
zscore = (rolling_momentum - self.mom_mean) / self.mom_std
state = int(round(zscore * 3 + 5))
if state < 0:
state = 0
if state > 9:
state = 9
return state
def bollinger_state(self, date, prices_to_date):
roling_band = self.bollinger_bands[:date]
current_price = prices_to_date[-1]
last_price = prices_to_date[-2]
current_band = roling_band.ix[-1]
last_band = roling_band.ix[-2]
current_price_state = 1
last_price_state = 1
if current_price > current_band['UPPER_BAND']:
current_price_state += 1
if current_price < current_band['LOWER_BAND']:
current_price_state -= 1
if last_price > last_band['UPPER_BAND']:
last_price_state += 1
if last_price < last_band['LOWER_BAND']:
last_price_state -= 1
# return (current_price_state, last_price_state)
return current_price_state * 3 + last_price_state
# this method should use the existing policy and test it against new data
def testPolicy(self, symbol = "IBM", \
sd=dt.datetime(2009,1,1), \
ed=dt.datetime(2010,1,1), \
sv = 10000):
# here we build a fake set of trades
# your code should return the same sort of data
start = sd - dt.timedelta(days=60)
dates = pd.date_range(start, ed)
data = ut.get_data([symbol], dates)
data.fillna(method="ffill", inplace=True)
data.fillna(method="bfill", inplace=True)
prices = indicators.normalized_prices(data[symbol])
trades = pd.DataFrame(index=prices[sd:].index, columns=[symbol])
self.rsi = indicators.rolling_rsi(prices, 5)
self.bollinger_bands = indicators.bollinger_bands(prices, 15)
self.momentum = indicators.rolling_momentum(prices, 2)
action = 2
holding = 0
for i in range(len(trades)):
prices_to_date = prices[:trades.index[i]]
state = self.create_state(trades.index[i], prices_to_date, action)
action = self.learner.querysetstate(state)
if action is 0:
trades.ix[i] = -1000 - holding
holding = -1000
if action is 1:
trades.ix[i] = 1000 - holding
holding = 1000
if action is 2:
trades.ix[i] = 0
return trades
class StrategyLearner(object):
# constructor
def __init__(self, verbose=False, impact=0.0, dyna=200, epochs=3):
self.verbose = verbose
self.impact = impact
self._z_params = dict()
self._state_dict = dict()
self._learner = None
self._state_order = []
self._actions = [-1, 0, 1] # long 1, cash 0, short -1
self._dyna = dyna
self._epochs = epochs
# this method should create a QLearner, and train it for trading
def addEvidence(self, symbol="IBM",
sd=dt.datetime(2008, 1, 1),
ed=dt.datetime(2009, 1, 1),
sv=10000, n=1):
# add your code to do learning here
# Get price info
syms = [symbol]
adj_sd = sd - timedelta(days=30)
dates = pd.date_range(adj_sd, ed)
prices_all = ut.get_data(syms, dates) # automatically adds SPY
prices = prices_all[syms] # only portfolio symbols
# Calculate indicators
df = add_all_indicators(prices, syms[0], add_helper_data=False)
# Filter to time range
df = df.loc[sd:ed, :].copy()
# Get state df
state_df = self.fit_transform_state(df, symbol, n_days=n)
# Initialize Q-learner
self._learner = QLearner(num_states=len(self._state_dict), num_actions=len(self._actions), dyna=self._dyna)
# Fit learner
for _ in range(self._epochs):
self.fit_learner(state_df)
return
# this method should use the existing policy and test it against new data
def testPolicy(self, symbol="IBM",
sd=dt.datetime(2009, 1, 1),
ed=dt.datetime(2010, 1, 1),
sv=10000):
# Get price info
syms = [symbol]
adj_sd = sd - timedelta(days=30)
dates = pd.date_range(adj_sd, ed)
prices_all = ut.get_data(syms, dates) # automatically adds SPY
prices = prices_all[syms] # only portfolio symbols
# Calculate indicators
df = add_all_indicators(prices, syms[0], add_helper_data=False)
# Filter to time range
df = df.loc[sd:ed, :].copy()
# Get state df
state_df = self.transform_state(df)
trades_df = self.test_learner(state_df, symbol)
return trades_df
def fit_learner(self, state_df):
# Single iteration through training data
# Mix the order up so dyna doesn't overfit to early experiences
# return nothing
i = 0
state = list(state_df.iloc[i].loc[self._state_order])
position = self._actions[1] # Action cash is position 0 at index 1
state.append(position)
state_num = self._state_dict[tuple(state)]
action = self._learner.querysetstate(state_num)
for i in range(1, state_df.shape[0]):
orig_position = position
# Calculate reward
position = self._actions[action]
reward = state_df.iloc[i]["reward"] * position
# Adjust reward for impact
# Depending on original position...
# If the position changes, reduce reward by impact
if orig_position != position:
reward = reward - (abs(reward) * self.impact)
# Calculate reward
# Long (1) and price goes down...
# return * 1
# Long (1) and price goes up...
# return * 1
# Cash (0) and price goes down...
# return * -1
# Cash (0) and price goes up...
# return * -1
# Short (-1) and price goes down...
# return * -1
# Short (-1) and price goes up...
# return * -1
# Update state
state = list(state_df.loc[:, self._state_order].iloc[i])
state.append(position)
state_num = self._state_dict[tuple(state)]
# Query with new state and reward for last action
action = self._learner.query(state_num, reward)
return
def test_learner(self, state_df, symbol):
# Test Q-Learner in sample for cumulative return (create trades df)
actions_list = []
i = 0
state = list(state_df.iloc[i].loc[self._state_order])
position = self._actions[1] # Action cash is position 0 at index 1
state.append(position)
state_num = self._state_dict[tuple(state)]
action = self._learner.querysetstate(state_num)
actions_list.append(self._actions[action])
for i in range(state_df.shape[0]):
position = self._actions[action]
# Update state
state = list(state_df.loc[:, self._state_order].iloc[i])
state.append(position)
state_num = self._state_dict[tuple(state)]
action = self._learner.querysetstate(state_num)
actions_list.append(self._actions[action])
trades_df = pd.DataFrame(index=state_df.index, data=actions_list[:-1], columns=["action"])
trades_df[symbol] = 0
current_position = 0
for i in trades_df.index:
action = trades_df.loc[i, "action"]
if action == current_position:
trades_df.loc[i, symbol] = 0
elif action == -1 and current_position == 1:
trades_df.loc[i, symbol] = -2000
elif action == 1 and current_position == -1:
trades_df.loc[i, symbol] = 2000
elif action == -1 and current_position == 0:
trades_df.loc[i, symbol] = -1000
elif action == 1 and current_position == 0:
trades_df.loc[i, symbol] = 1000
elif action == 0 and current_position == 1:
trades_df.loc[i, symbol] = -1000
elif action == 0 and current_position == -1:
trades_df.loc[i, symbol] = 1000
else:
raise ValueError("Impossible")
current_position = action
return trades_df.drop("action", axis=1)
def fit_transform_state(self, df, symbol, n_days):
# Symbol specifies the price column name
# Accepts df with price and indicators
# Digitize indicators
# Store z parameters for later use
# Calculate reward
# Create state dict for mapping digitized state to integer
# Return state_df
# Create state DF
state_df = pd.DataFrame(index=df.index)
# Add and digitize indicators
# Calculate target: n-day future return
state_df["reward"] = df[symbol].iloc[::-1].rolling(window=n_days + 1).apply(lambda x: (x[0] / x[-1]) - 1).iloc[::-1]
state_values = []
# How many states are there, and how do I map them to a single integer?
# unique_boll = range(state_df["bollinger_band"].max() + 1)
# Add bollinger band
state_df["bollinger_band"], unique_n = self.digitize_bollinger(df["bollinger_band"])
unique_boll = list(range(unique_n))
state_values.append(unique_boll)
self._state_order.append("bollinger_band")
# Add divergence
state_df["divergence"], unique_n = self.digitize_divergence(df["divergence"])
unique_div = list(range(unique_n))
state_values.append(unique_div)
self._state_order.append("divergence")
# Add momentum
state_df["momentum"], unique_n = self.digitize_momentum(df["momentum"])
unique_mom = list(range(unique_n))
state_values.append(unique_mom)
self._state_order.append("momentum")
# Add D
state_df["D"], unique_n = self.digitize_d(df["D"])
unique_d = list(range(unique_n))
state_values.append(unique_d)
self._state_order.append("D")
state_values.append(self._actions)
# Drop NA's, without reward
state_df = state_df.dropna()
all_states = list(itertools.product(*state_values))
self._state_dict = {k: v for v, k in enumerate(all_states)}
return state_df
def transform_state(self, df):
# Accepts df with indicators
# Digitize indicators using existing params
# Return state_df
# Create state DF
state_df = pd.DataFrame(index=df.index)
state_df["bollinger_band"], _ = self.digitize_bollinger(df["bollinger_band"])
state_df["divergence"], _ = self.digitize_divergence(df["divergence"])
state_df["momentum"], _ = self.digitize_momentum(df["momentum"])
state_df["D"], _ = self.digitize_d(df["D"])
return state_df
@staticmethod
def digitize_bollinger(indicator):
bins = [-1.03, 0, 1.03]
return np.digitize(indicator, bins), len(bins) + 1
@staticmethod
def digitize_divergence(indicator):
bins = [-.25, 0, .25]
return np.digitize(indicator, bins), len(bins) + 1
@staticmethod
def digitize_momentum(indicator):
bins = [-.27, .27]
return np.digitize(indicator, bins), len(bins) + 1
@staticmethod
def digitize_d(indicator):
bins = [.2, .8]
return np.digitize(indicator, bins), len(bins) + 1
def author(self):
return 'cfarr31'
def test_lunar_lander(env,
framework="pytorch",
repeat=10,
hidden_layer_dimensions: List[int] = [128, 64],
test_episode_count=100,
alpha=1e-4,
gamma=0.99,
epsilon_start=1.0,
epsilon_decay=0.998,
epsilon_min=0.0,
replay_memory_size=2**16,
replay_sample_size=32,
model_saving_folder=join('.', "models"),
model_file=None,
render=False):
if model_file:
files = [join(model_saving_folder, model_file)]
else:
_, files = _get_file_path(
folder_path=model_saving_folder,
framework=framework,
extension_name='.mod',
hidden_layer_dimensions=hidden_layer_dimensions,
mean_reward=None,
alpha=alpha,
gamma=gamma,
epsilon_start=epsilon_start,
epsilon_decay=epsilon_decay,
epsilon_min=epsilon_min,
replay_memory_size=replay_memory_size,
replay_sample_size=replay_sample_size,
use_dropout=False,
timestamp=None)
if files:
dimensions = _get_dimensions(env, hidden_layer_dimensions)
logs = True
for file in files:
mean_rewards = [None] * repeat
start_episode_idx = 0
print("==========test model '{}'==========".format(basename(file)))
for repeat_idx in range(repeat):
q_fun = _get_qfun(framework, dimensions, alpha, file)
lunar_lander = QLearner(env=env,
q_fun=q_fun,
epsilon_decay=epsilon_decay,
epsilon_min=epsilon_min,
gamma=gamma)
mean_reward, logs = lunar_lander.test(
start_episode_idx=start_episode_idx,
episode_count=test_episode_count,
continued_learning=False,
logging=logs,
render=render)
start_episode_idx += test_episode_count
mean_rewards[repeat_idx] = mean_reward
logs.to_csv(get_path_without_extension(file) +
'_test{}_tmr{}.csv'.format(
int(time()), int(sum(mean_rewards) / repeat)),
index=False)
def addEvidence(self, symbol = "IBM", \
sd=dt.datetime(2008,1,1), \
ed=dt.datetime(2009,1,1), \
sv = 10000):
# add your code to do learning here
# example usage of the old backward compatible util function
syms = [symbol]
dates = pd.date_range(sd, ed)
prices_all = ut.get_data(syms, dates) # automatically adds SPY
prices = prices_all[syms] # only portfolio symbols
prices_SPY = prices_all['SPY'] # only SPY, for comparison later
if self.verbose: print prices
# example use with new colname
volume_all = ut.get_data(syms, dates,
colname="Volume") # automatically adds SPY
volume = volume_all[syms] # only portfolio symbols
volume_SPY = volume_all['SPY'] # only SPY, for comparison later
if self.verbose: print volume
# feature engineering, get your new features for the state space.
state_bolli, state_bandwidth, state_momentum, state_rsi = self.add_indicators(
prices, save_data=self.save_data, symb=symbol)
# compute the current state, well really all the states.
states = self.discretize(state_bolli, state_bandwidth, state_momentum,
state_rsi)
number_of_bins = self.nbins
self.learner = QLearner(num_states=(number_of_bins ** 4), \
num_actions = 3, \
alpha = 0.5, \
gamma = 0.9, \
rar = 0.0, \
radr = 0.0, \
dyna = 0, \
verbose = False)
converged = False
count = 0
converged_yet = 0.
while (not converged) and (count < 30):
total_reward = 0
prices_dataframe, df_trades = self.init_trades_and_prices_df(
prices)
holdings = 0
for i in range(state_bolli.shape[0]):
current_state = states[i]
action = self.learner.querysetstate(current_state)
# first day
reward = self.compute_last_reward(prices, holdings, action, i)
# get the current state and the reward to get an action.
action = self.learner.query(current_state, reward)
diggity_day = prices.index[i]
if action == 0:
df_trades, holdings = self.compute_trades_and_holdings_sell(
df_trades, prices_dataframe, symbol, holdings,
diggity_day)
if action == 1:
df_trades, holdings = self.compute_trades_and_holdings_buy(
df_trades, prices_dataframe, symbol, holdings,
diggity_day)
total_reward += reward
cumulative_return = self.compute_cumulative_return(
prices, prices_dataframe, df_trades, sv)
count += 1
if abs(converged_yet - cumulative_return) * 100. < 0.0001:
converged = True
else:
converged_yet = cumulative_return
return pd.DataFrame(df_trades[symbol])
class StrategyLearner(object):
# constructor
def __init__(self, verbose=False, impact=0.0, save_data=True):
self.verbose = verbose
self.impact = impact
self.nbins = 5
self.save_data = save_data
def author(self):
return "mdunn34"
def bollinger(self, prices, window_n=20, k=2):
std = prices.rolling(window=window_n).std()
mean = prices.rolling(window=window_n).mean()
upper_bound = mean + (k * std)
lower_bound = mean - (k * std)
band = (prices - lower_bound) / (upper_bound - lower_bound)
normed_band = (band - (band).mean()) / (band.std())
bandwidth = (upper_bound - lower_bound) / mean * 100
normed_band = normed_band.fillna(method="bfill")
bandwidth = bandwidth.fillna(method="bfill")
return normed_band, bandwidth
def add_indicators(self,
prices,
window_n=5,
n_bins=5,
save_data=False,
symb=None):
n_bins = self.nbins
prices_new = prices.copy()
def plot_data(df,
type,
title="Stock prices",
xlabel="Date",
ylabel="Value",
symb=None):
"""Plot stock prices with a custom title and meaningful axis labels."""
ax = df.plot(title=title, fontsize=12)
ax.set_xlabel(xlabel)
ax.set_ylabel(ylabel)
plt.savefig('{}_{}.png'.format(type, str(symb)))
for symb in prices.columns:
#bollinger bands here.
bolli, bandwidth = self.bollinger(prices[symb], window_n=window_n)
momentum = ((prices[symb] / prices[symb].shift(-1)) *
100).fillna(method="ffill")
# rsi
difference = prices[symb].diff()
diff_up, diff_down = difference.copy(), difference.copy()
diff_up[diff_up < 0] = 0
diff_down[diff_down > 0] = 0
rsi = 100. - (100. /
(1. +
(diff_up.rolling(window=window_n).mean() /
diff_down.rolling(window=window_n).mean().abs())))
rsi = rsi.fillna(method="bfill")
if save_data == True:
print("Correlation between RSI and Momentum",
np.corrcoef(rsi, momentum))
plot_data(bandwidth,
type='bandwidth',
title='Bandwidth Values')
plt.clf()
plot_data(bolli,
type='bollinger',
title='Normed Bollinger Values')
plt.clf()
plot_data(momentum, type='momentum', title='Momentum Values')
plt.clf()
plot_data(rsi, type='rsi', title='RSI Values')
plt.clf()
bolli = pd.cut(bolli, bins=n_bins, labels=False)
bandwidth = pd.cut(bandwidth, bins=n_bins, labels=False)
momentum = pd.cut(momentum, bins=n_bins, labels=False)
rsi = pd.cut(rsi, bins=n_bins, labels=False)
return np.asarray(bolli), np.asarray(bandwidth), np.asarray(
momentum), np.asarray(rsi)
def discretize(self, state_bolli, state_bandwidth, state_momentum,
state_rsi):
return (state_bolli) + (state_bandwidth**2) + (state_momentum**
3) + (state_rsi**4)
def init_trades_and_prices_df(self, prices):
prices_dataframe = prices.copy()
prices_dataframe['Cash'] = np.ones(prices.shape[0])
df_trades = pd.DataFrame(np.zeros(prices_dataframe.shape),
index=prices_dataframe.index,
columns=prices_dataframe.columns)
return prices_dataframe, df_trades
def compute_last_reward(self, prices, holdings, action, i):
if i == 0:
reward = 0
else:
# compute the daily return.
daily_return = (
(prices.iloc[i] - prices.iloc[i - 1]) / prices.iloc[i]) * 100
if holdings != 0:
# Sell
if action == 0:
reward = -1.0 * daily_return
# Buy
elif action == 1:
reward = daily_return
else:
reward = -9
else:
reward = 0
return reward
def compute_trades_and_holdings_buy(self, df_trades, prices_dataframe,
symbol, holdings, diggity_day):
if holdings == -1000:
df_trades["Cash"].loc[diggity_day] = df_trades["Cash"].loc[
diggity_day] + 2000. * prices_dataframe[symbol].loc[
diggity_day] * -1.
df_trades[symbol].loc[
diggity_day] = df_trades[symbol].loc[diggity_day] + 2000.
holdings = holdings + 2000.
if holdings == 0:
df_trades["Cash"].loc[diggity_day] = df_trades["Cash"].loc[
diggity_day] + 1000. * prices_dataframe[symbol].loc[
diggity_day] * -1.
df_trades[symbol].loc[
diggity_day] = df_trades[symbol].loc[diggity_day] + 1000.
holdings = holdings + 1000.
return df_trades, holdings
def compute_trades_and_holdings_sell(self, df_trades, prices_dataframe,
symbol, holdings, diggity_day):
if holdings == 1000:
df_trades["Cash"].loc[diggity_day] = df_trades["Cash"].loc[
diggity_day] + 2000.0 * prices_dataframe[symbol].loc[
diggity_day]
df_trades[symbol].loc[
diggity_day] = df_trades[symbol].loc[diggity_day] - 2000.
holdings = holdings - 2000
if holdings == 0:
df_trades["Cash"].loc[diggity_day] = df_trades["Cash"].loc[
diggity_day] + 1000. * prices_dataframe[symbol].loc[diggity_day]
df_trades[symbol].loc[
diggity_day] = df_trades[symbol].loc[diggity_day] - 1000.0
holdings = holdings - 1000.
return df_trades, holdings
def compute_cumulative_return(self, prices, prices_dataframe, df_trades,
sv):
holdings_df = pd.DataFrame(np.zeros(df_trades.shape),
columns=df_trades.columns,
index=df_trades.index)
values_df = holdings_df.copy()
first_diggity_day = prices.index[0]
holdings_df["Cash"].loc[first_diggity_day] = sv
holdings_df.loc[first_diggity_day] = holdings_df.loc[
first_diggity_day] + df_trades.loc[first_diggity_day]
holdings_df = holdings_df.cumsum()
values_df = (holdings_df * prices_dataframe)
df_port_val = values_df.sum(axis=1)
cumulative_return = (df_port_val.iloc[-1] - sv) / sv
return cumulative_return
# this method should create a QLearner, and train it for trading
def addEvidence(self, symbol = "IBM", \
sd=dt.datetime(2008,1,1), \
ed=dt.datetime(2009,1,1), \
sv = 10000):
# add your code to do learning here
# example usage of the old backward compatible util function
syms = [symbol]
dates = pd.date_range(sd, ed)
prices_all = ut.get_data(syms, dates) # automatically adds SPY
prices = prices_all[syms] # only portfolio symbols
prices_SPY = prices_all['SPY'] # only SPY, for comparison later
if self.verbose: print prices
# example use with new colname
volume_all = ut.get_data(syms, dates,
colname="Volume") # automatically adds SPY
volume = volume_all[syms] # only portfolio symbols
volume_SPY = volume_all['SPY'] # only SPY, for comparison later
if self.verbose: print volume
# feature engineering, get your new features for the state space.
state_bolli, state_bandwidth, state_momentum, state_rsi = self.add_indicators(
prices, save_data=self.save_data, symb=symbol)
# compute the current state, well really all the states.
states = self.discretize(state_bolli, state_bandwidth, state_momentum,
state_rsi)
number_of_bins = self.nbins
self.learner = QLearner(num_states=(number_of_bins ** 4), \
num_actions = 3, \
alpha = 0.5, \
gamma = 0.9, \
rar = 0.0, \
radr = 0.0, \
dyna = 0, \
verbose = False)
converged = False
count = 0
converged_yet = 0.
while (not converged) and (count < 30):
total_reward = 0
prices_dataframe, df_trades = self.init_trades_and_prices_df(
prices)
holdings = 0
for i in range(state_bolli.shape[0]):
current_state = states[i]
action = self.learner.querysetstate(current_state)
# first day
reward = self.compute_last_reward(prices, holdings, action, i)
# get the current state and the reward to get an action.
action = self.learner.query(current_state, reward)
diggity_day = prices.index[i]
if action == 0:
df_trades, holdings = self.compute_trades_and_holdings_sell(
df_trades, prices_dataframe, symbol, holdings,
diggity_day)
if action == 1:
df_trades, holdings = self.compute_trades_and_holdings_buy(
df_trades, prices_dataframe, symbol, holdings,
diggity_day)
total_reward += reward
cumulative_return = self.compute_cumulative_return(
prices, prices_dataframe, df_trades, sv)
count += 1
if abs(converged_yet - cumulative_return) * 100. < 0.0001:
converged = True
else:
converged_yet = cumulative_return
return pd.DataFrame(df_trades[symbol])
# this method should use the existing policy and test it against new data
def testPolicy(self, symbol = "IBM", \
sd=dt.datetime(2008,1,1), \
ed=dt.datetime(2009,1,1), \
sv = 10000):
# here we build a fake set of trades
# your code should return the same sort of data
dates = pd.date_range(sd, ed)
prices_all = ut.get_data([symbol], dates) # automatically adds SPY
prices = prices_all[[
symbol,
]] # only portfolio symbols
trades_SPY = prices_all['SPY'] # only SPY, for comparison later
"""
trades.values[:,:] = 0 # set them all to nothing
trades.values[0,:] = 1000 # add a BUY at the start
trades.values[40,:] = -1000 # add a SELL
trades.values[41,:] = 1000 # add a BUY
trades.values[60,:] = -2000 # go short from long
trades.values[61,:] = 2000 # go long from short
trades.values[-1,:] = -1000 #exit on the last day
if self.verbose: print type(trades) # it better be a DataFrame!
if self.verbose: print trades
if self.verbose: print prices_all
"""
prices_dataframe, df_trades = self.init_trades_and_prices_df(prices)
holdings = 0
state_bolli, state_bandwidth, state_momentum, state_rsi = self.add_indicators(
prices)
states = self.discretize(state_bolli, state_bandwidth, state_momentum,
state_rsi)
for i in range(state_bolli.shape[0]):
current_state = states[i]
action = self.learner.querysetstate(current_state)
reward = self.compute_last_reward(prices, holdings, action, i)
diggity_day = prices.index[i]
if action == 0:
df_trades, holdings = self.compute_trades_and_holdings_sell(
df_trades, prices_dataframe, symbol, holdings, diggity_day)
if action == 1:
df_trades, holdings = self.compute_trades_and_holdings_buy(
df_trades, prices_dataframe, symbol, holdings, diggity_day)
return pd.DataFrame(df_trades[symbol])
def Qlearning_Otest():
''' Testing function for Q learning on O-Track'''
q = QLearner(0.5, 0.9, 0.9, "O")
q.track.show()
q.train((3, 4, 0, 1))
q.train((4, 3, 0, 1))
q.train((20, 2, -1, 0))
q.train((21, 4, -1, -1))
q.train((20, 20, 0, -1))
q.train((18, 22, 1, -1))
q.train((4, 22, 1, 0))
q.train((3, 20, 1, 1))
q.train()
for i in range(10):
print(q.trial_run())
print('Total reward={r} over {n} iterations'.format(r=sum(rewards), n=iterations))
plt.figure()
plt.plot(range(1,iterations+1), reward_iter)
plt.show()
if __name__ == '__main__':
SDP = firemdp.solveMDP()
print("Finite Horizon")
firemdp.printPolicy(SDP.policy[:, 0])
fm = ForestManagement()
vi, vi_policy, _, _ = fm.value_iteration()
pi, pi_policy, _, _ = fm.policy_iteration()
print("Value Iteration")
firemdp.printPolicy(vi_policy)
# fm.eval_policy(vi_policy,iterations=1000)
print("Policy Iteration")
firemdp.printPolicy(pi_policy)
# fm.eval_policy(pi_policy, iterations=1000)
ql = QLearner(fm.name, fm.prob, fm.rewards)
ql.q_learning_trials(trials=20, vi=vi, pi=pi)
run_stats,ql_policy = ql.q_learning(gamma=0.9911, alpha=0.3695, alpha_decay=0.9998, alpha_min=0.0747,
epsilon=0.8608, epsilon_decay=0.9996, n_iter=47366,returnStats=True)
print("QLearning Policy")
firemdp.printPolicy(ql_policy)
# fm.eval_policy(ql_policy, iterations=1000)
sys.exit()