def __init__(self, env, n_episodes=3000, max_env_steps=None, gamma=0.9,
epsilon=0.5, epsilon_min=0.05, epsilon_log_decay=0.001, alpha=1e-3,
memory_size=10000, batch_size=256, c=10, hidden_layers=2, hidden_size=24,
render=False, debug=False):
self.memory = ReplayMemory(capacity=memory_size)
self.env = env
# hyper-parameter setting
self.gamma = gamma
self.epsilon = epsilon
self.epsilon_min = epsilon_min
self.epsilon_decay = epsilon_log_decay
self.alpha = alpha
self.n_episodes = n_episodes
self.batch_size = batch_size
self.c = c
if max_env_steps is not None:
self.env._max_episode_steps = max_env_steps
self.observation_space_size = env.observation_space.shape[0]
self.action_space_size = env.action_space.n
self.render = render
self.debug = debug
if debug:
self.loss_list = []
# if gpu is to be used
self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# Init model 1
# Use the nn package to define our model as a sequence of layers. nn.Sequential
# is a Module which contains other Modules, and applies them in sequence to
# produce its output. Each Linear Module computes output from input using a
# linear function, and holds internal Tensors for its weight and bias.
# After constructing the model we use the .to() method to move it to the
# desired device.
self.model = DuelingNet(self.observation_space_size, self.action_space_size, hidden_layers, hidden_size) \
.to(self.device)
self.target = DuelingNet(self.observation_space_size, self.action_space_size, hidden_layers, hidden_size) \
.to(self.device)
self.target.load_state_dict(self.model.state_dict())
self.target.eval()
self.model.train()
# The nn package also contains definitions of popular loss functions; in this
# case we will use Mean Squared Error (MSE) as our loss function. Setting
# reduction='sum' means that we are computing the *sum* of squared errors rather
# than the mean; this is for consistency with the examples above where we
# manually compute the loss, but in practice it is more common to use mean
# squared error as a loss by setting reduction='elementwise_mean'.
self.loss_fn = torch.nn.MSELoss()
# Use the optim package to define an Optimizer that will update the weights of
# the model for us. Here we will use Adam; the optim package contains many other
# optimization algoriths. The first argument to the Adam constructor tells the
# optimizer which Tensors it should update.
self.optimizer = torch.optim.Adam(self.model.parameters(), lr=alpha)
def __init__(self, env, model, target, double=False, n_episodes=3000, max_env_steps=None, gamma=0.9,
epsilon=0.5, epsilon_min=0.05, epsilon_log_decay=0.001, alpha=1e-3,
memory_size=10000, batch_size=256, c=10, hidden_layers=1, hidden_size=6,
render=False, debug=False):
self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
self.model = model.to(self.device)
self.target = target.to(self.device)
self.target.load_state_dict(self.model.state_dict())
self.target.eval()
self.model.train()
self.double = double
self.memory = ReplayMemory(capacity=memory_size)
self.env = env
self.gamma = gamma
self.epsilon = epsilon
self.epsilon_min = epsilon_min
self.epsilon_decay = epsilon_log_decay
self.alpha = alpha
self.n_episodes = n_episodes
self.batch_size = batch_size
self.c = c
if max_env_steps is not None:
self.env._max_episode_steps = max_env_steps
self.render = render
self.debug = debug
if debug:
self.loss_list = []
self.loss_fn = torch.nn.MSELoss()
self.optimizer = torch.optim.Adam(self.model.parameters(), lr=alpha)
class DuelingDQN:
def __init__(self, env, n_episodes=3000, max_env_steps=None, gamma=0.9,
epsilon=0.5, epsilon_min=0.05, epsilon_log_decay=0.001, alpha=1e-3,
memory_size=10000, batch_size=256, c=10, hidden_layers=2, hidden_size=24,
render=False, debug=False):
self.memory = ReplayMemory(capacity=memory_size)
self.env = env
# hyper-parameter setting
self.gamma = gamma
self.epsilon = epsilon
self.epsilon_min = epsilon_min
self.epsilon_decay = epsilon_log_decay
self.alpha = alpha
self.n_episodes = n_episodes
self.batch_size = batch_size
self.c = c
if max_env_steps is not None:
self.env._max_episode_steps = max_env_steps
self.observation_space_size = env.observation_space.shape[0]
self.action_space_size = env.action_space.n
self.render = render
self.debug = debug
if debug:
self.loss_list = []
# if gpu is to be used
self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# Init model 1
# Use the nn package to define our model as a sequence of layers. nn.Sequential
# is a Module which contains other Modules, and applies them in sequence to
# produce its output. Each Linear Module computes output from input using a
# linear function, and holds internal Tensors for its weight and bias.
# After constructing the model we use the .to() method to move it to the
# desired device.
self.model = DuelingNet(self.observation_space_size, self.action_space_size, hidden_layers, hidden_size) \
.to(self.device)
self.target = DuelingNet(self.observation_space_size, self.action_space_size, hidden_layers, hidden_size) \
.to(self.device)
self.target.load_state_dict(self.model.state_dict())
self.target.eval()
self.model.train()
# The nn package also contains definitions of popular loss functions; in this
# case we will use Mean Squared Error (MSE) as our loss function. Setting
# reduction='sum' means that we are computing the *sum* of squared errors rather
# than the mean; this is for consistency with the examples above where we
# manually compute the loss, but in practice it is more common to use mean
# squared error as a loss by setting reduction='elementwise_mean'.
self.loss_fn = torch.nn.MSELoss()
# Use the optim package to define an Optimizer that will update the weights of
# the model for us. Here we will use Adam; the optim package contains many other
# optimization algoriths. The first argument to the Adam constructor tells the
# optimizer which Tensors it should update.
self.optimizer = torch.optim.Adam(self.model.parameters(), lr=alpha)
def choose_action(self, state, epsilon):
"""Chooses the next action according to the model trained and the policy"""
# exploits the current knowledge if the random number > epsilon, otherwise explores
if np.random.random() <= epsilon:
return self.env.action_space.sample()
else:
with torch.no_grad():
q = self.model(state)
argmax = torch.argmax(q)
return argmax.item()
def get_epsilon(self, episode):
"""Returns an epsilon that decays over time until a minimum epsilon value is reached; in this case the minimum
value is returned"""
return max(self.epsilon_min, self.epsilon * math.exp(-self.epsilon_decay * episode))
def replay(self):
"""Previously stored (s, a, r, s') tuples are replayed (that is, are added into the model). The size of the
tuples added is determined by the batch_size parameter"""
transitions, _ = self.memory.sample(self.batch_size)
# Transpose the batch (see https://stackoverflow.com/a/19343/3343043 for
# detailed explanation). This converts batch-array of Transitions
# to Transition of batch-arrays.
batch = Transition(*zip(*transitions))
# Compute a mask of non-final states and concatenate the batch elements
# (a final state would've been the one after which simulation ended)
non_final_next_states = torch.stack([s for s in batch.next_state if s is not None])
non_final_mask = torch.stack(batch.done)
state_batch = torch.stack(batch.state)
action_batch = torch.stack(batch.action)
reward_batch = torch.stack(batch.reward)
# Compute Q(s_t, a) - the model computes Q(s_t), then we select the
# columns of actions taken. These are the actions which would've been taken
# for each batch state according to policy_net
state_action_values = self.model(state_batch).gather(1, action_batch)
# Compute V(s_{t+1}) for all next states.
# Expected values of actions for non_final_next_states are computed based
# on the "older" target_net; selecting their best reward with max(1)[0].
# This is merged based on the mask, such that we'll have either the expected
# state value or 0 in case the state was final.
with torch.no_grad():
next_state_values = torch.zeros(self.batch_size, device=self.device)
next_state_values[non_final_mask] = self.target(non_final_next_states).max(1)[0].detach()
# Compute the expected Q values
expected_state_action_values = reward_batch + self.gamma * next_state_values
# Compute loss
loss = self.loss_fn(state_action_values, expected_state_action_values.unsqueeze(1))
if self.debug:
self.loss_list.append(loss)
# Optimize the model
self.optimizer.zero_grad()
loss.backward()
for param in self.model.parameters():
param.grad.data.clamp_(-1, 1)
self.optimizer.step()
def run(self):
"""Main loop that controls the execution of the agent"""
scores = []
mean_scores = []
j = 0 # used for model2 update every c steps
for e in range(self.n_episodes):
state = self.env.reset()
state = torch.tensor(state, device=self.device, dtype=torch.float)
done = False
cum_reward = 0
while not done:
action = self.choose_action(
state,
self.get_epsilon(e))
next_state, reward, done, _ = self.env.step(action)
next_state = torch.tensor(next_state, device=self.device, dtype=torch.float)
cum_reward += reward
self.memory.push(
state, #Converted to tensor in choose_action method
torch.tensor([action], device=self.device),
None if done else next_state,
torch.tensor(reward, device=self.device).clamp_(-1, 1),
torch.tensor(not done, device=self.device, dtype=torch.bool))
if self.memory.__len__() >= self.batch_size:
self.replay()
state = next_state
j += 1
# update second model
if j % self.c == 0:
self.target.load_state_dict(self.model.state_dict())
self.target.eval()
scores.append(cum_reward)
mean_score = np.mean(scores)
mean_scores.append(mean_score)
if e % 100 == 0 and self.debug:
print('[Episode {}] - Mean reward {}.'.format(e, mean_score))
# noinspection PyUnboundLocalVariable
print('[Episode {}] - Mean reward {}.'.format(e, mean_score))
return scores, mean_scores
def save(self, path):
torch.save(self.model, path)
def load(self, path):
self.model = torch.load(path)
class DQNSolver:
def __init__(self, env, model, target, double=False, n_episodes=3000, max_env_steps=None, gamma=0.9,
epsilon=0.5, epsilon_min=0.05, epsilon_log_decay=0.001, alpha=1e-3,
memory_size=10000, batch_size=256, c=10, hidden_layers=1, hidden_size=6,
render=False, debug=False):
self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
self.model = model.to(self.device)
self.target = target.to(self.device)
self.target.load_state_dict(self.model.state_dict())
self.target.eval()
self.model.train()
self.double = double
self.memory = ReplayMemory(capacity=memory_size)
self.env = env
self.gamma = gamma
self.epsilon = epsilon
self.epsilon_min = epsilon_min
self.epsilon_decay = epsilon_log_decay
self.alpha = alpha
self.n_episodes = n_episodes
self.batch_size = batch_size
self.c = c
if max_env_steps is not None:
self.env._max_episode_steps = max_env_steps
self.render = render
self.debug = debug
if debug:
self.loss_list = []
self.loss_fn = torch.nn.MSELoss()
self.optimizer = torch.optim.Adam(self.model.parameters(), lr=alpha)
def choose_action(self, state, epsilon):
"""Chooses the next action according to the model trained and the policy"""
# exploits the current knowledge if the random number > epsilon, otherwise explores
if np.random.random() <= epsilon:
return self.env.action_space.sample()
else:
with torch.no_grad():
q = self.model(state)
argmax = torch.argmax(q)
return argmax.item()
def get_epsilon(self, episode):
"""Returns an epsilon that decays over time until a minimum epsilon value is reached; in this case the minimum
value is returned"""
return max(self.epsilon_min, self.epsilon * math.exp(-self.epsilon_decay * episode))
def replay(self):
"""Previously stored (s, a, r, s') tuples are replayed (that is, are added into the model). The size of the
tuples added is determined by the batch_size parameter"""
transitions, _ = self.memory.sample(self.batch_size)
# Transpose the batch (see https://stackoverflow.com/a/19343/3343043 for
# detailed explanation). This converts batch-array of Transitions
# to Transition of batch-arrays.
batch = Transition(*zip(*transitions))
# Compute a mask of non-final states and concatenate the batch elements
# (a final state would've been the one after which simulation ended)
current_tensors = [s for s in batch.next_state if s is not None]
if current_tensors:
non_final_next_states = torch.stack(current_tensors)
else:
return
non_final_mask = torch.stack(batch.done)
state_batch = torch.stack(batch.state)
action_batch = torch.stack(batch.action)
reward_batch = torch.stack(batch.reward)
# Compute Q(s_t, a) - the model computes Q(s_t), then we select the
# columns of actions taken. These are the actions which would've been taken
# for each batch state according to policy_net
state_action_values = self.model(state_batch).gather(1, action_batch)
# Compute V(s_{t+1}) for all next states.
# Expected values of actions for non_final_next_states are computed based
# on the "older" target_net; selecting their best reward with max(1)[0].
# This is merged based on the mask, such that we'll have either the expected
# state value or 0 in case the state was final.
with torch.no_grad():
next_state_values = torch.zeros(self.batch_size, device=self.device)
if not self.double:
next_state_values[non_final_mask] = self.target(non_final_next_states).max(1)[0].detach()
else:
non_final_best_actions_index = torch.argmax(self.model(non_final_next_states), 1).unsqueeze(1)
next_state_values[non_final_mask] = self.target(non_final_next_states). \
gather(1, non_final_best_actions_index).squeeze(1).detach()
expected_state_action_values = reward_batch + self.gamma * next_state_values
# Compute loss
loss = self.loss_fn(state_action_values, expected_state_action_values.unsqueeze(1))
if self.debug:
self.loss_list.append(loss)
# Optimize the model
self.optimizer.zero_grad()
loss.backward()
for param in self.model.parameters():
param.grad.data.clamp_(-1, 1)
self.optimizer.step()
def run(self):
"""Main loop that controls the execution of the agent"""
scores = []
mean_scores = []
j = 0 # used for model2 update every c steps
for e in range(self.n_episodes):
state = self.env.reset()
state = torch.tensor(state, device=self.device, dtype=torch.float)
done = False
cum_reward = 0
while not done:
action = self.choose_action(
state,
self.get_epsilon(e))
next_state, reward, done, _ = self.env.step(action)
next_state = torch.tensor(next_state, device=self.device, dtype=torch.float)
cum_reward += reward
self.memory.push(
state, # Converted to tensor in choose_action method
torch.tensor([action], device=self.device),
None if done else next_state,
torch.tensor(reward, device=self.device).clamp_(-1, 1),
torch.tensor(not done, device=self.device, dtype=torch.bool))
if self.memory.__len__() >= self.batch_size:
self.replay()
state = next_state
j += 1
# update second model
if j % self.c == 0:
self.target.load_state_dict(self.model.state_dict())
self.target.eval()
scores.append(cum_reward)
mean_score = np.mean(scores)
mean_scores.append(mean_score)
if e % 100 == 0 and self.debug:
print('[Episode {}] - Mean reward {}.'.format(e, mean_score))
# noinspection PyUnboundLocalVariable
print('[Episode {}] - Mean reward {}.'.format(e, mean_score))
return scores, mean_scores
def save(self, path):
torch.save(self.model, path)
def load(self, path):
self.model = torch.load(path)