def __init__(self, env_id, max_step = 1e5, prior_alpha = 0.6, prior_beta_start = 0.4,
epsilon_start = 1.0, epsilon_final = 0.01, epsilon_decay = 500,
batch_size = 32, gamma = 0.99, target_update_interval=1000, save_interval = 1e4,
):
self.prior_beta_start = prior_beta_start
self.max_step = int(max_step)
self.batch_size = batch_size
self.gamma = gamma
self.target_update_interval = target_update_interval
self.save_interval = save_interval
self.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
self.env = gym.make(env_id)
self.model = DuelingDQN(self.env).to(self.device)
self.target_model = DuelingDQN(self.env).to(self.device)
self.target_model.load_state_dict(self.model.state_dict())
self.replay_buffer = PrioritizedReplayBuffer(100000,alpha=prior_alpha)
self.optimizer = optim.Adam(self.model.parameters())
self.writer = SummaryWriter(comment="-{}-learner".format(self.env.unwrapped.spec.id))
# decay function
self.scheduler = optim.lr_scheduler.StepLR(self.optimizer,step_size=1000,gamma=0.99)
self.beta_by_frame = lambda frame_idx: min(1.0, self.prior_beta_start + frame_idx * (1.0 - self.prior_beta_start) / 1000)
self.epsilon_by_frame = lambda frame_idx: epsilon_final + (epsilon_start - epsilon_final) * math.exp(-1. * frame_idx / epsilon_decay)
def __init__(self, model, opt, learning=True):
super().__init__()
self.memory = PrioritizedReplayBuffer(100000, 0.6)
self.previous_state = None
self.previous_action = None
self.previous_legal_actions = None
self.step = 0
self.model_vae = model[0]
self.model_dqn = model[1]
self.model_dqn_target = model[2]
self.opt_vae = opt[0]
self.opt_dqn = opt[1]
self.loss_vae = 0
self.loss_dqn = 0
self.batch_size = 32
self.max_tile = 0
self.totalCorrect = 0
self.total = 0
self.acc = 0
self.beta = 0.7
#self.test_q = 0
self.epsilon_schedule = LinearSchedule(500000,
initial_p=0.99,
final_p=0.01)
self.learning = learning
def __init__(self, model, opt, learning=True):
super().__init__()
self.memory = PrioritizedReplayBuffer(3000, 0.6)
self.previous_state = None
self.previous_action = None
self.previous_legal_actions = None
self.step = 0
self.model = model
self.opt = opt
self.loss = 0
self.batch_size = 32
self.test_q = 0
self.max_tile = 0
self.beta = 0.7
self.reconver_step = 0
#self.test_q = 0
self.epsilon_schedule = LinearSchedule(100000,
initial_p=0.99,
final_p=0.01)
self.learning = learning
def __init__(self,
env,
save_location,
start_episode=1,
saved_model=None,
prioritized_replay=False):
self.env = env
self.num_actions = env.action_space.n
self.start_episode = start_episode
self.save_location = save_location
self.saved_model = saved_model
self.prioritized_replay = prioritized_replay
self.alpha = 0.6
self.beta = 0
self.learning_rate = 1e-4
self.gamma = 0.98
self.buffer_limit = 10**5
self.training_frame_start = 10000 * 5
self.batch_size = 32
self.eps_start = 1
self.eps_end = 0.01
self.decay_factor = 10**5
if prioritized_replay:
self.memory = PrioritizedReplayBuffer(size=self.buffer_limit,
alpha=self.alpha)
self.prioritized_replay_eps = 1e-5
else:
self.memory = ReplayBuffer(size=self.buffer_limit)
if saved_model:
self.epsilon_decay = lambda x: self.eps_end
else:
self.epsilon_decay = lambda x: self.eps_end + (
self.eps_start - self.eps_end) * math.exp(-1. * x / self.
decay_factor)
self.save_interval = 100000
self.update_target_interval = 10000
self.device = device
self.q = Qnet(84, 84, in_channels=4,
n_actions=self.num_actions).to(device)
self.q_target = Qnet(84, 84, in_channels=4,
n_actions=self.num_actions).to(device)
class VAE_DQNAgent(agent):
def __init__(self, model, opt, learning=True):
super().__init__()
self.memory = PrioritizedReplayBuffer(100000, 0.6)
self.previous_state = None
self.previous_action = None
self.previous_legal_actions = None
self.step = 0
self.model_vae = model[0]
self.model_dqn = model[1]
self.model_dqn_target = model[2]
self.opt_vae = opt[0]
self.opt_dqn = opt[1]
self.loss_vae = 0
self.loss_dqn = 0
self.batch_size = 32
self.max_tile = 0
self.totalCorrect = 0
self.total = 0
self.acc = 0
self.beta = 0.7
#self.test_q = 0
self.epsilon_schedule = LinearSchedule(500000,
initial_p=0.99,
final_p=0.01)
self.learning = learning
def should_explore(self):
self.epsilon = self.epsilon_schedule.value(self.step)
return random.random() < self.epsilon
def action(self):
if self.learning:
self.step += 1
legalActions = self.legal_actions(deepcopy(self.gb.board))
board = deepcopy(self.gb.board)
board = oneHotMap(board)
if self.learning and self.should_explore():
q_values = None
action = random.choice(legalActions)
choice = self.actions[action]
else:
#mark
state = torch.from_numpy(board).type(
torch.FloatTensor).cuda().view(-1, 17, 4, 4)
action, q_values = self.predict(state, legalActions)
choice = self.actions[action]
if self.learning:
reward = self.gb.currentReward
if reward != 0:
reward = np.log2(reward)
if (self.previous_state is not None
and self.previous_action is not None):
self.memory.add(self.previous_state, self.previous_action,
self.previous_legal_actions, reward,
legalActions, board, 0)
self.previous_state = board
self.previous_action = action
self.previous_legal_actions = legalActions
if not self.learning:
state = torch.from_numpy(board).type(
torch.FloatTensor).cuda().view(-1, 17, 4, 4)
recon_batch, _, _ = self.model_vae(state)
target_board = reverseOneHotMap(recon_batch.data.cpu().numpy())
#print(target_board.shape)
self.totalCorrect += np.sum(self.gb.board == target_board)
self.total += 16
self.acc = self.totalCorrect / self.total
if self.learning:
self.update()
return choice
def enableLearning(self):
self.model_vae.train()
self.model_dqn.train()
self.learning = True
self.max_tile = 0
self.reset()
def disableLearning(self):
self.model_vae.eval()
self.model_dqn.eval()
self.totalCorrect = 0
self.total = 0
self.acc = 0
self.learning = False
def end_episode(self):
if not self.learning:
m = np.max(self.gb.board)
if m > self.max_tile:
self.max_tile = m
return
#print(self.gb.board)
board = deepcopy(self.gb.board)
board = oneHotMap(board)
#legalActions = self.legal_actions(deepcopy(self.gb.board))
#print(legalActions)
self.memory.add(self.previous_state, self.previous_action,
self.previous_legal_actions, self.gb.currentReward, [],
board, 1)
self.reset()
def reset(self):
self.previous_state = None
self.previous_action = None
self.previous_legal_actions = None
def update(self):
if self.step < self.batch_size:
return
batch = self.memory.sample(self.batch_size, self.beta)
(states, actions, legal_actions, reward, next_legal_actions,
next_states, is_terminal, weights, batch_idxes) = batch
batch_idx = 1
terminal = torch.tensor(is_terminal).type(torch.cuda.FloatTensor)
reward = torch.tensor(reward).type(torch.cuda.FloatTensor)
states = torch.from_numpy(states).type(torch.FloatTensor).cuda().view(
-1, 17, 4, 4)
next_states = torch.from_numpy(next_states).type(
torch.FloatTensor).cuda().view(-1, 17, 4, 4)
# Current Q Values
q_actions, q_values, mu, logvar = self.predict_batch(states)
batch_index = torch.arange(self.batch_size, dtype=torch.long)
#print(actions)
#print(q_values)
#self.test_q = q_values
q_values = q_values[batch_index, actions]
#print(q_values)
# Calculate target
q_actions_next, q_values_next, _, _ = self.predict_batch_target(
next_states, legalActions=next_legal_actions)
q_max = q_values_next.max(1)[0].detach()
q_max = (1 - terminal) * q_max
q_target = reward + 0.99 * q_max
recon_batch, mu, logvar = self.model_vae(states)
self.opt_vae.zero_grad()
self.opt_dqn.zero_grad()
loss_vae = self.model_vae.loss_function(recon_batch, states, mu,
logvar)
loss_dqn = self.model_dqn.loss_function(q_target, q_values)
loss_vae.backward()
loss_dqn.backward()
self.opt_vae.step()
self.opt_dqn.step()
#train_loss = loss_vae.item() + loss_dqn.item()
self.loss_vae += loss_vae.item() / len(states)
self.loss_dqn += loss_dqn.item() / len(states)
# Update priorities
td_errors = q_values - q_target
new_priorities = torch.abs(td_errors) + 1e-6 # prioritized_replay_eps
self.memory.update_priorities(batch_idxes, new_priorities.data)
if self.step % 4000 == 0:
self.model_dqn_target.load_state_dict(self.model_dqn.state_dict())
def predict_batch(self, input, legalActions=None):
q_values, mu, logvar = self.model_dqn(input)
if legalActions is None:
values, q_actions = q_values.max(1)
else:
q_values_true = torch.full((self.batch_size, 4), -100000000).cuda()
for i, action in enumerate(legalActions):
q_values_true[i, action] = q_values[i, action]
values, q_actions = q_values_true.max(1)
q_values = q_values_true
#print(q_values_true)
return q_actions, q_values, mu, logvar
def predict_batch_target(self, input, legalActions=None):
q_values, mu, logvar = self.model_dqn_target(input)
if legalActions is None:
values, q_actions = q_values.max(1)
else:
q_values_true = torch.full((self.batch_size, 4), -100000000).cuda()
for i, action in enumerate(legalActions):
q_values_true[i, action] = q_values[i, action]
values, q_actions = q_values_true.max(1)
q_values = q_values_true
#print(q_values_true)
return q_actions, q_values, mu, logvar
def predict(self, input, legalActions):
q_values, mu, logvar = self.model_dqn(input)
for action in range(4):
if action not in legalActions:
q_values[0, action] = -100000000
action = torch.argmax(q_values)
if int(action.item()) not in legalActions:
print(legalActions, q_values, action)
print("!!!!!!!!!!!!!!!!!!!!!!!!!")
return action.item(), q_values
def legal_actions(self, copy_gb):
legalActions = []
for i in range(4):
try_gb = gameboard(4, deepcopy(copy_gb))
changed = try_gb.takeAction(self.actions[i])
if changed:
legalActions.append(i)
return legalActions
class train_DQN():
def __init__(self, env_id, max_step = 1e5, prior_alpha = 0.6, prior_beta_start = 0.4,
epsilon_start = 1.0, epsilon_final = 0.01, epsilon_decay = 500,
batch_size = 32, gamma = 0.99, target_update_interval=1000, save_interval = 1e4,
):
self.prior_beta_start = prior_beta_start
self.max_step = int(max_step)
self.batch_size = batch_size
self.gamma = gamma
self.target_update_interval = target_update_interval
self.save_interval = save_interval
self.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
self.env = gym.make(env_id)
self.model = DuelingDQN(self.env).to(self.device)
self.target_model = DuelingDQN(self.env).to(self.device)
self.target_model.load_state_dict(self.model.state_dict())
self.replay_buffer = PrioritizedReplayBuffer(100000,alpha=prior_alpha)
self.optimizer = optim.Adam(self.model.parameters())
self.writer = SummaryWriter(comment="-{}-learner".format(self.env.unwrapped.spec.id))
# decay function
self.scheduler = optim.lr_scheduler.StepLR(self.optimizer,step_size=1000,gamma=0.99)
self.beta_by_frame = lambda frame_idx: min(1.0, self.prior_beta_start + frame_idx * (1.0 - self.prior_beta_start) / 1000)
self.epsilon_by_frame = lambda frame_idx: epsilon_final + (epsilon_start - epsilon_final) * math.exp(-1. * frame_idx / epsilon_decay)
def update_target(self,current_model, target_model):
target_model.load_state_dict(current_model.state_dict())
def compute_td_loss(self,batch_size, beta):
state, action, reward, next_state, done, weights, indices = self.replay_buffer.sample(batch_size, beta)
state = torch.FloatTensor(state).to(self.device)
next_state = torch.FloatTensor(next_state).to(self.device)
action = torch.LongTensor(action).to(self.device)
reward = torch.FloatTensor(reward).to(self.device)
done = torch.FloatTensor(done).to(self.device)
weights = torch.FloatTensor(weights).to(self.device)
batch = (state, action, reward, next_state, done, weights)
# q_values = self.model(state)
# next_q_values = self.target_model(next_state)
# q_value = q_values.gather(1, action.unsqueeze(1)).squeeze(1)
# next_q_value = next_q_values.max(1)[0]
# expected_q_value = reward + self.gamma * next_q_value * (1 - done)
# td_error = torch.abs(expected_q_value.detach() - q_value)
# loss = (td_error).pow(2) * weights
# prios = loss+1e-5#0.9 * torch.max(td_error)+(1-0.9)*td_error
# loss = loss.mean()
loss, prios = utils.compute_loss(self.model,self.target_model, batch,1)
self.optimizer.zero_grad()
loss.backward()
self.scheduler.step()
self.replay_buffer.update_priorities(indices, prios)
self.optimizer.step()
return loss
def train(self):
losses = []
all_rewards = []
episode_reward = 0
episode_idx = 0
episode_length = 0
state = self.env.reset()
for frame_idx in range(self.max_step):
epsilon = self.epsilon_by_frame(frame_idx)
action,_ = self.model.act(torch.FloatTensor((state)).to(self.device), epsilon)
next_state, reward, done, _ = self.env.step(action)
self.replay_buffer.add(state, action, reward, next_state, done)
state = next_state
episode_reward += reward
episode_length += 1
if done:
state = self.env.reset()
all_rewards.append(episode_reward)
self.writer.add_scalar("actor/episode_reward", episode_reward, episode_idx)
self.writer.add_scalar("actor/episode_length", episode_length, episode_idx)
# print("episode: ",episode_idx, " reward: ", episode_reward)
episode_reward = 0
episode_length = 0
episode_idx += 1
if len(self.replay_buffer) > self.batch_size:
beta = self.beta_by_frame(frame_idx)
loss = self.compute_td_loss(self.batch_size, beta)
losses.append(loss.item())
self.writer.add_scalar("learner/loss", loss, frame_idx)
if frame_idx % self.target_update_interval == 0:
print("update target...")
self.update_target(self.model, self.target_model)
if frame_idx % self.save_interval == 0 or frame_idx == self.max_step-1:
print("save model...")
self.save_model(frame_idx)
def save_model(self, idx):
torch.save(self.model.state_dict(), "./model{}.pth".format(idx))
def load_model(self,idx):
with open("model{}.pth".format(idx), "rb") as f:
print("loading weights_{}".format(idx))
self.model.load_state_dict(torch.load(f,map_location="cpu"))
class DQNAgent_Vanila(agent):
def __init__(self, model, opt, learning=True):
super().__init__()
self.memory = PrioritizedReplayBuffer(3000, 0.6)
self.previous_state = None
self.previous_action = None
self.previous_legal_actions = None
self.step = 0
self.model = model
self.opt = opt
self.loss = 0
self.batch_size = 32
self.test_q = 0
self.max_tile = 0
self.beta = 0.7
self.reconver_step = 0
#self.test_q = 0
self.epsilon_schedule = LinearSchedule(100000,
initial_p=0.99,
final_p=0.01)
self.learning = learning
def should_explore(self):
self.epsilon = self.epsilon_schedule.value(self.step +
self.reconver_step)
return random.random() < self.epsilon
def action(self):
if self.learning:
self.step += 1
legalActions = self.legal_actions(deepcopy(self.gb.board))
board = deepcopy(self.gb.board)
board = oneHotMap(board)
if self.learning and self.should_explore():
q_values = None
action = random.choice(legalActions)
choice = self.actions[action]
else:
#mark
state = torch.from_numpy(board).type(
torch.FloatTensor).cuda().view(-1, 17, 4, 4)
action, q_values = self.predict(state, legalActions)
choice = self.actions[action]
if self.learning:
reward = self.gb.currentReward
if reward != 0:
reward = np.log2(reward)
if (self.previous_state is not None
and self.previous_action is not None):
self.memory.add(self.previous_state, self.previous_action,
self.previous_legal_actions, reward,
legalActions, board, 0)
self.previous_state = board
self.previous_action = action
self.previous_legal_actions = legalActions
if self.learning:
self.update()
return choice
def enableLearning(self):
self.model.train()
self.learning = True
self.max_tile = 0
self.reset()
def disableLearning(self):
self.model.eval()
self.learning = False
def end_episode(self):
if not self.learning:
m = np.max(self.gb.board)
if m > self.max_tile:
self.max_tile = m
return
#print(self.gb.board)
board = deepcopy(self.gb.board)
board = oneHotMap(board)
#legalActions = self.legal_actions(deepcopy(self.gb.board))
#print(legalActions)
self.memory.add(self.previous_state, self.previous_action,
self.previous_legal_actions, self.gb.currentReward, [],
board, 1)
self.reset()
def reset(self):
self.previous_state = None
self.previous_action = None
self.previous_legal_actions = None
def update(self):
if self.step < self.batch_size:
return
batch = self.memory.sample(self.batch_size, self.beta)
(states, actions, legal_actions, reward, next_legal_actions,
next_states, is_terminal, weights, batch_idxes) = batch
terminal = torch.tensor(is_terminal).type(torch.cuda.FloatTensor)
reward = torch.tensor(reward).type(torch.cuda.FloatTensor)
states = torch.from_numpy(states).type(torch.FloatTensor).cuda().view(
-1, 17, 4, 4)
next_states = torch.from_numpy(next_states).type(
torch.FloatTensor).cuda().view(-1, 17, 4, 4)
# Current Q Values
_, q_values = self.predict_batch(states)
batch_index = torch.arange(self.batch_size, dtype=torch.long)
#print(actions)
#print(q_values)
q_values = q_values[batch_index, actions]
#print(q_values)
# Calculate target
q_actions_next, q_values_next = self.predict_batch(
next_states, legalActions=next_legal_actions)
#print(q_values_next)
q_max = q_values_next.max(1)[0].detach()
q_max = (1 - terminal) * q_max
# if sum(terminal == 1) > 0:
# print(reward)
# print( (terminal == 1).nonzero())
# print(terminal)
# print(next_legal_actions)
# print(q_max)
# input()
q_target = reward + 0.99 * q_max
self.opt.zero_grad()
loss = self.model.loss_function(q_target, q_values)
loss.backward()
self.opt.step()
#train_loss = loss_vae.item() + loss_dqn.item()
self.loss += loss.item() / len(states)
# Update priorities
td_errors = q_values - q_target
new_priorities = torch.abs(td_errors) + 1e-6 # prioritized_replay_eps
self.memory.update_priorities(batch_idxes, new_priorities.data)
def predict_batch(self, input, legalActions=None):
q_values = self.model(input)
if legalActions is None:
values, q_actions = q_values.max(1)
else:
q_values_true = torch.full((self.batch_size, 4), -100000000).cuda()
for i, action in enumerate(legalActions):
q_values_true[i, action] = q_values[i, action]
values, q_actions = q_values_true.max(1)
q_values = q_values_true
#print(q_values_true)
return q_actions, q_values
def predict(self, input, legalActions):
q_values = self.model(input)
for action in range(4):
if action not in legalActions:
q_values[0, action] = -100000000
action = torch.argmax(q_values)
return action.item(), q_values
def legal_actions(self, copy_gb):
legalActions = []
for i in range(4):
try_gb = gameboard(4, deepcopy(copy_gb))
changed = try_gb.takeAction(self.actions[i])
if changed:
legalActions.append(i)
return legalActions
'''
def learn(env,
q_func,
lr=5e-4,
max_timesteps=100000,
buffer_size=50000,
exploration_fraction=0.1,
exploration_final_eps=0.02,
train_freq=1,
batch_size=32,
print_freq=100,
checkpoint_freq=10000,
learning_starts=1000,
gamma=1.0,
target_network_update_freq=500,
prioritized_replay=False,
prioritized_replay_alpha=0.6,
prioritized_replay_beta0=0.4,
prioritized_replay_beta_iters=None,
prioritized_replay_eps=1e-6,
param_noise=False,
callback=None):
"""Train a deepq model.
Parameters
-------
env: gym.Env
environment to train on
q_func: (tf.Variable, int, str, bool) -> tf.Variable
the model that takes the following inputs:
observation_in: object
the output of observation placeholder
num_actions: int
number of actions
scope: str
reuse: bool
should be passed to outer variable scope
and returns a tensor of shape (batch_size, num_actions) with values of every action.
lr: float
learning rate for adam optimizer
max_timesteps: int
number of env steps to optimizer for
buffer_size: int
size of the replay buffer
exploration_fraction: float
fraction of entire training period over which the exploration rate is annealed
exploration_final_eps: float
final value of random action probability
train_freq: int
update the model every `train_freq` steps.
set to None to disable printing
batch_size: int
size of a batched sampled from replay buffer for training
print_freq: int
how often to print out training progress
set to None to disable printing
checkpoint_freq: int
how often to save the model. This is so that the best version is restored
at the end of the training. If you do not wish to restore the best version at
the end of the training set this variable to None.
learning_starts: int
how many steps of the model to collect transitions for before learning starts
gamma: float
discount factor
target_network_update_freq: int
update the target network every `target_network_update_freq` steps.
prioritized_replay: True
if True prioritized replay buffer will be used.
prioritized_replay_alpha: float
alpha parameter for prioritized replay buffer
prioritized_replay_beta0: float
initial value of beta for prioritized replay buffer
prioritized_replay_beta_iters: int
number of iterations over which beta will be annealed from initial value
to 1.0. If set to None equals to max_timesteps.
prioritized_replay_eps: float
epsilon to add to the TD errors when updating priorities.
callback: (locals, globals) -> None
function called at every steps with state of the algorithm.
If callback returns true training stops.
Returns
-------
act: ActWrapper
Wrapper over act function. Adds ability to save it and load it.
See header of baselines/deepq/categorical.py for details on the act function.
"""
# Create all the functions necessary to train the model
sess = tf.Session()
sess.__enter__()
# capture the shape outside the closure so that the env object is not serialized
# by cloudpickle when serializing make_obs_ph
observation_space_shape = env.observation_space.shape
def make_obs_ph(name):
return BatchInput(observation_space_shape, name=name)
act, train, update_target, debug = deepq.build_train(
make_obs_ph=make_obs_ph,
q_func=q_func,
num_actions=env.action_space.n,
optimizer=tf.train.AdamOptimizer(learning_rate=lr),
gamma=gamma,
grad_norm_clipping=10,
param_noise=param_noise)
act_params = {
'make_obs_ph': make_obs_ph,
'q_func': q_func,
'num_actions': env.action_space.n,
}
act = ActWrapper(act, act_params)
# Create the replay buffer
if prioritized_replay:
replay_buffer = PrioritizedReplayBuffer(buffer_size,
alpha=prioritized_replay_alpha)
if prioritized_replay_beta_iters is None:
prioritized_replay_beta_iters = max_timesteps
beta_schedule = LinearSchedule(prioritized_replay_beta_iters,
initial_p=prioritized_replay_beta0,
final_p=1.0)
else:
replay_buffer = ReplayBuffer(buffer_size)
beta_schedule = None
# Create the schedule for exploration starting from 1.
exploration = LinearSchedule(schedule_timesteps=int(exploration_fraction *
max_timesteps),
initial_p=1.0,
final_p=exploration_final_eps)
# Initialize the parameters and copy them to the target network.
U.initialize()
update_target()
episode_rewards = [0.0]
saved_mean_reward = None
obs = env.reset()
reset = True
with tempfile.TemporaryDirectory() as td:
model_saved = False
model_file = os.path.join(td, "model")
for t in range(max_timesteps):
if callback is not None:
if callback(locals(), globals()):
break
# Take action and update exploration to the newest value
kwargs = {}
if not param_noise:
update_eps = exploration.value(t)
update_param_noise_threshold = 0.
else:
update_eps = 0.
# Compute the threshold such that the KL divergence between perturbed and non-perturbed
# policy is comparable to eps-greedy exploration with eps = exploration.value(t).
# See Appendix C.1 in Parameter Space Noise for Exploration, Plappert et al., 2017
# for detailed explanation.
update_param_noise_threshold = -np.log(1. - exploration.value(
t) + exploration.value(t) / float(env.action_space.n))
kwargs['reset'] = reset
kwargs[
'update_param_noise_threshold'] = update_param_noise_threshold
kwargs['update_param_noise_scale'] = True
action = act(np.array(obs)[None], update_eps=update_eps,
**kwargs)[0]
env_action = action
reset = False
new_obs, rew, done, _ = env.step(env_action)
# Store transition in the replay buffer.
replay_buffer.add(obs, action, rew, new_obs, float(done))
obs = new_obs
episode_rewards[-1] += rew
if done:
obs = env.reset()
episode_rewards.append(0.0)
reset = True
if t > learning_starts and t % train_freq == 0:
# Minimize the error in Bellman's equation on a batch sampled from replay buffer.
if prioritized_replay:
experience = replay_buffer.sample(
batch_size, beta=beta_schedule.value(t))
(obses_t, actions, rewards, obses_tp1, dones, weights,
batch_idxes) = experience
else:
obses_t, actions, rewards, obses_tp1, dones = replay_buffer.sample(
batch_size)
weights, batch_idxes = np.ones_like(rewards), None
td_errors = train(obses_t, actions, rewards, obses_tp1, dones,
weights)
if prioritized_replay:
new_priorities = np.abs(td_errors) + prioritized_replay_eps
replay_buffer.update_priorities(batch_idxes,
new_priorities)
if t > learning_starts and t % target_network_update_freq == 0:
# Update target network periodically.
update_target()
mean_100ep_reward = round(np.mean(episode_rewards[-101:-1]), 1)
num_episodes = len(episode_rewards)
if done and print_freq is not None and len(
episode_rewards) % print_freq == 0:
logger.record_tabular("steps", t)
logger.record_tabular("episodes", num_episodes)
logger.record_tabular("mean 100 episode reward",
mean_100ep_reward)
logger.record_tabular("% time spent exploring",
int(100 * exploration.value(t)))
logger.dump_tabular()
if (checkpoint_freq is not None and t > learning_starts
and num_episodes > 100 and t % checkpoint_freq == 0):
if saved_mean_reward is None or mean_100ep_reward > saved_mean_reward:
if print_freq is not None:
logger.log(
"Saving model due to mean reward increase: {} -> {}"
.format(saved_mean_reward, mean_100ep_reward))
save_state(model_file)
model_saved = True
saved_mean_reward = mean_100ep_reward
if model_saved:
if print_freq is not None:
logger.log("Restored model with mean reward: {}".format(
saved_mean_reward))
load_state(model_file)
return act
def __init__(self,
state_size,
action_size,
buffer_size=int(1e5),
batch_size=64,
gamma=0.99,
tau=1e-3,
learn_rate=5e-4,
update_every=4,
per_epsilon=1e-5,
per_alpha=0.6,
per_beta=0.9,
device=DEFAULT_DEVICE,
seed=0):
""" Initialize an object.
:param state_size: (int) Dimension of each state
:param action_size: (int) Dimension of each action
:param buffer_size: (int) Replay buffer size
:param batch_size: (int) Minibatch size used during learning
:param gamma: (float) Discount factor
:param tau: (float) Scaling parameter for soft update
:param learn_rate: (float) Learning rate used by optimizer
:param update_every: (int) Steps between updates of target network
:param per_epsilon: (float) PER hyperparameter, constant added to each error
:param per_alpha: (float) PER hyperparameter, exponent applied to each probability
:param per_beta: (float) PER hyperparameter, bias correction exponent for probability weight
:param device: (torch.device) Object representing the device where to allocate tensors
:param seed: (int) Seed used for PRNG
"""
# Save copy of model parameters
self.state_size = state_size
self.action_size = action_size
self.seed = random.seed(seed)
self.device = device
# Save copy of hyperparameters
self.buffer_size = buffer_size
self.batch_size = batch_size
self.gamma = gamma
self.tau = tau
self.learn_rate = learn_rate
self.update_every = update_every
self.per_epsilon = per_epsilon
self.per_alpha = per_alpha
self.per_beta = per_beta
# Q networks
self.qnetwork_local = DuelingQNetwork(state_size, action_size,
seed).to(device)
self.qnetwork_target = DuelingQNetwork(state_size, action_size,
seed).to(device)
self.optimizer = optim.Adam(self.qnetwork_local.parameters(),
lr=learn_rate)
# Replay memory
self.memory = PrioritizedReplayBuffer(memory_size=buffer_size,
device=device,
update_every=update_every,
seed=seed)
# Initialize time step (for updating every self.update_every steps)
self.t_step = 0
self.episode = 0
class Agent():
""" Interacts with and learns from the environment.
This agent implements a few improvements over the vanilla DQN, making it
a Double Dueling Deep Q-Learning Network with Prioritized Experience Replay.
* Deep Q-Learning Network: RL where a deep learning network
is used for the Q-network estimate.
* Double DQN: The local network from DQN is used to select the
optimal action during learning, but the policy estimate for
that action is computed using the target network.
* Dueling DQN: The deep learning network explicitly estimates
the value function and the advantage functions separately.
* DQN-PER: Experiences are associated with a probability weight
based upon the absolute error between the estimated Q-value
and the target Q-value at time of estimation -- prioritizing
experiences that help learn more.
"""
def __init__(self,
state_size,
action_size,
buffer_size=int(1e5),
batch_size=64,
gamma=0.99,
tau=1e-3,
learn_rate=5e-4,
update_every=4,
per_epsilon=1e-5,
per_alpha=0.6,
per_beta=0.9,
device=DEFAULT_DEVICE,
seed=0):
""" Initialize an object.
:param state_size: (int) Dimension of each state
:param action_size: (int) Dimension of each action
:param buffer_size: (int) Replay buffer size
:param batch_size: (int) Minibatch size used during learning
:param gamma: (float) Discount factor
:param tau: (float) Scaling parameter for soft update
:param learn_rate: (float) Learning rate used by optimizer
:param update_every: (int) Steps between updates of target network
:param per_epsilon: (float) PER hyperparameter, constant added to each error
:param per_alpha: (float) PER hyperparameter, exponent applied to each probability
:param per_beta: (float) PER hyperparameter, bias correction exponent for probability weight
:param device: (torch.device) Object representing the device where to allocate tensors
:param seed: (int) Seed used for PRNG
"""
# Save copy of model parameters
self.state_size = state_size
self.action_size = action_size
self.seed = random.seed(seed)
self.device = device
# Save copy of hyperparameters
self.buffer_size = buffer_size
self.batch_size = batch_size
self.gamma = gamma
self.tau = tau
self.learn_rate = learn_rate
self.update_every = update_every
self.per_epsilon = per_epsilon
self.per_alpha = per_alpha
self.per_beta = per_beta
# Q networks
self.qnetwork_local = DuelingQNetwork(state_size, action_size,
seed).to(device)
self.qnetwork_target = DuelingQNetwork(state_size, action_size,
seed).to(device)
self.optimizer = optim.Adam(self.qnetwork_local.parameters(),
lr=learn_rate)
# Replay memory
self.memory = PrioritizedReplayBuffer(memory_size=buffer_size,
device=device,
update_every=update_every,
seed=seed)
# Initialize time step (for updating every self.update_every steps)
self.t_step = 0
self.episode = 0
def step(self, state, action, reward, next_state, done):
""" Store a single agent step, learning every N steps
:param state: (array-like) Initial state on the visit
:param action: (int) Action on the visit
:param reward: (float) Reward received on the visit
:param next_state: (array-like) State reached after the visit
:param done: (bool) Flag whether the next state is a terminal state
"""
# Save experience in replay memory
self.memory.add(state, action, reward, next_state, done)
# Learn every self.update_every time steps
self.t_step = (self.t_step + 1) % self.update_every
if self.t_step == 0:
# If enough samples are available in memory, get random subset and learn
if len(self.memory) > self.batch_size:
experiences = self.memory.sample(batch_size=self.batch_size,
alpha=self.per_alpha,
beta=self.per_beta)
self.learn(experiences)
# Keep track of episode number
if done:
self.episode += 1
def act(self, state, eps=0.):
""" Returns the selected action for the given state according to the current policy
:param state: (array_like) Current state
:param eps: (float) Epsilon, for epsilon-greedy action selection
:return: action (int)
"""
state = torch.from_numpy(state).float().unsqueeze(0).to(self.device)
self.qnetwork_local.eval()
with torch.no_grad():
action_values = self.qnetwork_local(state)
self.qnetwork_local.train()
# Epsilon-greedy action selection
# Convert types to np.int32 for compatibility with environment
if random.random() > eps:
return np.argmax(action_values.cpu().data.numpy()).astype(np.int32)
else:
return random.choice(np.arange(self.action_size)).astype(np.int32)
def learn(self, experiences):
""" Update value parameters using given batch of indexed experience tuples
:param experiences: (Tuple[torch.Tensor, np.array]) (s, a, r, s', done, index) tuples
"""
states, actions, rewards, next_states, dones, indexes = experiences
# Get max predicted Q values (for next states) from target model
# Double DQN: use local network to select action with maximum value,
# then use target network to get Q value for that action
Q_next_indices = self.qnetwork_local(next_states).detach().argmax(
1).unsqueeze(1)
Q_next_values = self.qnetwork_target(next_states).detach()
Q_targets_next = Q_next_values.gather(1, Q_next_indices)
# Compute Q target for current states
Q_targets = rewards + self.gamma * Q_targets_next * (1 - dones)
# Get expected Q values from local model
Q_expected = self.qnetwork_local(states).gather(1, actions)
# Compute estimation error (for Prioritized Experience Replay) and update weights
Q_error = (torch.abs(Q_expected.detach() - Q_targets.detach()) +
self.per_epsilon).squeeze()
self.memory.update(indexes, Q_error)
# Compute loss
loss = F.mse_loss(Q_expected, Q_targets)
# Minimize the loss
self.optimizer.zero_grad()
loss.backward()
self.optimizer.step()
# Update target network
for target_param, local_param in zip(self.qnetwork_target.parameters(),
self.qnetwork_local.parameters()):
target_param.data.copy_(self.tau * local_param.data +
(1.0 - self.tau) * target_param.data)
plt.close()
if __name__ == '__main__':
args = parser.parse_args()
# environment
env, brain, brain_name, action_size, state_size = create_environment()
# replay memory
if args.buffer_type == "prioritized":
memory = PrioritizedReplayBuffer(
action_size=action_size,
state_size=state_size,
buffer_size=args.buffer_size,
batch_size=args.batch_size,
update_buffer_steps=args.update_buffer_steps,
seed=args.seed,
alpha=args.alpha,
beta=args.beta,
beta_inc=args.beta_inc)
else:
memory = ReplayBuffer(action_size=action_size,
state_size=state_size,
buffer_size=args.buffer_size,
batch_size=args.batch_size,
seed=args.seed)
# model
network = DuelQNetwork if args.network == "linear_duel" else QNetwork
# agent