def dqn_learing(env,
q_func,
optimizer_spec,
exploration,
stopping_criterion=None,
replay_buffer_size=1000000,
batch_size=32,
gamma=0.99,
learning_starts=50000,
learning_freq=4,
frame_history_len=4,
target_update_freq=10000):
"""Run Deep Q-learning algorithm.
You can specify your own convnet using q_func.
All schedules are w.r.t. total number of steps taken in the environment.
Parameters
----------
env: gym.Env
gym environment to train on.
q_func: function
Model to use for computing the q function. It should accept the
following named arguments:
input_channel: int
number of channel of input.
num_actions: int
number of actions
optimizer_spec: OptimizerSpec
Specifying the constructor and kwargs, as well as learning rate schedule
for the optimizer
exploration: Schedule (defined in utils.schedule)
schedule for probability of chosing random action.
stopping_criterion: (env) -> bool
should return true when it's ok for the RL algorithm to stop.
takes in env and the number of steps executed so far.
replay_buffer_size: int
How many memories to store in the replay buffer.
batch_size: int
How many transitions to sample each time experience is replayed.
gamma: float
Discount Factor
learning_starts: int
After how many environment steps to start replaying experiences
learning_freq: int
How many steps of environment to take between every experience replay
frame_history_len: int
How many past frames to include as input to the model.
target_update_freq: int
How many experience replay rounds (not steps!) to perform between
each update to the target Q network
"""
assert type(env.observation_space) == gym.spaces.Box
assert type(env.action_space) == gym.spaces.Discrete
###############
# BUILD MODEL #
###############
if len(env.observation_space.shape) == 1:
# This means we are running on low-dimensional observations (e.g. RAM)
input_arg = env.observation_space.shape[0]
else:
img_h, img_w, img_c = env.observation_space.shape
input_arg = frame_history_len * img_c
num_actions = env.action_space.n
# Construct an epilson greedy policy with given exploration schedule
def select_epilson_greedy_action(model, obs, t):
sample = random.random()
eps_threshold = exploration.value(t)
if sample > eps_threshold:
obs = torch.from_numpy(obs).type(dtype).unsqueeze(0) / 255.0
# Use volatile = True if variable is only used in inference mode, i.e. don’t save the history
return model(Variable(obs, volatile=True)).data.max(1)[1].cpu()
else:
return torch.IntTensor([[random.randrange(num_actions)]])
# Initialize target q function and q function, i.e. build the model.
######
model = DQN(in_channels=input_arg, num_actions=num_actions)
target_Q = DQN(in_channels=input_arg, num_actions=num_actions)
if USE_CUDA:
target_Q = target_Q.cuda()
model = model.cuda()
######
# Construct Q network optimizer function
optimizer = optimizer_spec.constructor(model.parameters(),
**optimizer_spec.kwargs)
# Construct the replay buffer
replay_buffer = ReplayBuffer(replay_buffer_size, frame_history_len)
###############
# RUN ENV #
###############
num_param_updates = 0
mean_episode_reward = -float('nan')
best_mean_episode_reward = -float('inf')
last_obs = env.reset()
LOG_EVERY_N_STEPS = 10000
for t in count():
### 1. Check stopping criterion
if stopping_criterion is not None and stopping_criterion(env):
break
### 2. Step the env and store the transition
# At this point, "last_obs" contains the latest observation that was
# recorded from the simulator. Here, your code needs to store this
# observation and its outcome (reward, next observation, etc.) into
# the replay buffer while stepping the simulator forward one step.
# At the end of this block of code, the simulator should have been
# advanced one step, and the replay buffer should contain one more
# transition.
# Specifically, last_obs must point to the new latest observation.
# Useful functions you'll need to call:
# obs, reward, done, info = env.step(action)
# this steps the environment forward one step
# obs = env.reset()
# this resets the environment if you reached an episode boundary.
# Don't forget to call env.reset() to get a new observation if done
# is true!!
# Note that you cannot use "last_obs" directly as input
# into your network, since it needs to be processed to include context
# from previous frames. You should check out the replay buffer
# implementation in dqn_utils.py to see what functionality the replay
# buffer exposes. The replay buffer has a function called
# encode_recent_observation that will take the latest observation
# that you pushed into the buffer and compute the corresponding
# input that should be given to a Q network by appending some
# previous frames.
# Don't forget to include epsilon greedy exploration!
# And remember that the first time you enter this loop, the model
# may not yet have been initialized (but of course, the first step
# might as well be random, since you haven't trained your net...)
##### OUR CODE
idx = replay_buffer.store_frame(last_obs)
encoded_obs = replay_buffer.encode_recent_observation()
if t > learning_starts:
action = select_epilson_greedy_action(model, encoded_obs, t)[0]
else:
action = random.randrange(num_actions)
obs, reward, done, info = env.step(action)
reward = max(-1.0, min(reward, 1.0))
replay_buffer.store_effect(idx, action, reward, done)
if done:
obs = env.reset()
last_obs = obs
#####
# at this point, the environment should have been advanced one step (and
# reset if done was true), and last_obs should point to the new latest
# observation
### 3. Perform experience replay and train the network.
# Note that this is only done if the replay buffer contains enough samples
# for us to learn something useful -- until then, the model will not be
# initialized and random actions should be taken
if (t > learning_starts and t % learning_freq == 0
and replay_buffer.can_sample(batch_size)):
# Here, you should perform training. Training consists of four steps:
# 3.a: use the replay buffer to sample a batch of transitions (see the
# replay buffer code for function definition, each batch that you sample
# should consist of current observations, current actions, rewards,
# next observations, and done indicator).
# Note: Move the variables to the GPU if avialable
obs_batch, act_batch, rew_batch, next_obs_batch, done_mask = replay_buffer.sample(
batch_size)
obs_batch = Variable(
torch.from_numpy(obs_batch).type(dtype) / 255.0)
next_obs_batch = Variable(
torch.from_numpy(next_obs_batch).type(dtype) / 255.0)
act_batch = Variable(
torch.Tensor(act_batch).type(torch.LongTensor))
rew_batch = Variable(torch.from_numpy(rew_batch))
done_mask = Variable(
torch.Tensor([1. if val == 0 else 0. for val in done_mask]))
if USE_CUDA:
done_mask = done_mask.cuda()
act_batch = act_batch.cuda()
rew_batch = rew_batch.cuda()
obs_batch = obs_batch.cuda()
next_obs_batch = next_obs_batch.cuda()
# 3.b: fill in your own code to compute the Bellman error. This requires
# evaluating the current and next Q-values and constructing the corresponding error.
# Note: don't forget to clip the error between [-1,1], multiply is by -1 (since pytorch minimizes) and
# maskout post terminal status Q-values (see ReplayBuffer code).
# We choose Q based on action taken.
current_Q_values = model(obs_batch).gather(
1, act_batch.unsqueeze(1)) #[0, act_batch]
# 5. Obtain maxQ' and set our target value for chosen action using the bellman equation.
next_max_q = target_Q(next_obs_batch).detach().max(1)[0]
next_Q_values = torch.mul(done_mask, next_max_q)
target_Q_values = rew_batch + (gamma * next_Q_values)
if USE_CUDA:
target_Q_values = target_Q_values.cuda()
d_error = target_Q_values.unsqueeze(1) - current_Q_values
d_error = d_error.clamp(-1, 1) * -1
# 3.c: train the model. To do this, use the bellman error you calculated perviously.
# Pytorch will differentiate this error for you, to backward the error use the following API:
# current.backward(d_error.data.unsqueeze(1))
# Where "current" is the variable holding current Q Values and d_error is the clipped bellman error.
# Your code should produce one scalar-valued tensor.
# Note: don't forget to call optimizer.zero_grad() before the backward call and
# optimizer.step() after the backward call.
optimizer.zero_grad()
current_Q_values.backward(d_error)
optimizer.step()
num_param_updates += 1
# 3.d: periodically update the target network by loading the current Q network weights into the
# target_Q network. see state_dict() and load_state_dict() methods.
# you should update every target_update_freq steps, and you may find the
# variable num_param_updates useful for this (it was initialized to 0)
#####
if num_param_updates % target_update_freq == 0:
target_Q.load_state_dict(model.state_dict())
# YOUR CODE HERE
#####
### 4. Log progress and keep track of statistics
episode_rewards = get_wrapper_by_name(env,
"Monitor").get_episode_rewards()
if len(episode_rewards) > 0:
mean_episode_reward = np.mean(episode_rewards[-100:])
if hasattr(exploration, 'add_reward'):
exploration.add_reward(episode_rewards)
if len(episode_rewards) > 100:
best_mean_episode_reward = max(best_mean_episode_reward,
mean_episode_reward)
Statistic["mean_episode_rewards"].append(mean_episode_reward)
Statistic["best_mean_episode_rewards"].append(best_mean_episode_reward)
if t % LOG_EVERY_N_STEPS == 0 and t > learning_starts:
print("Timestep %d" % (t, ))
print("mean reward (100 episodes) %f" % mean_episode_reward)
print("best mean reward %f" % best_mean_episode_reward)
print("episodes %d" % len(episode_rewards))
print("exploration %f" % exploration.value(t))
sys.stdout.flush()
# Dump statistics to pickle
with open('statistics.pkl', 'wb') as f:
pickle.dump(Statistic, f)
print("Saved to %s" % 'statistics.pkl')
def dqn_eval(env, scheduler, optimizer_constructor, model_type, batch_size, rp_start, rp_size,
exp_frame, exp_initial, exp_final, gamma, target_update_steps, frames_per_epoch,
frames_per_state, output_directory, last_checkpoint, envo):
env.seed(7)
random.seed(7)
gym.undo_logger_setup()
logging.basicConfig(filename=envo+'_'+model_type+'_eval.log',level=logging.INFO)
num_actions = env.action_space.n
print('No. of actions: ', num_actions)
print(env.unwrapped.get_action_meanings())
# initialize action value and target network with the same weights
model = DQN(num_actions, use_bn=False)
if use_cuda:
model.cuda()
saved_params = None
directory = None
index = []
for (dirpath, dirnames, filenames) in os.walk(output_directory):
directory = dirpath
saved_params = filenames
count = 0
counter = 0
chckpoint_index = get_index_from_checkpoint_path(last_checkpoint)
for x in saved_params:
temp = get_index_from_checkpoint_file(x)
if temp > chckpoint_index:
index.append(temp)
index = sorted(index, key=int)
for w in index:
path = directory + '/' + model_type + '_weights_' + str(w) + '.pth'
model.load_state_dict(torch.load(path))
print(path)
print('saved weights loaded...')
eval_epsilon = 0.05
env.reset()
total_reward = []
rewards_per_episode = 0
action_value = torch.zeros(num_actions)
current_state, _, _, _ = play_game(env, frames_per_state, action=0, evaluate=True)
average_action = {k: [] for k in range(num_actions)}
for i in range(NUM_GAMES):
for frame in range(int(MAX_FRAMES_PER_GAME/frames_per_state)):
eval_choice = random.uniform(0,1)
# epsilon greedy algorithm
if eval_choice <= eval_epsilon:
action = LongTensor([[random.randrange(num_actions)]])
else:
action = get_greedy_action(model, current_state)
curr_obs, reward, done, _ = play_game(env, frames_per_state, action[0][0], evaluate=True)
average_action[action[0,0]].append(get_Q_value(model, action.view(1,1), curr_obs))
current_state = curr_obs
rewards_per_episode += reward
if done:
env.reset()
total_reward.append(rewards_per_episode)
rewards_per_episode = 0
current_state, _, _, _ = play_game(env, frames_per_state, action=0, evaluate=True)
break
average_reward = sum(total_reward)/float(len(total_reward))
total_action = 0
for i in range(num_actions):
total_action += sum(average_action[i])/len(average_action[i])
average_action_value = total_action/num_actions
#Compute Standard Deviation
diff = 0
for x in total_reward:
diff += (x - average_reward)*(x - average_reward)
var = diff/len(total_reward)
std_dev = math.sqrt(var)
eval_content = 'Average Score: ', average_reward
eval_std_dev = 'Standard Deviation: ', std_dev
average_action_value_content = 'Average Action Value: ', average_action_value
print(average_action_value_content)
print(eval_content)
print(eval_std_dev)
log_content = path + ' ' + str(average_reward) + ' ' + str(average_action_value) + ' ' + str(std_dev)
logging.info(log_content)
count += 1
print(count)
class Agent_DQN():
def __init__(self, env, args):
# Parameters for q-learning
super(Agent_DQN, self).__init__()
self.env = env
state = env.reset()
state = state.transpose(2, 0, 1)
self.policy_net = DQN(state.shape,
self.env.action_space.n) # Behavior Q
self.target_net = DQN(state.shape, self.env.action_space.n) # Target Q
self.target_net.load_state_dict(self.policy_net.state_dict())
#Initial Q
if USE_CUDA:
print("Using CUDA . . . ")
self.policy_net = self.policy_net.cuda()
self.target_net = self.target_net.cuda()
print('hyperparameters and network initialized')
if args.test_dqn or LOAD == True:
print('loading trained model')
checkpoint = torch.load('trainData')
self.policy_net.load_state_dict(checkpoint['model_state_dict'])
self.target_net.load_state_dict(self.policy_net.state_dict())
def init_game_setting(self):
print('loading trained model')
checkpoint = torch.load('trainData')
self.policy_net.load_state_dict(checkpoint['model_state_dict'])
def push(self, state, action, reward, next_state, done):
state = np.expand_dims(state, 0)
next_state = np.expand_dims(next_state, 0)
memory.append((state, action, reward, next_state, done))
def replay_buffer(self):
state, action, reward, next_state, done = zip(
*random.sample(memory, batch_size))
return np.concatenate(state), action, reward, np.concatenate(
next_state), done
def __len__(self):
return len(self.buffer)
def make_action(self, observation, test=True):
observation = observation.transpose(2, 0, 1)
if np.random.random() > EPSILON or test == True:
observation = Variable(torch.FloatTensor(
np.float32(observation)).unsqueeze(0),
volatile=True)
q_value = self.policy_net.forward(observation)
action = q_value.max(1)[1].data[0]
action = int(action.item())
else:
action = random.randrange(4)
return action
def optimize_model(self):
states, actions, next_states, rewards, dones = self.replay_buffer()
states_v = Variable(torch.FloatTensor(np.float32(states)))
next_states_v = Variable(torch.FloatTensor(np.float32(next_states)),
volatile=True)
actions_v = Variable(torch.LongTensor(actions))
rewards_v = Variable(torch.FloatTensor(rewards))
done = Variable(torch.FloatTensor(dones))
state_action_values = self.policy_net(states_v).gather(
1, actions_v.unsqueeze(1)).squeeze(1)
next_state_values = self.target_net(next_states_v).max(1)[0]
expected_q_value = rewards_v + next_state_values * GAMMA * (
1 - done) #+ rewards_v
loss = (state_action_values -
Variable(expected_q_value.data)).pow(2).mean()
return loss
def train(self):
optimizer = optim.Adam(self.policy_net.parameters(), lr=ALPHA)
# Fill the memory with experiences
print('Gathering experiences ...')
meanScore = 0
AvgRewards = []
AllScores = []
step = 1
iEpisode = 0
while meanScore < 50:
state = self.env.reset()
done = False
EpisodeScore = 0
tBegin = time.time()
done = False
while not done:
action = self.make_action(state)
nextState, reward, done, _ = self.env.step(action)
self.push(state.transpose(2, 0, 1), action,
nextState.transpose(2, 0, 1), reward, done)
state = nextState
if len(memory) > StartLearning:
loss = self.optimize_model()
optimizer.zero_grad()
loss.backward()
optimizer.step()
else:
iEpisode = 0
continue
# Update exploration factor
EPSILON = EPS_END + (EPS_START - EPS_END) * math.exp(
-1. * step / EPS_DECAY)
storeEpsilon.append(EPSILON)
step += 1
EpisodeScore += reward
if step % TARGET_UPDATE == 0:
print('Updating Target Network . . .')
self.target_net.load_state_dict(
self.policy_net.state_dict())
iEpisode += 1
AllScores.append(EpisodeScore)
meanScore = np.mean(AllScores[-100:])
AvgRewards.append(meanScore)
if len(memory) > StartLearning:
print('Episode: ', iEpisode, ' score:', EpisodeScore,
' Avg Score:', meanScore, ' epsilon: ', EPSILON, ' t: ',
time.time() - tBegin, ' loss:', loss.item())
else:
print('Gathering Data . . .')
if iEpisode % 500 == 0:
torch.save(
{
'epoch': iEpisode,
'model_state_dict': self.policy_net.state_dict(),
'optimizer_state_dict': optimizer.state_dict(),
'loss': loss,
'AvgRewards': AvgRewards
}, 'trainData')
os.remove("Rewards.csv")
with open('Rewards.csv', mode='w') as dataFile:
rewardwriter = csv.writer(dataFile,
delimiter=',',
quotechar='"',
quoting=csv.QUOTE_MINIMAL)
rewardwriter.writerow(AvgRewards)
print('======== Complete ========')
torch.save(
{
'epoch': iEpisode,
'model_state_dict': self.policy_net.state_dict(),
'optimizer_state_dict': optimizer.state_dict(),
'loss': loss,
'AvgRewards': AvgRewards
}, 'trainData')
with open('Rewards.csv', mode='w') as dataFile:
rewardwriter = csv.writer(dataFile,
delimiter=',',
quotechar='"',
quoting=csv.QUOTE_MINIMAL)
rewardwriter.writerow(AvgRewards)
def ddqn_train(env, scheduler, optimizer_constructor, model_type, batch_size,
rp_start, rp_size, exp_frame, exp_initial, exp_final, gamma,
target_update_steps, frames_per_epoch, frames_per_state,
output_directory, last_checkpoint, envo):
"""
Implementation of the training algorithm for DDQN.
"""
gym.undo_logger_setup()
logging.basicConfig(filename=envo + '_' + model_type + 'ddqn_training.log',
level=logging.INFO)
num_actions = env.action_space.n
env.reset()
print('No. of actions: ', num_actions)
print(env.unwrapped.get_action_meanings())
# initialize action value and target network with the same weights
model = DQN(num_actions, use_bn=False)
target = DQN(num_actions, use_bn=False)
if use_cuda:
model.cuda()
target.cuda()
exp_replay = None
episodes_count = 1
if last_checkpoint:
model.load_state_dict(torch.load(last_checkpoint))
print(last_checkpoint)
print('weights loaded...')
exp_replay = initialize_replay_resume(env, rp_start, rp_size,
frames_per_state, model)
episodes_count = get_index_from_checkpoint_path(last_checkpoint)
else:
exp_replay = initialize_replay(env, rp_start, rp_size,
frames_per_state)
target.load_state_dict(model.state_dict())
# scheduler = Scheduler(exp_frame, exp_initial, exp_final)
optimizer = optimizer_constructor.type(
model.parameters(),
lr=optimizer_constructor.kwargs['lr'],
alpha=optimizer_constructor.kwargs['alpha'],
eps=optimizer_constructor.kwargs['eps'])
frames_count = 1
frames_per_episode = 1
epsiodes_durations = []
rewards_per_episode = 0
rewards_duration = []
loss_per_epoch = []
current_state, _, _, _ = play_game(env, frames_per_state)
print('Starting training...')
count = 0
while True:
epsilon = scheduler.anneal_linear(frames_count)
choice = random.uniform(0, 1)
# epsilon greedy algorithm
if choice <= epsilon:
action = LongTensor([[random.randrange(num_actions)]])
else:
action = get_greedy_action(model, current_state)
curr_obs, reward, done, _ = play_game(env, frames_per_state,
action[0][0])
rewards_per_episode += reward
reward = Tensor([reward])
exp_replay.push(current_state, action, reward, curr_obs)
current_state = curr_obs
#sample random mini-batch
obs_sample = exp_replay.sample(batch_size)
batch = Experience(
*zip(*obs_sample)
) #unpack the batch into states, actions, rewards and next_states
#compute y
if len(exp_replay) >= batch_size:
loss = ddqn_compute_y(batch, batch_size, model, target, gamma)
optimizer.zero_grad()
loss.backward()
for param in model.parameters():
param.grad.data.clamp_(-1, 1)
optimizer.step()
loss_per_epoch.append(loss.data.cpu().numpy())
frames_count += 1
frames_per_episode += frames_per_state
if done:
rewards_duration.append(rewards_per_episode)
rewards_per_episode = 0
frames_per_episode = 1
episodes_count += 1
env.reset()
current_state, _, _, _ = play_game(env, frames_per_state)
if episodes_count % 100 == 0:
avg_episode_reward = sum(rewards_duration) / 100.0
avg_reward_content = 'Episode from', episodes_count - 99, ' to ', episodes_count, ' has an average of ', avg_episode_reward, ' reward and loss of ', sum(
loss_per_epoch)
print(avg_reward_content)
logging.info(avg_reward_content)
rewards_duration = []
loss_per_epoch = []
# update weights of target network for every TARGET_UPDATE_FREQ steps
if frames_count % target_update_steps == 0:
target.load_state_dict(model.state_dict())
# print('weights updated at frame no. ', frames_count)
#Save weights every 250k frames
if frames_count % 250000 == 0:
util.make_sure_path_exists(output_directory + '/' + envo + '/')
torch.save(
model.state_dict(), output_directory + envo + '_' +
model_type + '/weights_' + str(frames_count) + '.pth')
#Print frame count for every 1000000 (one million) frames:
if frames_count % 1000000 == 0:
training_update = 'frame count: ', frames_count, 'episode count: ', episodes_count, 'epsilon: ', epsilon
print(training_update)
logging.info(training_update)
class Agent_DQN(Agent):
def __init__(self, env, args):
"""
Initialize every things you need here.
For example: building your model
"""
super(Agent_DQN, self).__init__(env)
self.env = env
self.args = args
self.episode = 0
self.n_actions = self.env.action_space.n
self.epsilon_start = 1.0
self.epsilon_final = 0.025
self.epsilon_decay = 3000
self.epsilon_by_frame = lambda frame_idx: self.epsilon_final + (
self.epsilon_start - self.epsilon_final) * math.exp(
-1. * frame_idx / self.epsilon_decay)
self.epsilon = 0
self.eval_net = DQN().cuda()
self.target_net = DQN().cuda()
self.target_net.load_state_dict(self.eval_net.state_dict())
self.criterion = nn.MSELoss()
#self._model = Net(self.env.observation_space.shape, self.env.action_space.n)
self._use_cuda = torch.cuda.is_available()
self.optim = torch.optim.Adam(self.eval_net.parameters(),
lr=self.args.learning_rate)
if self._use_cuda:
self.eval_net = self.eval_net.cuda()
self.target_net = self.target_net.cuda()
self.criterion = self.criterion.cuda()
# self.replaybuffer = ReplayBuffer(args.buffer_size)
self.buffer = deque(maxlen=10000)
if args.test_dqn:
#you can load your model here
print('loading trained model')
self.eval_net.load_state_dict(torch.load(args.model_dqn))
self.target_net.load_state_dict(self.eval_net.state_dict())
if self._use_cuda:
self.eval_net = self.eval_net.cuda()
self.target_net = self.target_net.cuda()
##################
# YOUR CODE HERE #
##################
def init_game_setting(self):
"""
Testing function will call this function at the begining of new game
Put anything you want to initialize if necessary
"""
##################
# YOUR CODE HERE #
##################
pass
def push(self, state, action, reward, next_state, done):
state = np.expand_dims(state, 0)
next_state = np.expand_dims(next_state, 0)
self.buffer.append((state, action, reward, next_state, done))
def replay_buffer(self, batch_size):
state, action, reward, next_state, done = zip(
*random.sample(self.buffer, batch_size))
return np.concatenate(state), action, reward, np.concatenate(
next_state), done
def train(self):
"""
Implement your training algorithm here
"""
##################
# YOUR CODE HERE #
##################
print('begin train...')
# if self.args.log_file is not None:
# fp_log = open(self.args.log_file, 'w', buffering=1)
fout = open('dqn_score.log', 'w')
if os.path.exists('model') == False:
os.makedirs('model')
losses = []
all_rewards = []
episode_reward = 0
best_mean_reward = 0
state = self.env.reset()
for i_step in range(self.args.max_steps):
self.epsilon = self.epsilon_by_frame(i_step)
action = self.make_action(state)
next_state, reward, done, _ = self.env.step(action)
self.push(state, action, reward, next_state, done)
state = next_state
episode_reward += reward
if done:
state = self.env.reset()
all_rewards.append(episode_reward)
self.episode += 1
print('{},{}'.format(self.episode, episode_reward))
fout.write('Episode{},episode_reward{}\n'.format(
self.episode, episode_reward))
episode_reward = 0
if len(self.buffer) == self.args.buffer_size:
if i_step % self.args.eval_net_update_step == 0:
loss = self.optimize_model()
losses.append(loss)
if i_step % self.args.target_net_update_step == 0:
self.target_net.load_state_dict(self.eval_net.state_dict())
if i_step % self.args.save_freq == 0:
mean_reward = \
sum(all_rewards[-100:]) / 100
if best_mean_reward < mean_reward:
print('save best model with mean reward = %f' %
mean_reward)
best_mean_reward = mean_reward
torch.save(self.eval_net.state_dict(), self.args.model_dqn)
def make_action(self, observation, test=True):
"""
Return predicted action of your agent
Input:
observation: np.array
stack 4 last preprocessed frames, shape: (84, 84, 4)
Return:
action: int
the predicted action from trained model
"""
##################
# YOUR CODE HERE #
##################
observation = torch.cuda.FloatTensor(
observation.reshape((1, 84, 84, 4))).transpose(1,
3).transpose(2, 3)
# print(type(observation))
Q_value = self.eval_net.forward(observation).data.cpu().numpy()
if random.random() > self.epsilon:
action = np.argmax(Q_value)
else:
action = self.env.get_random_action()
return action
def optimize_model(self):
state, action, reward, next_state, done = self.replay_buffer(
self.args.batch_size)
state = torch.FloatTensor(np.float32(state)).permute(0, 3, 1, 2)
next_state = torch.FloatTensor(np.float32(next_state)).permute(
0, 3, 1, 2)
action = torch.LongTensor(action)
reward = torch.FloatTensor(reward)
done = torch.ByteTensor(done)
if self._use_cuda:
state = state.cuda()
next_state = next_state.cuda()
action = action.cuda()
reward = reward.cuda()
done = done.cuda()
q_values = self.eval_net(state)
# next_q_values = self.target_net(next_state).detach()
q_value = q_values.gather(1, action.unsqueeze(1)).squeeze(1)
next_q_values = self.target_net(next_state).detach()
next_q_value = next_q_values.max(1)[0]
expected_q_value = reward + self.args.gamma * next_q_value * (1 - done)
loss = self.criterion(q_value, expected_q_value.data)
self.optim.zero_grad()
loss.backward()
self.optim.step()
return loss
class Agent_DQN(Agent):
def __init__(self, env, args):
"""
Initialize everything you need here.
For example:
paramters for neural network
initialize Q net and target Q net
parameters for repaly buffer
parameters for q-learning; decaying epsilon-greedy
...
"""
super(Agent_DQN, self).__init__(env)
###########################
# YOUR IMPLEMENTATION HERE #
self.env = env
self.batch_size = BATCH_SIZE
self.gamma = 0.999
self.eps_start = EPS_START
self.eps_decay = EPS_DECAY
self.TARGET_UPDATE = TARGET_UPDATE
self.policy_net = DQN(self.env.action_space.n)
self.target_net = DQN(self.env.action_space.n)
self.target_net.load_state_dict(self.policy_net.state_dict())
self.target_net.eval()
if use_cuda:
self.policy_net.cuda()
self.target_net.cuda()
self.optimizer = optim.RMSprop(self.policy_net.parameters(), lr=1e-5)
self.memory = deque(maxlen=10000)
if args.test_dqn:
# you can load your model here
print('loading trained model')
###########################
# YOUR IMPLEMENTATION HERE #
def init_game_setting(self):
"""
Testing function will call this function at the begining of new game
Put anything you want to initialize if necessary.
If no parameters need to be initialized, you can leave it as blank.
"""
###########################
# YOUR IMPLEMENTATION HERE #
###########################
pass
def make_action(self, observation, test=True):
"""
Return predicted action of your agent
Input:
observation: np.array
stack 4 last preprocessed frames, shape: (84, 84, 4)
Return:
action: int
the predicted action from trained model
"""
###########################
# YOUR IMPLEMENTATION HERE #
global steps_done
self.policy_net.eval()
sample = random.random()
eps_threshold = EPS_END + (EPS_START - EPS_END) * \
math.exp(-1. * steps_done / EPS_DECAY)
steps_done += 1
if sample > eps_threshold:
return self.policy_net(
Variable(torch.from_numpy(observation),
volatile=True).type(FloatTensor)).data.max(1)[1].view(
1, 1)
else:
return LongTensor([[random.randrange(self.env.action_space.n)]])
###########################
return action
def push(self, s, a, r, s_, done):
""" You can add additional arguments as you need.
Push new data to buffer and remove the old one if the buffer is full.
Hints:
-----
you can consider deque(maxlen = 10000) list
"""
###########################
# YOUR IMPLEMENTATION HERE #
self.memory.append((s, a, r, s_, done))
if len(self.memory) > self.maxlen:
self.replay_memory_store.popleft()
self.memory_counter += 1
###########################
def replay_buffer(self):
""" You can add additional arguments as you need.
Select batch from buffer.
"""
###########################
# YOUR IMPLEMENTATION HERE #
#print("memory", len(self.memory), self.BATCH_SIZE)
minibatch = random.sample(self.memory, self.BATCH_SIZE)
minibatch = np.array(minibatch).transpose(0, 3, 1, 2)
minibatch = torch.tensor(minibatch / 255.0)
###########################
return minibatch
def optimize_model(self):
if len(self.memory) < BATCH_SIZE:
return
transitions = self.memory.sample(BATCH_SIZE)
# Transpose the batch (see http://stackoverflow.com/a/19343/3343043 for
# detailed explanation).
batch = Transition(*zip(*transitions))
# Compute a mask of non-final states and concatenate the batch elements
non_final_mask = ByteTensor(
tuple(map(lambda s: s is not None, batch.next_state)))
non_final_next_states = Variable(torch.cat(
[s for s in batch.next_state if s is not None]),
volatile=True).cuda()
state_batch = Variable(torch.cat(batch.state)).cuda()
action_batch = Variable(torch.cat(batch.action)).cuda()
reward_batch = Variable(torch.cat(batch.reward)).cuda()
# Compute Q(s_t, a) - the model computes Q(s_t), then we select the
# columns of actions taken
self.policy_net.train()
state_action_values = self.policy_net(state_batch).gather(
1, action_batch)
# Compute V(s_{t+1}) for all next states.
next_state_values = Variable(
torch.zeros(BATCH_SIZE).type(Tensor)).cuda()
next_state_values[non_final_mask] = self.target_net(
non_final_next_states).max(1)[0]
# Compute the expected Q values
expected_state_action_values = (next_state_values *
GAMMA) + reward_batch
# Undo volatility (which was used to prevent unnecessary gradients)
expected_state_action_values = Variable(
expected_state_action_values.data).cuda()
# Compute Huber loss
loss = F.smooth_l1_loss(state_action_values,
expected_state_action_values)
# Optimize the model
self.optimizer.zero_grad()
loss.backward()
for param in self.policy_net.parameters():
param.grad.data.clamp_(-1, 1)
self.optimizer.step()
def train(self):
"""
Implement your training algorithm here
"""
###########################
# YOUR IMPLEMENTATION HERE #
num_episodes = 1400000
for i_episode in range(num_episodes):
# Initialize the environment and state
observation = self.env.reset()
observation = observation.transpose((2, 0, 1))
observation = observation[np.newaxis, :]
state = observation
for t in count():
# Select and perform an action
action = self.make_action(state)
next_state, reward, done, _ = self.env.step(action[0, 0])
next_state = next_state.transpose((2, 0, 1))
next_state = next_state[np.newaxis, :]
reward = Tensor([reward])
# Store the transition in memory
self.memory.push(torch.from_numpy(state), action,
torch.from_numpy(next_state), reward)
# Observe new state
if not done:
state = next_state
else:
state = None
# Perform one step of the optimization (on the target network)
self.optimize_model()
if done:
print(
'resetting env. episode %d \'s reward total was %d.' %
(i_episode + 1, t + 1))
break
# Update the target network
if i_episode % TARGET_UPDATE == 0:
self.target_net.load_state_dict(self.policy_net.state_dict())
if i_episode % 50 == 0:
checkpoint_path = os.path.join('save_dir', 'model-best.pth')
torch.save(self.policy_net.state_dict(), checkpoint_path)
print("model saved to {}".format(checkpoint_path))
def ddqn_rankWeight_train(env, exploreScheduler, betaScheduler, optimizer_constructor, model_type, batch_size, rp_start, rp_size,
exp_frame, exp_initial, exp_final, prob_alpha, gamma, target_update_steps, frames_per_epoch,
frames_per_state, output_directory, last_checkpoint, max_frames, envo):
"""
Implementation of the training algorithm for DDQN using Rank-based prioritization.
Information with regards to the algorithm can be found in the paper,
"Prioritized Experience Replay" by Tom Schaul, John Quan, Ioannis Antonoglou and
David Silver. Refer to section 3.3 in the paper for more info.
"""
gym.undo_logger_setup()
logging.basicConfig(filename=envo+'_'+'ddqn_rank_weightedLoss_training.log',level=logging.INFO)
num_actions = env.action_space.n
env.reset()
print('No. of actions: ', num_actions)
print(env.unwrapped.get_action_meanings())
# initialize action value and target network with the same weights
model = DQN(num_actions)
target = DQN(num_actions)
if use_cuda:
model.cuda()
target.cuda()
frames_count = 1
if last_checkpoint:
model.load_state_dict(torch.load(last_checkpoint))
print(last_checkpoint)
print('weights loaded...')
#TODO: Implementation of resume
# exp_replay = util.initialize_rank_replay_resume(env, rp_start, rp_size, frames_per_state,
# model, target, gamma, batch_size)
# frames_count = get_index_from_checkpoint_path(last_checkpoint)
else:
exp_replay = util.initialize_rank_replay(env, rp_start, rp_size, frames_per_state,
model, target, gamma, prob_alpha)
target.load_state_dict(model.state_dict())
optimizer = optimizer_constructor.type(model.parameters(), lr=optimizer_constructor.kwargs['lr'],
alpha=optimizer_constructor.kwargs['alpha'], eps=optimizer_constructor.kwargs['eps'] )
episodes_count = 1
frames_per_episode = 1
epsiodes_durations = []
rewards_per_episode = 0
rewards_duration = []
loss_per_epoch = []
wLoss_func = Weighted_Loss()
current_state, _, _, _ = util.play_game(env, frames_per_state)
print('Starting training...')
for frames_count in range(1, max_frames):
epsilon=exploreScheduler.anneal_linear(frames_count)
beta = betaScheduler.anneal_linear(frames_count)
choice = random.uniform(0,1)
# epsilon greedy algorithm
if choice <= epsilon:
action = LongTensor([[random.randrange(num_actions)]])
else:
action = util.get_greedy_action(model, current_state)
curr_obs, reward, done, _ = util.play_game(env, frames_per_state, action[0][0])
rewards_per_episode += reward
reward = Tensor([[reward]])
current_state_ex = Variable(current_state, volatile=True)
curr_obs_ex = Variable(curr_obs, volatile=True)
action_ex = Variable(action, volatile=True)
reward_ex = Variable(reward, volatile=True)
#compute td-error for one sample
td_error = ddqn_compute_td_error(batch_size=1, state_batch=current_state_ex, reward_batch=reward_ex, action_batch=action_ex,
next_state_batch=curr_obs_ex, model=model, target=target, gamma=gamma)
td_error = torch.pow(torch.abs(td_error)+1e-8, prob_alpha)
exp_replay.push(current_state, action, reward, curr_obs, td_error)
current_state = curr_obs
# compute y
if len(exp_replay) >= batch_size:
# Get batch samples
obs_samples, obs_ranks, obs_priorityVals = exp_replay.sample(batch_size)
num_samples_per_batch = len(obs_samples)
obs_priorityTensor = torch.from_numpy(np.array(obs_priorityVals))
p_batch = 1/ obs_priorityTensor
w_batch_raw = (1/len(exp_replay) * p_batch)**beta
max_weight = exp_replay.get_max_weight(beta)
w_batch = w_batch_raw/max_weight
w_batch = w_batch.type(Tensor)
batch = Experience(*zip(*obs_samples))
loss, new_weights = ddqn_compute_y(batch, num_samples_per_batch, model, target, gamma, w_batch, wLoss_func)
loss_abs = torch.abs(new_weights)
exp_replay.update(obs_ranks, loss_abs)
optimizer.zero_grad()
loss.backward()
for param in model.parameters():
param.grad.data.clamp_(-1,1)
optimizer.step()
loss_per_epoch.append(loss.data.cpu().numpy()[0])
frames_per_episode+= frames_per_state
if done:
rewards_duration.append(rewards_per_episode)
rewards_per_episode = 0
frames_per_episode=1
episodes_count+=1
env.reset()
current_state, _, _, _ = util.play_game(env, frames_per_state)
if episodes_count % 100 == 0:
avg_episode_reward = sum(rewards_duration)/100.0
avg_reward_content = 'Episode from', episodes_count-99, ' to ', episodes_count, ' has an average of ', avg_episode_reward, ' reward and loss of ', sum(loss_per_epoch)
print(avg_reward_content)
logging.info(avg_reward_content)
rewards_duration = []
loss_per_epoch = []
# update weights of target network for every TARGET_UPDATE_FREQ steps
if frames_count % target_update_steps == 0:
target.load_state_dict(model.state_dict())
# sort memory replay every half of it's capacity iterations
if frames_count % int(rp_size/2) == 0:
exp_replay.sort()
#Save weights every 250k frames
if frames_count % 250000 == 0:
util.make_sure_path_exists(output_directory+'/'+envo+'/')
torch.save(model.state_dict(), output_directory+'/'+envo+'/rank_weightedLoss_'+ str(frames_count)+'.pth')
#Print frame count and sort experience replay for every 1000000 (one million) frames:
if frames_count % 1000000 == 0:
training_update = 'frame count: ', frames_count, 'episode count: ', episodes_count, 'epsilon: ', epsilon
print(training_update)
logging.info(training_update)
def ddqn_rank_train(env, exploreScheduler, betaScheduler,
optimizer_constructor, model_type, batch_size, rp_start,
rp_size, exp_frame, exp_initial, exp_final, prob_alpha,
gamma, target_update_steps, frames_per_epoch,
frames_per_state, output_directory, last_checkpoint,
max_frames, envo):
"""
Implementation of the training algorithm for DDQN using Rank-based prioritization.
Information with regards to the algorithm can be found in the paper,
"Prioritized Experience Replay" by Tom Schaul, John Quan, Ioannis Antonoglou and
David Silver. Refer to section 3.3 in the paper for more info.
"""
gym.undo_logger_setup()
logging.basicConfig(filename=envo + '_' +
'ddqn_rank_weighted_training.log',
level=logging.INFO)
num_actions = env.action_space.n
env.reset()
print('No. of actions: ', num_actions)
print(env.unwrapped.get_action_meanings())
# initialize action value and target network with the same weights
model = DQN(num_actions)
target = DQN(num_actions)
if use_cuda:
model.cuda()
target.cuda()
frames_count = 1
if last_checkpoint:
model.load_state_dict(torch.load(last_checkpoint))
print(last_checkpoint)
print('weights loaded...')
#TODO: Implementation of resume
# exp_replay = util.initialize_rank_replay_resume(env, rp_start, rp_size, frames_per_state,
# model, target, gamma, batch_size)
# frames_count = get_index_from_checkpoint_path(last_checkpoint)
else:
exp_replay = util.initialize_rank_replay(env, rp_start, rp_size,
frames_per_state, model,
target, gamma, prob_alpha)
target.load_state_dict(model.state_dict())
optimizer = optimizer_constructor.type(
model.parameters(),
lr=optimizer_constructor.kwargs['lr'],
alpha=optimizer_constructor.kwargs['alpha'],
eps=optimizer_constructor.kwargs['eps'])
episodes_count = 1
epsiodes_durations = []
rewards_per_episode = 0
rewards_duration = []
loss_per_epoch = []
current_state, _, _, _ = util.play_game(env, frames_per_state)
wLoss_func = Weighted_Loss()
print('Starting training...')
for frames_count in range(1, max_frames):
epsilon = exploreScheduler.anneal_linear(frames_count)
beta = betaScheduler.anneal_linear(frames_count)
choice = random.uniform(0, 1)
# epsilon greedy algorithm
if choice <= epsilon:
action = LongTensor([[random.randrange(num_actions)]])
else:
action = util.get_greedy_action(model, current_state)
curr_obs, reward, done, _ = util.play_game(env, frames_per_state,
action[0][0])
rewards_per_episode += reward
reward = Tensor([[reward]])
td_error = 1
temp_exp = Experience(current_state, action, reward, curr_obs,
td_error)
current_state = curr_obs
# compute y
if len(exp_replay) >= batch_size:
# Get batch samples
# start = time.time()
if frames_count % rp_size == 0:
obs_samples, obs_priorityVals = exp_replay.sample(batch_size -
1,
prob_alpha,
sort=True)
else:
obs_samples, obs_priorityVals = exp_replay.sample(batch_size -
1,
prob_alpha,
sort=False)
obs_samples.append(temp_exp)
obs_priorityVals.append(td_error)
obs_pVals_tensor = torch.from_numpy(np.array(obs_priorityVals))
# print("P(i): ", obs_pVals_tensor)
IS_weights = torch.pow((obs_pVals_tensor * rp_size), -beta)
max_weight = torch.max(IS_weights)
IS_weights_norm = torch.div(IS_weights, max_weight).type(Tensor)
IS_weights_norm[-1] = torch.max(IS_weights_norm)
# print("Norm W(i): ", IS_weights_norm)
batch = Experience(*zip(*obs_samples))
loss, new_weights = ddqn_compute_y(batch, batch_size, model,
target, gamma, IS_weights_norm,
wLoss_func)
new_weights = torch.pow(new_weights, prob_alpha)
new_exp = Experience(temp_exp.state, temp_exp.action,
temp_exp.reward, temp_exp.next_state,
new_weights[batch_size - 1])
exp_replay.update(obs_samples, new_weights, new_exp)
optimizer.zero_grad()
loss.backward()
# print("loss: ", loss.data)
optimizer.step()
loss_per_epoch.append(loss.data.cpu().numpy()[0])
else:
exp_replay.push(new_exp.state, new_exp.action, new_exp.reward,
new_exp.next_state, td_error)
# end = time.time()
# duration = end-start
# print('duration : ', duration)
if done:
# print('Game: ', rewards_per_episode)
rewards_duration.append(rewards_per_episode)
rewards_per_episode = 0
episodes_count += 1
env.reset()
current_state, _, _, _ = util.play_game(env, frames_per_state)
if episodes_count % 100 == 0:
avg_episode_reward = sum(rewards_duration) / 100.0
avg_reward_content = 'Episode from', episodes_count - 99, ' to ', episodes_count, ' has an average of ', avg_episode_reward, ' reward and loss of ', sum(
loss_per_epoch)
print(avg_reward_content)
logging.info(avg_reward_content)
rewards_duration = []
loss_per_epoch = []
# update weights of target network for every TARGET_UPDATE_FREQ steps
if frames_count % target_update_steps == 0:
target.load_state_dict(model.state_dict())
#Save weights every 250k frames
if frames_count % 250000 == 0:
util.make_sure_path_exists(output_directory + '/' + envo + '/')
torch.save(
model.state_dict(), output_directory + '/' + envo +
'/rank_uniform' + str(frames_count) + '.pth')
#Print frame count and sort experience replay for every 1000000 (one million) frames:
if frames_count % 1000000 == 0:
training_update = 'frame count: ', frames_count, 'episode count: ', episodes_count, 'epsilon: ', epsilon
print(training_update)
logging.info(training_update)
class Agent_DQN(Agent):
def __init__(self, env, args):
"""
Initialize everything you need here.
For example:
paramters for neural network
initialize Q net and target Q net
parameters for repaly buffer
parameters for q-learning; decaying epsilon-greedy
...
"""
super(Agent_DQN,self).__init__(env)
###########################
# YOUR IMPLEMENTATION HERE #
self.epsilon_start = 1
self.epsilon_end = 0.02
self.epsilon_decay = 200000
self.epsilon = self.epsilon_start
self.gamma = 0.99
self.env = env
self.buffer_size = 30000
self.buffer = deque(maxlen=30000)
self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
self.policy_net = DQN().to(self.device)
self.target_net = DQN().to(self.device)
self.target_net.load_state_dict(self.policy_net.state_dict())
self.optimizer = torch.optim.Adam(self.policy_net.parameters(),lr=0.00015)
self.reward_array = []
self.reward_x_axis = []
self.batch_size = 32
if args.test_dqn:
#you can load your model here
print('loading trained model')
###########################
# YOUR IMPLEMENTATION HERE #
def init_game_setting(self):
"""
Testing function will call this function at the begining of new game
Put anything you want to initialize if necessary.
If no parameters need to be initialized, you can leave it as blank.
"""
###########################
# YOUR IMPLEMENTATION HERE #
self.policy_net.load_state_dict(torch.load('policy_model'))
self.target_net.load_state_dict(self.policy_net.state_dict())
self.policy_net = self.policy_net.cuda()
self.target_net = self.target_net.cuda()
###########################
pass
def make_action(self, observation, test=True):
"""
Return predicted action of your agent
Input:
observation: np.array
stack 4 last preprocessed frames, shape: (84, 84, 4)
Return:
action: int
the predicted action from trained model
"""
###########################
# YOUR IMPLEMENTATION HERE #
if test==True:
self.epsilon = 0
observation=torch.cuda.FloatTensor(observation.reshape((1,84,84,4))).transpose(1,3).transpose(2,3)
q = self.policy_net(observation).data.cpu().numpy()
if random.random() > self.epsilon:
action = np.argmax(q)
else:
action = random.randint(0,4)
return action
def push(self,data):
""" You can add additional arguments as you need.
Push new data to buffer and remove the old one if the buffer is full.
Hints:
-----
you can consider deque(maxlen = 10000) list
"""
###########################
# YOUR IMPLEMENTATION HERE #
self.buffer.append(data)
###########################
def replay_buffer(self,batch_size):
""" You can add additional arguments as you need.
Select batch from buffer.
"""
###########################
# YOUR IMPLEMENTATION HERE #
###########################
return random.sample(self.buffer,batch_size)
def play_game(self,start_state):
action = self.make_action(start_state)
n_s,r,terminal,_ = self.env.step(action)
self.push((start_state,action,r,n_s,terminal))
return n_s,r,terminal
def loss_function(self):
data = self.replay_buffer(self.batch_size)
s,a,r,n_s,terminal = zip(*data)
s = torch.FloatTensor(np.float32(s)).permute(0,3,1,2).to(self.device)
a = torch.LongTensor(a).to(self.device)
r = torch.FloatTensor(r).to(self.device)
n_s = torch.FloatTensor(np.float32(n_s)).permute(0,3,1,2).to(self.device).to(self.device)
terminal = torch.FloatTensor(terminal).to(self.device)
q = self.policy_net(s).gather(1,a.unsqueeze(1)).squeeze(1)
n_q = self.target_net(n_s).detach().max(1)[0]
expected_q = r + self.gamma * n_q * (1 - terminal)
loss = F.smooth_l1_loss(q, expected_q.data)
self.optimizer.zero_grad()
loss.backward()
self.optimizer.step()
def train(self):
"""
Implement your training algorithm here
"""
###########################
# YOUR IMPLEMENTATION HERE #
rewards_array = []
reward_ = 0
best_mean = 0
print_rate = 100
last_saved = None
start_state = self.env.reset()
for frames in range (3500000):
self.epsilon = self.epsilon_end + (self.epsilon_start - self.epsilon_end) * math.exp(-1. *frames / self.epsilon_decay)
n_s,r,terminal = self.play_game(start_state)
start_state = n_s
reward_ += r
if terminal:
start_state = self.env.reset()
rewards_array.append(reward_)
if len(rewards_array) % print_rate==0:
print('%%%%%%%%%%%%%%%%%%%%%%%%%')
print('Frames = ', frames)
print('Current Epsilon = ', self.epsilon)
print('Episode = ', len(rewards_array))
print('Reward = ', np.mean(rewards_array[-100:]))#sum(rewards_array[-100:]) / 100)
print('Buffer Length = ', len(self.buffer))
self.reward_array.append(np.mean(rewards_array[-100:]))
self.reward_x_axis.append(len(rewards_array))
self.print_graph()
if last_saved != None:
print("last saved = ", best_mean)
print('%%%%%%%%%%%%%%%%%%%%%%%%%')
reward_ = 0
if len(self.buffer)<10000:
continue
if len(self.buffer) > 10000 and frames % 4 ==0:
self.loss_function()
if frames % 1000 == 0:
print("Target net updated")
self.target_net.load_state_dict(self.policy_net.state_dict())
mean_reward = np.mean(rewards_array[-100:])
if mean_reward > best_mean and frames % 100==0:
print("Saving model with reward = ", mean_reward)
best_mean = mean_reward
last_saved = mean_reward
torch.save(self.policy_net.state_dict(), 'policy_model_')
###########################
def print_graph(self):
fig = plt.figure()
ax = plt.subplot(111)
ax.plot(self.reward_x_axis,self.reward_array,label='$y = Rewards, $x = episodes')
ax.legend()
fig.savefig('plot.png')
def ddqn_rankBatch_train(env, scheduler, optimizer_constructor, model_type,
batch_size, rp_start, rp_size, exp_frame, exp_initial,
exp_final, inital_beta, gamma, target_update_steps,
frames_per_epoch, frames_per_state, output_directory,
last_checkpoint):
"""
Implementation of the training algorithm for DDQN using Rank-based prioritization.
Information with regards to the algorithm can be found in the paper,
"Prioritized Experience Replay" by Tom Schaul, John Quan, Ioannis Antonoglou and
David Silver. Refer to section 3.3 in the paper for more info.
"""
gym.undo_logger_setup()
logging.basicConfig(filename='ddqn_rank_training.log', level=logging.INFO)
num_actions = env.action_space.n
env.reset()
print('No. of actions: ', num_actions)
print(env.unwrapped.get_action_meanings())
# initialize action value and target network with the same weights
model = DQN(num_actions, use_bn=False)
target = DQN(num_actions, use_bn=False)
if use_cuda:
model.cuda()
target.cuda()
frames_count = 1
if last_checkpoint:
model.load_state_dict(torch.load(last_checkpoint))
print(last_checkpoint)
print('weights loaded...')
exp_replay = util.initialize_rank_replay_resume(
env, rp_start, rp_size, frames_per_state, model, target, gamma,
batch_size)
frames_count = get_index_from_checkpoint_path(last_checkpoint)
else:
exp_replay = util.initialize_rank_replay(env, rp_start, rp_size,
frames_per_state, model,
target, gamma)
target.load_state_dict(model.state_dict())
optimizer = optimizer_constructor.type(
model.parameters(),
lr=optimizer_constructor.kwargs['lr'],
alpha=optimizer_constructor.kwargs['alpha'],
eps=optimizer_constructor.kwargs['eps'])
episodes_count = 1
frames_per_episode = 1
epsiodes_durations = []
rewards_per_episode = 0
rewards_duration = []
loss_per_epoch = []
current_state, _, _, _ = util.play_game(env, frames_per_state)
print('Starting training...')
count = 0
while True:
epsilon = scheduler.anneal_linear(frames_count)
choice = random.uniform(0, 1)
# epsilon greedy algorithm
if choice <= epsilon:
action = LongTensor([[random.randrange(num_actions)]])
else:
action = util.get_greedy_action(model, current_state)
curr_obs, reward, done, _ = util.play_game(env, frames_per_state,
action[0][0])
rewards_per_episode += reward
reward = Tensor([[reward]])
current_state_ex = Variable(current_state, volatile=True)
curr_obs_ex = Variable(curr_obs, volatile=True)
action_ex = Variable(action, volatile=True)
reward_ex = Variable(reward, volatile=True)
#compute td-error for one sample
td_error = ddqn_compute_td_error(batch_size=1,
state_batch=current_state_ex,
reward_batch=reward_ex,
action_batch=action_ex,
next_state_batch=curr_obs_ex,
model=model,
target=target,
gamma=gamma)
td_error = torch.abs(td_error)
exp_replay.push(current_state_ex, action_ex, reward_ex, curr_obs_ex,
td_error)
current_state = curr_obs
# compute y
if len(exp_replay) >= batch_size:
# Get batch samples
obs_samples, obs_ranks, obs_priorityVals = exp_replay.sample(
batch_size)
obs_priorityTensor = torch.from_numpy(np.array(obs_priorityVals))
p_batch = 1 / obs_priorityTensor
w_batch = (1 / len(exp_replay) * p_batch)**inital_beta
max_weight = exp_replay.get_max_weight(inital_beta)
params_grad = []
for i in range(len(obs_samples)):
sample = obs_samples[i]
sample.state.volatile = False
sample.next_state.volatile = False
sample.reward.volatile = False
sample.action.volatile = False
loss = ddqn_compute_y(batch_size=1,
state_batch=sample.state,
reward_batch=sample.reward,
action_batch=sample.action,
next_state_batch=sample.next_state,
model=model,
target=target,
gamma=gamma)
loss_abs = torch.abs(loss)
exp_replay.update(obs_ranks[i], loss_abs)
for param in model.parameters():
if param.grad is not None:
param.grad.data.zero_()
loss.backward()
#accumulate weight change
if i == 0:
for param in model.parameters():
tmp = ((w_batch[i] / max_weight) *
loss.data[0]) * param.grad.data
params_grad.append(tmp)
else:
paramIndex = 0
for param in model.parameters():
tmp = ((w_batch[i] / max_weight) *
loss.data[0]) * param.grad.data
params_grad[paramIndex] = tmp + params_grad[paramIndex]
paramIndex += 1
# update weights
paramIndex = 0
for param in model.parameters():
param.data += params_grad[paramIndex].mul(
optimizer_constructor.kwargs['lr']).type(Tensor)
paramIndex += 1
frames_count += 1
frames_per_episode += frames_per_state
if done:
rewards_duration.append(rewards_per_episode)
rewards_per_episode = 0
frames_per_episode = 1
episodes_count += 1
env.reset()
current_state, _, _, _ = util.play_game(env, frames_per_state)
if episodes_count % 100 == 0:
avg_episode_reward = sum(rewards_duration) / 100.0
avg_reward_content = 'Episode from', episodes_count - 99, ' to ', episodes_count, ' has an average of ', avg_episode_reward, ' reward and loss of ', sum(
loss_per_epoch)
print(avg_reward_content)
logging.info(avg_reward_content)
rewards_duration = []
loss_per_epoch = []
# update weights of target network for every TARGET_UPDATE_FREQ steps
if frames_count % target_update_steps == 0:
target.load_state_dict(model.state_dict())
# print('weights updated at frame no. ', frames_count)
#Save weights every 250k frames
if frames_count % 250000 == 0:
util.make_sure_path_exists(output_directory + model_type + '/')
torch.save(model.state_dict(),
'rank_weights_' + str(frames_count) + '.pth')
#Print frame count and sort experience replay for every 1000000 (one million) frames:
if frames_count % 1000000 == 0:
training_update = 'frame count: ', frames_count, 'episode count: ', episodes_count, 'epsilon: ', epsilon
print(training_update)
logging.info(training_update)
exp_replay.sort()
