def trainSQL0(file_name="SQL0",
env=GridworldEnv(1),
batch_size=128,
gamma=0.999,
beta=5,
eps_start=0.9,
eps_end=0.05,
eps_decay=1000,
is_plot=False,
num_episodes=200,
max_num_steps_per_episode=1000,
learning_rate=0.0001,
memory_replay_size=10000,
n_step=10,
target_update=10):
"""
Soft Q-learning training routine when observation vector is input
Retuns rewards and durations logs.
"""
num_actions = env.action_space.n
input_size = env.observation_space.shape[0]
model = DQN(input_size, num_actions)
target_model = DQN(input_size, num_actions)
target_model.load_state_dict(model.state_dict())
optimizer = optim.Adam(model.parameters(), lr=learning_rate)
# optimizer = optim.RMSprop(model.parameters(), )
use_cuda = torch.cuda.is_available()
if use_cuda:
model.cuda()
memory = ReplayMemory(memory_replay_size, n_step, gamma)
episode_durations = []
mean_durations = []
episode_rewards = []
mean_rewards = []
steps_done, t = 0, 0
for i_episode in range(num_episodes):
if i_episode % 20 == 0:
clear_output()
if i_episode != 0:
print("Cur episode:", i_episode, "steps done:", episode_durations[-1],
"exploration factor:", eps_end + (eps_start - eps_end) * \
math.exp(-1. * steps_done / eps_decay), "reward:", env.episode_total_reward)
# Initialize the environment and state
state = torch.from_numpy(env.reset()).type(torch.FloatTensor).view(
-1, input_size)
for t in count():
# Select and perform an action
action = select_action(state, model, num_actions, eps_start,
eps_end, eps_decay, steps_done)
next_state_tmp, reward, done, _ = env.step(action[0, 0])
reward = Tensor([reward])
# Observe new state
next_state = torch.from_numpy(next_state_tmp).type(
torch.FloatTensor).view(-1, input_size)
if done:
next_state = None
# Store the transition in memory
memory.push(model, target_model, state, action, next_state, reward)
# Move to the next state
state = next_state
# plot_state(state)
# env.render()
# Perform one step of the optimization (on the target network)
optimize_model(model, target_model, optimizer, memory, batch_size,
gamma, beta) #### Difference w.r.t DQN
if done or t + 1 >= max_num_steps_per_episode:
episode_durations.append(t + 1)
episode_rewards.append(
env.episode_total_reward
) ##### Modify for OpenAI envs such as CartPole
if is_plot:
plot_durations(episode_durations, mean_durations)
plot_rewards(episode_rewards, mean_rewards)
steps_done += 1
break
if i_episode % target_update == 0 and i_episode != 0:
target_model.load_state_dict(model.state_dict())
print('Complete')
env.render(close=True)
env.close()
if is_plot:
plt.ioff()
plt.show()
## Store Results
np.save(file_name + '-sql0-rewards', episode_rewards)
np.save(file_name + '-sql0-durations', episode_durations)
return model, episode_rewards, episode_durations
def trainDQN(file_name="DQN",
env=GridworldEnv(1),
batch_size=128,
gamma=0.999,
eps_start=0.9,
eps_end=0.05,
eps_decay=1000,
is_plot=False,
num_episodes=500,
max_num_steps_per_episode=1000,
learning_rate=0.0001,
memory_replay_size=10000):
"""
DQN training routine. Retuns rewards and durations logs.
Plot environment screen
"""
if is_plot:
env.reset()
plt.ion()
plt.figure()
plt.imshow(get_screen(env).cpu().squeeze(0).squeeze(0).numpy(),
interpolation='none')
plt.title("")
plt.draw()
plt.pause(0.00001)
num_actions = env.action_space.n
model = DQN(num_actions)
optimizer = optim.Adam(model.parameters(), lr=learning_rate)
use_cuda = torch.cuda.is_available()
if use_cuda:
model.cuda()
memory = ReplayMemory(memory_replay_size)
episode_durations = []
mean_durations = []
episode_rewards = []
mean_rewards = []
steps_done = 0 # total steps
for i_episode in range(num_episodes):
if i_episode % 20 == 0:
clear_output()
print("Cur episode:", i_episode, "steps done:", steps_done,
"exploration factor:", eps_end + (eps_start - eps_end) * \
math.exp(-1. * steps_done / eps_decay))
# Initialize the environment and state
env.reset()
# last_screen = env.current_grid_map
# (1, 1, 8, 8)
current_screen = get_screen(env)
state = current_screen # - last_screen
for t in count():
# Select and perform an action
action = select_action(state, model, num_actions, eps_start,
eps_end, eps_decay, steps_done)
steps_done += 1
_, reward, done, _ = env.step(action[0, 0])
reward = Tensor([reward])
# Observe new state
last_screen = current_screen
current_screen = get_screen(env)
if not done:
next_state = current_screen # - last_screen
else:
next_state = None
# Store the transition in memory
memory.push(state, action, next_state, reward)
# Move to the next state
state = next_state
# plot_state(state)
# env.render()
# Perform one step of the optimization (on the target network)
optimize_model(model, optimizer, memory, batch_size, gamma)
if done or t + 1 >= max_num_steps_per_episode:
episode_durations.append(t + 1)
episode_rewards.append(env.episode_total_reward)
if is_plot:
plot_durations(episode_durations, mean_durations)
plot_rewards(episode_rewards, mean_rewards)
break
print('Complete')
env.render(close=True)
env.close()
if is_plot:
plt.ioff()
plt.show()
## Store Results
np.save(file_name + '-dqn-rewards', episode_rewards)
np.save(file_name + '-dqn-durations', episode_durations)
return model, episode_rewards, episode_durations
class DQNAgent():
"""Deep Q-learning agent."""
# def __init__(self,
# env, device=DEVICE, summary_writer=writer, # noqa
# hyperparameters=DQN_HYPERPARAMS): # noqa
rewards = []
total_reward = 0
birth_time = 0
n_iter = 0
n_games = 0
ts_frame = 0
ts = time.time()
# Memory = namedtuple(
# 'Memory', ['obs', 'action', 'new_obs', 'reward', 'done'],
# verbose=False, rename=False)
Memory = namedtuple('Memory',
['obs', 'action', 'new_obs', 'reward', 'done'],
rename=False)
def __init__(self, env, hyperparameters, device, summary_writer=None):
"""Set parameters, initialize network."""
state_space_shape = env.observation_space.shape
action_space_size = env.action_space.n
self.env = env
self.online_network = DQN(state_space_shape,
action_space_size).to(device)
self.target_network = DQN(state_space_shape,
action_space_size).to(device)
# XXX maybe not really necesary?
self.update_target_network()
self.experience_replay = None
self.accumulated_loss = []
self.device = device
self.optimizer = optim.Adam(self.online_network.parameters(),
lr=hyperparameters['learning_rate'])
self.double_DQN = hyperparameters['double_DQN']
# Discount factor
self.gamma = hyperparameters['gamma']
# XXX ???
self.n_multi_step = hyperparameters['n_multi_step']
self.replay_buffer = ReplayBuffer(hyperparameters['buffer_capacity'],
hyperparameters['n_multi_step'],
hyperparameters['gamma'])
self.birth_time = time.time()
self.iter_update_target = hyperparameters['n_iter_update_target']
self.buffer_start_size = hyperparameters['buffer_start_size']
self.summary_writer = summary_writer
# Greedy search hyperparameters
self.epsilon_start = hyperparameters['epsilon_start']
self.epsilon = hyperparameters['epsilon_start']
self.epsilon_decay = hyperparameters['epsilon_decay']
self.epsilon_final = hyperparameters['epsilon_final']
def get_max_action(self, obs):
'''
Forward pass of the NN to obtain the action of the given observations
'''
# convert the observation in tensor
state_t = torch.tensor(np.array([obs])).to(self.device)
# forward pass
q_values_t = self.online_network(state_t)
# get the maximum value of the output (i.e. the best action to take)
_, act_t = torch.max(q_values_t, dim=1)
return int(act_t.item())
def act(self, obs):
'''
Greedy action outputted by the NN in the CentralControl
'''
return self.get_max_action(obs)
def act_eps_greedy(self, obs):
'''
E-greedy action
'''
# In case of a noisy net, it takes a greedy action
# if self.noisy_net:
# return self.act(obs)
if np.random.random() < self.epsilon:
return self.env.action_space.sample()
else:
return self.act(obs)
def update_target_network(self):
"""Update target network weights with current online network values."""
self.target_network.load_state_dict(self.online_network.state_dict())
def set_optimizer(self, learning_rate):
self.optimizer = optim.Adam(self.online_network.parameters(),
lr=learning_rate)
def sample_and_optimize(self, batch_size):
'''
Sample batch_size memories from the buffer and optimize them
'''
# This should be the part where it waits until it has enough
# experience
if len(self.replay_buffer) > self.buffer_start_size:
# sample
mini_batch = self.replay_buffer.sample(batch_size)
# optimize
# l_loss = self.cc.optimize(mini_batch)
l_loss = self.optimize(mini_batch)
self.accumulated_loss.append(l_loss)
# update target NN
if self.n_iter % self.iter_update_target == 0:
self.update_target_network()
def optimize(self, mini_batch):
'''
Optimize the NN
'''
# reset the grads
self.optimizer.zero_grad()
# caluclate the loss of the mini batch
loss = self._calulate_loss(mini_batch)
loss_v = loss.item()
# do backpropagation
loss.backward()
# one step of optimization
self.optimizer.step()
return loss_v
def _calulate_loss(self, mini_batch):
'''
Calculate mini batch's MSE loss.
It support also the double DQN version
'''
states, actions, next_states, rewards, dones = mini_batch
# convert the data in tensors
states_t = torch.as_tensor(states, device=self.device)
next_states_t = torch.as_tensor(next_states, device=self.device)
actions_t = torch.as_tensor(actions, device=self.device)
rewards_t = torch.as_tensor(rewards,
dtype=torch.float32,
device=self.device)
done_t = torch.as_tensor(dones, dtype=torch.uint8,
device=self.device) # noqa
# Value of the action taken previously (recorded in actions_v)
# in state_t
state_action_values = self.online_network(states_t).gather(
1, actions_t[:, None]).squeeze(-1)
# NB gather is a differentiable function
# Next state value with Double DQN. (i.e. get the value predicted
# by the target nn, of the best action predicted by the online nn)
if self.double_DQN:
double_max_action = self.online_network(next_states_t).max(1)[1]
double_max_action = double_max_action.detach()
target_output = self.target_network(next_states_t)
# NB: [:,None] add an extra dimension
next_state_values = torch.gather(
target_output, 1, double_max_action[:, None]).squeeze(-1)
# Next state value in the normal configuration
else:
next_state_values = self.target_network(next_states_t).max(1)[0]
next_state_values = next_state_values.detach() # No backprop
# Use the Bellman equation
expected_state_action_values = rewards_t + \
(self.gamma**self.n_multi_step) * next_state_values
# compute the loss
return nn.MSELoss()(state_action_values, expected_state_action_values)
def reset_stats(self):
'''
Reset the agent's statistics
'''
self.rewards.append(self.total_reward)
self.total_reward = 0
self.accumulated_loss = []
self.n_games += 1
def add_env_feedback(self, obs, action, new_obs, reward, done):
'''
Acquire a new feedback from the environment. The feedback is
constituted by the new observation, the reward and the done boolean.
'''
# Create the new memory and update the buffer
new_memory = self.Memory(obs=obs,
action=action,
new_obs=new_obs,
reward=reward,
done=done)
# Append it to the replay buffer
self.replay_buffer.append(new_memory)
# update the variables
self.n_iter += 1
# TODO check this...
# decrease epsilon
self.epsilon = max(
self.epsilon_final,
self.epsilon_start - self.n_iter / self.epsilon_decay)
self.total_reward += reward
def print_info(self):
'''
Print information about the agent
'''
fps = (self.n_iter - self.ts_frame) / (time.time() - self.ts)
# TODO replace with proper logger
print('%d %d rew:%d mean_rew:%.2f eps:%.2f, fps:%d, loss:%.4f' %
(self.n_iter, self.n_games, self.total_reward,
np.mean(self.rewards[-40:]), self.epsilon, fps,
np.mean(self.accumulated_loss)))
self.ts_frame = self.n_iter
self.ts = time.time()
if self.summary_writer is not None:
self.summary_writer.add_scalar('reward', self.total_reward,
self.n_games)
self.summary_writer.add_scalar('mean_reward',
np.mean(self.rewards[-40:]),
self.n_games)
self.summary_writer.add_scalar('10_mean_reward',
np.mean(self.rewards[-10:]),
self.n_games)
self.summary_writer.add_scalar('epsilon', self.epsilon,
self.n_games)
self.summary_writer.add_scalar('loss',
np.mean(self.accumulated_loss),
self.n_games)
class Agent:
def __init__(
self,
state_size,
action_size,
n_agents,
buffer_size: int = 1e5,
batch_size: int = 256,
gamma: float = 0.995,
tau: float = 1e-3,
learning_rate: float = 7e-4,
update_every: int = 4,
):
"""
Initialize DQN agent using the agent-experience buffer
Args:
state_size (int): Size of the state observation returned by the
environment
action_size (int): Action space size
n_agents (int): Number of agents in the environment
buffer_size (int): Desired total experience buffer size
batch_size (int): Mini-batch size
gamma (float): Discount factor
tau (float): For soft update of target parameters
learning_rate (float): Learning rate
update_every (int): Number of steps before target network update
"""
self.state_size = state_size
self.action_size = action_size
self.n_agents = n_agents
# Q-Networks
self.policy_net = DQN(state_size, action_size).to(device)
self.target_net = DQN(state_size, action_size).to(device)
self.optimizer = optim.Adam(self.policy_net.parameters(),
lr=learning_rate)
self.memory = AgentReplayMemory(buffer_size, n_agents, state_size,
device)
self.t_step = 0
self.update_every = update_every
self.batch_size = batch_size
self.gamma = gamma
self.tau = tau
def step(self, states, actions, rewards, next_steps, done):
self.memory.push_agent_actions(states, actions, rewards, next_steps,
done)
self.t_step = (self.t_step + 1) % self.update_every
if self.t_step == 0:
if self.memory.at_capacity():
experience = self.memory.sample(self.batch_size)
self.learn(experience, self.gamma)
def act(self, states, eps=0):
states = torch.from_numpy(states).float().to(device)
self.policy_net.eval()
with torch.no_grad():
action_values = self.policy_net(states)
self.policy_net.train()
r = np.random.random(size=self.n_agents)
action_values = np.argmax(action_values.cpu().data.numpy(), axis=1)
random_choices = np.random.randint(0,
self.action_size,
size=self.n_agents)
return np.where(r > eps, action_values, random_choices)
def learn(self, experiences, gamma):
states, actions, rewards, next_states, dones = experiences
criterion = torch.nn.MSELoss()
self.policy_net.train()
self.target_net.eval()
# shape of output from the model (batch_size,action_dim) = (64,4)
predicted_targets = self.policy_net(states).gather(1, actions)
with torch.no_grad():
labels_next = self.target_net(next_states).detach().max(
1)[0].unsqueeze(1)
# .detach() -> Returns a new Tensor, detached from the current graph.
labels = rewards + (gamma * labels_next * (1 - dones))
loss = criterion(predicted_targets, labels).to(device)
self.optimizer.zero_grad()
loss.backward()
self.optimizer.step()
# ------------------- update target network ------------------- #
self.soft_update(self.policy_net, self.target_net, self.tau)
def soft_update(self, local_model, target_model, tau):
"""
Soft update model parameters.
θ_target = τ*θ_local + (1 - τ)*θ_target
Args:
local_model (PyTorch model): weights will be copied from
target_model (PyTorch model): weights will be copied to
tau (float): interpolation parameter
"""
for target_param, local_param in zip(target_model.parameters(),
local_model.parameters()):
target_param.data.copy_(tau * local_param.data +
(1 - tau) * target_param.data)
n_steps = 8
n_actions = env.action_space.n
img_height = 64
img_width = 64
policy_net = None
network_path = "target_net.pt"
if os.path.exists(network_path):
policy_net = torch.load(network_path)
print("successfully loaded existing network from file: " + network_path)
else:
policy_net = DQN(img_height, img_width, n_actions)
target_net = DQN(img_height, img_width, n_actions).to(device)
target_net.load_state_dict(policy_net.state_dict())
target_net.eval()
optimizer = optim.RMSprop(policy_net.parameters(), lr=LR)
memory = ReplayMemory(10000)
steps_done = 0
logfile = "train_log.txt"
with open(logfile, "w+") as f:
f.write("CS4803 MineRL Project Logs:\n")
def append_log(s):
with open(logfile, "a") as f:
f.write(s + "\n")
def state_from_obs(obs):
# get the camera image from the observation dict and convert the image
# to the correct shape: (C, H, W)
img = torch.tensor(obs["pov"] / 255.0, dtype=torch.float32)
flat_img = torch.transpose(torch.transpose(img, 0, 1), 0, 2).reshape(64*64*3)
def pick_action(observation, net):
if(random.random() < epsilon):
return random.randint(0, num_actions-1)
action = torch.argmax(
net(torch.tensor(observation).float().unsqueeze(0)))
return action
net = DQN()
net.load_state_dict(torch.load("model.h5", map_location="cpu"))
criterion = nn.MSELoss()
optimizer = optim.Adam(net.parameters(), lr=0.01)
starttime = time.time()
buffer = collections.deque(maxlen=N)
lr = 1e-3
for i in range(num_episodes):
observation = env.reset()
observation = preprocess(observation)
observation = [observation, observation, observation, observation]
j = 0
while(True):
j += 1
time.sleep(0.01 - ((time.time() - starttime) % 0.01))
if j % 4:
class AgentCartpole:
def __init__(self, p):
self.p = p
self.target_dqn = DQN(self.p['HIDDEN_DIM'])
self.eval_dqn = DQN(self.p['HIDDEN_DIM'])
self.memory = ReplayMemory(self.p['MEMORY_SIZE'], [4])
self.optimizer = torch.optim.Adam(self.eval_dqn.parameters(), self.p['LEARNING_RATE'])
try:
self.eval_dqn.load_state_dict(torch.load("Model/eval_dqn.data"))
self.target_dqn.load_state_dict(torch.load("Model/eval_dqn.data"))
print("Data has been loaded successfully")
except:
print("No data existing")
def act(self, state):
r = random.random()
if r > self.p['EPSILON']:
x = torch.FloatTensor(state)
q_value = self.eval_dqn(x)
action = torch.argmax(q_value).item()
return action
else:
action = random.randint(0, self.p['N_ACTIONS']-1)
return action
def learn(self):
if self.memory.index < self.p['BATCH_SIZE']:
return
# Get the state dict from the saved date
eval_dict = self.eval_dqn.state_dict()
target_dict = self.eval_dqn.state_dict()
# Updating the parameters of the target DQN
for w in eval_dict:
target_dict[w] = (1 - self.p['ALPHA']) * target_dict[w] + self.p['ALPHA'] * eval_dict[w]
self.target_dqn.load_state_dict(target_dict)
# Get a sample of size BATCH
batch_state, batch_action, batch_next_state, batch_reward, batch_done = self.memory.pop(self.p['BATCH_SIZE'])
# Update the treshold for the act() method if needed everytime the agent learn
if self.p["EPSILON"] > self.p["EPSILON_MIN"]:
self.p["EPSILON"] *= self.p["EPSILON_DECAY"]
loss = nn.MSELoss()
# Compute q values for the current evaluation
q_eval = self.eval_dqn(batch_state).gather(1, batch_action.long().unsqueeze(1)).reshape([self.p["BATCH_SIZE"]])
# Compute the next state q values
q_next = self.target_dqn(batch_next_state).detach()
# Compute the targetted q values
q_target = batch_reward + q_next.max(1)[0].reshape([self.p["BATCH_SIZE"]]) * self.p["GAMMA"]
self.optimizer.zero_grad()
l = loss(q_eval, q_target)
l.backward()
self.optimizer.step()
def random(self):
env = gym.make('CartPole-v1')
env = env.unwrapped
env.reset()
rewards = []
while True:
env.render()
action = env.action_space.pop(self.p['BATCH_SIZE'])
observation, reward, done, info = env.step(action)
rewards.append(reward)
if done:
break
env.close()
plt.ylabel("Rewards")
plt.xlabel("Nb interactions")
plt.plot(rewards)
plt.grid()
plt.show()
def dqn_cartpole(self):
env = gym.make('CartPole-v1')
env = env.unwrapped
rewards = []
for i in range(self.p['N_EPISODE']):
state = env.reset()
rewards.append(0)
for s in range(self.p['N_STEPS']):
# env.render()
action = self.act(state)
n_state, reward, done, _ = env.step(action)
if done:
reward = -1
rewards[-1] += reward
self.memory.push(state, action, n_state, reward, done)
self.learn()
state = n_state
print('Episode : ', i, ', Rewards : ', rewards[-1])
# Save the eval model after each episode
torch.save(self.eval_dqn.state_dict(), "Model/eval_dqn.data")
# Display result
n = 50
res = sum(([a]*n for a in [sum(rewards[i:i+n])//n for i in range(0,len(rewards),n)]), [])
print(rewards)
plt.ylabel("Rewards")
plt.xlabel("Episode")
plt.plot(rewards)
plt.plot(res)
plt.grid()
plt.legend(['Rewards per episode', 'Last 50 runs average'])
plt.show()
env.close()
class TrainNQL:
def __init__(self, epi, cfg=dcfg, validation=False):
#cpu or cuda
torch.cuda.empty_cache()
self.device = cfg.device #torch.device("cuda" if torch.cuda.is_available() else "cpu")
self.state_dim = cfg.proc_frame_size #State dimensionality 84x84.
self.state_size = cfg.state_size
#self.t_steps= tsteps
self.t_eps = cfg.t_eps
self.minibatch_size = cfg.minibatch_size
# Q-learning parameters
self.discount = cfg.discount #Discount factor.
self.replay_memory = cfg.replay_memory
self.bufferSize = cfg.bufferSize
self.target_q = cfg.target_q
self.validation = validation
if (validation):
self.episode = epi
else:
self.episode = int(epi) - 1
self.cfg = cfg
modelGray = 'results/ep' + str(self.episode) + '/modelGray.net'
modelDepth = 'results/ep' + str(self.episode) + '/modelDepth.net'
tModelGray = 'results/ep' + str(self.episode) + '/tModelGray.net'
tModelDepth = 'results/ep' + str(self.episode) + '/tModelDepth.net'
if os.path.exists(modelGray) and os.path.exists(modelDepth):
print("Loading model")
self.gray_policy_net = torch.load(modelGray).to(self.device)
self.gray_target_net = torch.load(tModelGray).to(self.device)
self.depth_policy_net = torch.load(modelDepth).to(self.device)
self.depth_target_net = torch.load(tModelDepth).to(self.device)
else:
print("New model")
self.gray_policy_net = DQN(noutputs=cfg.noutputs,
nfeats=cfg.nfeats,
nstates=cfg.nstates,
kernels=cfg.kernels,
strides=cfg.strides,
poolsize=cfg.poolsize).to(self.device)
self.gray_target_net = DQN(noutputs=cfg.noutputs,
nfeats=cfg.nfeats,
nstates=cfg.nstates,
kernels=cfg.kernels,
strides=cfg.strides,
poolsize=cfg.poolsize).to(self.device)
self.depth_policy_net = DQN(noutputs=cfg.noutputs,
nfeats=cfg.nfeats,
nstates=cfg.nstates,
kernels=cfg.kernels,
strides=cfg.strides,
poolsize=cfg.poolsize).to(self.device)
self.depth_target_net = DQN(noutputs=cfg.noutputs,
nfeats=cfg.nfeats,
nstates=cfg.nstates,
kernels=cfg.kernels,
strides=cfg.strides,
poolsize=cfg.poolsize).to(self.device)
if not validation and self.target_q and self.episode % self.target_q == 0:
print("cloning")
self.depth_policy_net = DQN(noutputs=cfg.noutputs,
nfeats=cfg.nfeats,
nstates=cfg.nstates,
kernels=cfg.kernels,
strides=cfg.strides,
poolsize=cfg.poolsize).to(self.device)
self.depth_target_net = DQN(noutputs=cfg.noutputs,
nfeats=cfg.nfeats,
nstates=cfg.nstates,
kernels=cfg.kernels,
strides=cfg.strides,
poolsize=cfg.poolsize).to(self.device)
self.gray_target_net.load_state_dict(self.gray_target_net.state_dict())
self.gray_target_net.eval()
self.depth_target_net.load_state_dict(
self.depth_target_net.state_dict())
self.depth_target_net.eval()
self.gray_optimizer = optim.RMSprop(self.gray_policy_net.parameters())
self.depth_optimizer = optim.RMSprop(
self.depth_policy_net.parameters())
self.memory = ReplayMemory(self.replay_memory)
def get_tensor_from_image(self, file):
convert = T.Compose([
T.ToPILImage(),
T.Resize((self.state_dim, self.state_dim),
interpolation=Image.BILINEAR),
T.ToTensor()
])
screen = Image.open(file)
screen = np.ascontiguousarray(screen, dtype=np.float32) / 255
screen = torch.from_numpy(screen)
screen = convert(screen).unsqueeze(0).to(self.device)
return screen
def get_data(self, episode, tsteps):
#images=torch.Tensor(tsteps,self.state_size,self.state_dim,self.state_dim).to(self.device)
#depths=torch.Tensor(tsteps,self.state_size,self.state_dim,self.state_dim).to(self.device)
images = []
depths = []
dirname_rgb = 'dataset/RGB/ep' + str(episode)
dirname_dep = 'dataset/Depth/ep' + str(episode)
for step in range(tsteps):
#proc_image=torch.Tensor(self.state_size,self.state_dim,self.state_dim).to(self.device)
#proc_depth=torch.Tensor(self.state_size,self.state_dim,self.state_dim).to(self.device)
proc_image = []
proc_depth = []
dirname_rgb = 'dataset/RGB/ep' + str(episode)
dirname_dep = 'dataset/Depth/ep' + str(episode)
for i in range(self.state_size):
grayfile = dirname_rgb + '/image_' + str(step + 1) + '_' + str(
i + 1) + '.png'
depthfile = dirname_dep + '/depth_' + str(
step + 1) + '_' + str(i + 1) + '.png'
#proc_image[i] = self.get_tensor_from_image(grayfile)
#proc_depth[i] = self.get_tensor_from_image(depthfile)
proc_image.append(grayfile)
proc_depth.append(depthfile)
#images[step]=proc_image
#depths[step]=proc_depth
images.append(proc_image)
depths.append(proc_depth)
return images, depths
def load_data(self):
rewards = torch.load('files/reward_history.dat')
actions = torch.load('files/action_history.dat')
ep_rewards = torch.load('files/ep_rewards.dat')
print("Loading images")
best_scores = range(len(actions))
buffer_selection_mode = 'default'
if (buffer_selection_mode == 'success_handshake'):
eps_values = []
for i in range(len(actions)):
hspos = 0
hsneg = 0
for step in range(len(actions[i])):
if (len(actions[i]) > 0):
if actions[i][step] == 3:
if rewards[i][step] > 0:
hspos = hspos + 1
elif rewards[i][step] == -0.1:
hsneg = hsneg + 1
accuracy = float(((hspos) / (hspos + hsneg)))
eps_values.append(accuracy)
best_scores = np.argsort(eps_values)
for i in best_scores:
print('Ep: ', i + 1)
dirname_gray = 'dataset/RGB/ep' + str(i + 1)
dirname_dep = 'dataset/Depth/ep' + str(i + 1)
files = []
if (os.path.exists(dirname_gray)):
files = os.listdir(dirname_gray)
k = 0
for file in files:
if re.match(r"image.*\.png", file):
k = k + 1
k = int(k / 8)
while (k % 4 != 0):
k = k - 1
if (k > self.bufferSize):
k = self.bufferSize
print(k)
#os.system("free -h")
#with torch.no_grad():
images, depths = self.get_data(i + 1, k)
print("Loading done")
for step in range(k - 1):
#print(len(rewards),i)
#print(len(rewards[i]), step)
reward = self.cfg.neutral_reward
if rewards[i][step] >= 1:
reward = self.cfg.hs_success_reward
elif rewards[i][step] < 0:
reward = self.cfg.hs_fail_reward
reward = torch.tensor([reward], device=self.device)
action = torch.tensor([[actions[i][step]]],
device=self.device,
dtype=torch.long)
#image = images[step].unsqueeze(0).to(self.device)
#depth = depths[step].unsqueeze(0).to(self.device)
#next_image = images[step+1].unsqueeze(0).to(self.device)
#next_depth = depths[step+1].unsqueeze(0).to(self.device)
image = images[step]
depth = depths[step]
next_image = images[step + 1]
next_depth = depths[step + 1]
self.memory.push(image, depth, action, next_image, next_depth,
reward)
#print("Memory size: ",getsizeof(self.memory))
#torch.cuda.empty_cache()
def train(self):
if len(self.memory) < self.minibatch_size:
return
for i in range(0, len(self.memory), self.minibatch_size):
#transitions = self.memory.sample(self.minibatch_size)
transitions = self.memory.pull(self.minibatch_size)
print('Batch train: ' + str(int(i / self.minibatch_size) + 1) +
"/" + str(int(len(self.memory) / self.minibatch_size) + 1))
aux_transitions = []
for t in transitions:
proc_sgray = torch.Tensor(self.state_size, self.state_dim,
self.state_dim).to(self.device)
proc_sdepth = torch.Tensor(self.state_size, self.state_dim,
self.state_dim).to(self.device)
proc_next_sgray = torch.Tensor(self.state_size, self.state_dim,
self.state_dim).to(self.device)
proc_next_sdepth = torch.Tensor(self.state_size,
self.state_dim,
self.state_dim).to(self.device)
count = 0
for sgray, sdepth, next_sgray, next_sdepth in zip(
t.sgray, t.sdepth, t.next_sgray, t.next_sdepth):
proc_sgray[count] = self.get_tensor_from_image(sgray)
proc_sdepth[count] = self.get_tensor_from_image(sdepth)
proc_next_sgray[count] = self.get_tensor_from_image(
next_sgray)
proc_next_sdepth[count] = self.get_tensor_from_image(
next_sdepth)
count += 1
proc_sgray = proc_sgray.unsqueeze(0).to(self.device)
proc_sdepth = proc_sdepth.unsqueeze(0).to(self.device)
proc_next_sgray = proc_next_sgray.unsqueeze(0).to(self.device)
proc_next_sdepth = proc_next_sdepth.unsqueeze(0).to(
self.device)
#('sgray','sdepth','action','next_sgray','next_sdepth','reward')
one_transition = Transition(proc_sgray, proc_sdepth, t.action,
proc_next_sgray, proc_next_sdepth,
t.reward)
aux_transitions.append(one_transition)
transitions = aux_transitions
# Transpose the batch (see https://stackoverflow.com/a/19343/3343043 for
# detailed explanation). This converts batch-array of Transitions
# to Transition of batch-arrays.
batch = Transition(*zip(*transitions))
#print(batch.sgray)
# Compute a mask of non-final states and concatenate the batch elements
# (a final state would've been the one after which simulation ended)
gray_non_final_mask = torch.tensor(tuple(
map(lambda s: s is not None, batch.next_sgray)),
device=self.device,
dtype=torch.bool)
gray_non_final_next_states = torch.cat(
[s for s in batch.next_sgray if s is not None])
depth_non_final_mask = torch.tensor(tuple(
map(lambda s: s is not None, batch.next_sdepth)),
device=self.device,
dtype=torch.bool)
depth_non_final_next_states = torch.cat(
[s for s in batch.next_sdepth if s is not None])
sgray_batch = torch.cat(batch.sgray)
sdepth_batch = torch.cat(batch.sdepth)
action_batch = torch.cat(batch.action)
reward_batch = torch.cat(batch.reward)
# Compute Q(s_t, a) - the model computes Q(s_t), then we select the
# columns of actions taken. These are the actions which would've been taken
# for each batch state according to policy_net
sgray_action_values = self.gray_policy_net(sgray_batch).gather(
1, action_batch)
sdepth_action_values = self.depth_policy_net(sdepth_batch).gather(
1, action_batch)
# Compute V(s_{t+1}) for all next states.
# Expected values of actions for non_final_next_states are computed based
# on the "older" target_net; selecting their best reward with max(1)[0].
# This is merged based on the mask, such that we'll have either the expected
# state value or 0 in case the state was final.
next_sgray_values = torch.zeros(self.minibatch_size,
device=self.device)
next_sgray_values[gray_non_final_mask] = self.gray_target_net(
gray_non_final_next_states).max(1)[0].detach()
next_sdepth_values = torch.zeros(self.minibatch_size,
device=self.device)
next_sdepth_values[depth_non_final_mask] = self.depth_target_net(
depth_non_final_next_states).max(1)[0].detach()
# Compute the expected Q values
expected_sgray_action_values = (next_sgray_values *
self.discount) + reward_batch
expected_sdepth_action_values = (next_sdepth_values *
self.discount) + reward_batch
# Compute Huber loss
gray_loss = F.smooth_l1_loss(
sgray_action_values, expected_sgray_action_values.unsqueeze(1))
depth_loss = F.smooth_l1_loss(
sdepth_action_values,
expected_sdepth_action_values.unsqueeze(1))
# Optimize the model
self.gray_optimizer.zero_grad()
gray_loss.backward()
for param in self.gray_policy_net.parameters():
param.grad.data.clamp_(-1, 1)
self.gray_optimizer.step()
# Optimize the model
self.depth_optimizer.zero_grad()
depth_loss.backward()
for param in self.depth_policy_net.parameters():
param.grad.data.clamp_(-1, 1)
self.depth_optimizer.step()
def test_dqn(args=get_args()):
env = make_atari_env(args)
args.state_shape = env.observation_space.shape or env.observation_space.n
args.action_shape = env.env.action_space.shape or env.env.action_space.n
# should be N_FRAMES x H x W
print("Observations shape:", args.state_shape)
print("Actions shape:", args.action_shape)
# make environments
train_envs = SubprocVectorEnv(
[lambda: make_atari_env(args) for _ in range(args.training_num)])
test_envs = SubprocVectorEnv(
[lambda: make_atari_env_watch(args) for _ in range(args.test_num)])
# seed
np.random.seed(args.seed)
torch.manual_seed(args.seed)
train_envs.seed(args.seed)
test_envs.seed(args.seed)
# define model
net = DQN(*args.state_shape, args.action_shape,
args.device).to(args.device)
optim = torch.optim.Adam(net.parameters(), lr=args.lr)
# define policy
policy = DQNPolicy(net,
optim,
args.gamma,
args.n_step,
target_update_freq=args.target_update_freq)
# load a previous policy
if args.resume_path:
policy.load_state_dict(
torch.load(args.resume_path, map_location=args.device))
print("Loaded agent from: ", args.resume_path)
# replay buffer: `save_last_obs` and `stack_num` can be removed together
# when you have enough RAM
buffer = VectorReplayBuffer(args.buffer_size,
buffer_num=len(train_envs),
ignore_obs_next=True,
save_only_last_obs=True,
stack_num=args.frames_stack)
# collector
train_collector = Collector(policy,
train_envs,
buffer,
exploration_noise=True)
test_collector = Collector(policy, test_envs, exploration_noise=True)
# log
log_path = os.path.join(args.logdir, args.task, 'dqn')
writer = SummaryWriter(log_path)
writer.add_text("args", str(args))
logger = BasicLogger(writer)
def save_fn(policy):
torch.save(policy.state_dict(), os.path.join(log_path, 'policy.pth'))
def stop_fn(mean_rewards):
if env.env.spec.reward_threshold:
return mean_rewards >= env.spec.reward_threshold
elif 'Pong' in args.task:
return mean_rewards >= 20
else:
return False
def train_fn(epoch, env_step):
# nature DQN setting, linear decay in the first 1M steps
if env_step <= 1e6:
eps = args.eps_train - env_step / 1e6 * \
(args.eps_train - args.eps_train_final)
else:
eps = args.eps_train_final
policy.set_eps(eps)
logger.write('train/eps', env_step, eps)
def test_fn(epoch, env_step):
policy.set_eps(args.eps_test)
# watch agent's performance
def watch():
print("Setup test envs ...")
policy.eval()
policy.set_eps(args.eps_test)
test_envs.seed(args.seed)
if args.save_buffer_name:
print(f"Generate buffer with size {args.buffer_size}")
buffer = VectorReplayBuffer(args.buffer_size,
buffer_num=len(test_envs),
ignore_obs_next=True,
save_only_last_obs=True,
stack_num=args.frames_stack)
collector = Collector(policy, test_envs, buffer)
result = collector.collect(n_step=args.buffer_size)
print(f"Save buffer into {args.save_buffer_name}")
# Unfortunately, pickle will cause oom with 1M buffer size
buffer.save_hdf5(args.save_buffer_name)
else:
print("Testing agent ...")
test_collector.reset()
result = test_collector.collect(n_episode=args.test_num,
render=args.render)
pprint.pprint(result)
if args.watch:
watch()
exit(0)
# test train_collector and start filling replay buffer
train_collector.collect(n_step=args.batch_size * args.training_num)
# trainer
result = offpolicy_trainer(policy,
train_collector,
test_collector,
args.epoch,
args.step_per_epoch,
args.step_per_collect,
args.test_num,
args.batch_size,
train_fn=train_fn,
test_fn=test_fn,
stop_fn=stop_fn,
save_fn=save_fn,
logger=logger,
update_per_step=args.update_per_step,
test_in_train=False)
pprint.pprint(result)
watch()
from network import Network, DQN
from memory import ReplayMemory
#%% hyper parameters
EPS_START = 0.9 # e-greedy threshold start value
EPS_END = 0.05 # e-greedy threshold end value
EPS_DECAY = 200 # e-greedy threshold decay
GAMMA = 0.8 # Q-learning discount factor
LR = 0.001 # NN optimizer learning rate
HIDDEN_LAYER = 256 # NN hidden layer size
BATCH_SIZE = 64 # Q-learning batch size
#%% DQN NETWORK ARCHITECTURE
model = DQN(4, 4, 4)
model.cuda()
optimizer = optim.Adam(model.parameters(), LR)
#%% SELECT ACTION USING GREEDY ALGORITHM
steps_done = 0
def select_action(state):
global steps_done
sample = random.random()
eps_threshold = EPS_END + (EPS_START - EPS_END) * math.exp(
-1. * steps_done / EPS_DECAY)
steps_done += 1
#print(state.shape)
#print(eps_threshold)
if sample > eps_threshold:
from basic_game import Game, Direction
from dqn import DQNTrainer, next_epsilon
from network import DQN
from simple_play import display_game, print_screen
from tensor_helper import game2tensor
SLEEP = 0.1
SAVE_TIME = 1000
WIDTH = 20
HEIGHT = 10
DECAY_STEP = 2000
BATCH_SIZE = 1
SIDE = math.sqrt(WIDTH * HEIGHT)
MAX_DIST = math.ceil(math.sqrt(WIDTH**2 + HEIGHT**2))
policy_network = DQN(width=WIDTH, height=HEIGHT)
optimizer = optim.SGD(policy_network.parameters(), lr=1e-5)
POLICY_PATH = 'data/policy.pt'
if os.path.isfile(POLICY_PATH):
policy_network = torch.load(POLICY_PATH)
class SnakeTrainer(DQNTrainer):
def __init__(self, *args, **kwargs):
super(SnakeTrainer, self).__init__(*args, **kwargs)
self.snake_step_info = dict()
def decide_epsilon(self, game: Game):
snake_len = len(game.snake)
decay = self.snake_step_info.get(snake_len, 1)
epsilon, self.snake_step_info[snake_len] = next_epsilon(
self.eps_start, self.eps_end, self.decay_step, decay)
