def __init__(self, memory, nb_status, nb_actions, action_noise=None,
gamma=0.99, tau=0.001, normalize_observations=True,
batch_size=128, observation_range=(-5., 5.), action_range=(-1., 1.),
actor_lr=1e-4, critic_lr=1e-3):
self.nb_status = nb_status
self.nb_actions = nb_actions
self.action_range = action_range
self.observation_range = observation_range
self.normalize_observations = normalize_observations
self.actor = Actor(self.nb_status, self.nb_actions)
self.actor_target = Actor(self.nb_status, self.nb_actions)
self.actor_optim = Adam(self.actor.parameters(), lr=actor_lr)
self.critic = Critic(self.nb_status, self.nb_actions)
self.critic_target = Critic(self.nb_status, self.nb_actions)
self.critic_optim = Adam(self.critic.parameters(), lr=critic_lr)
# Create replay buffer
self.memory = memory # SequentialMemory(limit=args.rmsize, window_length=args.window_length)
self.action_noise = action_noise
# Hyper-parameters
self.batch_size = batch_size
self.tau = tau
self.discount = gamma
if self.normalize_observations:
self.obs_rms = RunningMeanStd()
else:
self.obs_rms = None
def __init__(self, nb_status, nb_actions, args, writer):
self.clip_actor_grad = args.clip_actor_grad
self.nb_status = nb_status * args.window_length
self.nb_actions = nb_actions
self.discrete = args.discrete
self.pic = args.pic
self.writer = writer
self.select_time = 0
if self.pic:
self.nb_status = args.pic_status
# Create Actor and Critic Network
net_cfg = {
'hidden1':args.hidden1,
'hidden2':args.hidden2,
'use_bn':args.bn,
'init_method':args.init_method
}
if args.pic:
self.cnn = CNN(1, args.pic_status)
self.cnn_target = CNN(1, args.pic_status)
self.cnn_optim = Adam(self.cnn.parameters(), lr=args.crate)
self.actor = Actor(self.nb_status, self.nb_actions, **net_cfg)
self.actor_target = Actor(self.nb_status, self.nb_actions, **net_cfg)
self.actor_optim = Adam(self.actor.parameters(), lr=args.prate)
self.critic = Critic(self.nb_status, self.nb_actions, **net_cfg)
self.critic_target = Critic(self.nb_status, self.nb_actions, **net_cfg)
self.critic_optim = Adam(self.critic.parameters(), lr=args.rate)
hard_update(self.actor_target, self.actor) # Make sure target is with the same weight
hard_update(self.critic_target, self.critic)
if args.pic:
hard_update(self.cnn_target, self.cnn)
#Create replay buffer
self.memory = rpm(args.rmsize) # SequentialMemory(limit=args.rmsize, window_length=args.window_length)
self.random_process = Myrandom(size=nb_actions)
# Hyper-parameters
self.batch_size = args.batch_size
self.tau = args.tau
self.discount = args.discount
self.depsilon = 1.0 / args.epsilon
#
self.epsilon = 1.0
self.s_t = None # Most recent state
self.a_t = None # Most recent action
self.use_cuda = args.cuda
#
if self.use_cuda: self.cuda()
def __init__(self, nb_status, nb_actions, args):
self.num_actor = 3
self.nb_status = nb_status * args.window_length
self.nb_actions = nb_actions
self.discrete = args.discrete
self.pic = args.pic
if self.pic:
self.nb_status = args.pic_status
# Create Actor and Critic Network
net_cfg = {
'hidden1':args.hidden1,
'hidden2':args.hidden2,
'use_bn':args.bn
}
if args.pic:
self.cnn = CNN(3, args.pic_status)
self.cnn_optim = Adam(self.cnn.parameters(), lr=args.crate)
self.actors = [Actor(self.nb_status, self.nb_actions) for _ in range(self.num_actor)]
self.actor_targets = [Actor(self.nb_status, self.nb_actions) for _ in
range(self.num_actor)]
self.actor_optims = [Adam(self.actors[i].parameters(), lr=args.prate) for i in range(self.num_actor)]
self.critic = Critic(self.nb_status, self.nb_actions, **net_cfg)
self.critic_target = Critic(self.nb_status, self.nb_actions, **net_cfg)
self.critic_optim = Adam(self.critic.parameters(), lr=args.rate)
for i in range(self.num_actor):
hard_update(self.actor_targets[i], self.actors[i]) # Make sure target is with the same weight
hard_update(self.critic_target, self.critic)
#Create replay buffer
self.memory = rpm(args.rmsize) # SequentialMemory(limit=args.rmsize, window_length=args.window_length)
self.random_process = Myrandom(size=nb_actions)
# Hyper-parameters
self.batch_size = args.batch_size
self.tau = args.tau
self.discount = args.discount
self.depsilon = 1.0 / args.epsilon
#
self.epsilon = 1.0
self.s_t = None # Most recent state
self.a_t = None # Most recent action
self.use_cuda = args.cuda
#
if self.use_cuda: self.cuda()
def __init__(self, state_size, action_size, random_seed):
"""Initialize an Agent object.
Params
======
state_size (int): dimension of each state
action_size (int): dimension of each action
random_seed (int): random seed
"""
self.state_size = state_size
self.action_size = action_size
self.seed = random.seed(random_seed)
# Actor Network (w/ Target Network)
self.actor_local = Actor(state_size, action_size, random_seed).to(device)
self.actor_target = Actor(state_size, action_size, random_seed).to(device)
self.actor_optimizer = optim.Adam(self.actor_local.parameters(), lr=LR_ACTOR)
# Critic Network (w/ Target Network)
self.critic_local = Critic(state_size, action_size, random_seed).to(device)
self.critic_target = Critic(state_size, action_size, random_seed).to(device)
self.critic_optimizer = optim.Adam(self.critic_local.parameters(), lr=LR_CRITIC, weight_decay=WEIGHT_DECAY)
# Noise process
self.noise = OUNoise(action_size, random_seed)
# Replay memory
self.memory = ReplayBuffer(action_size, BUFFER_SIZE, BATCH_SIZE, random_seed)
def __init__(self, task):
self.task = task
self.state_size = task.state_size
self.action_size = task.action_size
self.action_low = task.action_low
self.action_high = task.action_high
# Actor (Policy) Model
self.actor_local = Actor(self.state_size, self.action_size, self.action_low, self.action_high)
self.actor_target = Actor(self.state_size, self.action_size, self.action_low, self.action_high)
# Critic (Value) Model
self.critic_local = Critic(self.state_size, self.action_size)
self.critic_target = Critic(self.state_size, self.action_size)
# Initialize target model parameters with local model parameters
self.critic_target.model.set_weights(self.critic_local.model.get_weights())
self.actor_target.model.set_weights(self.actor_local.model.get_weights())
# Noise process
self.exploration_mu = 0 #0
self.exploration_theta = 0.15
self.exploration_sigma = 0.2
self.noise = OUNoise(self.action_size, self.exploration_mu, self.exploration_theta, self.exploration_sigma)
# Replay memory
self.buffer_size = 100000
self.batch_size = 64
self.memory = ReplayBuffer(self.buffer_size, self.batch_size)
# Algorithm parameters
self.gamma = 0.99 # discount factor - 0.99
self.tau = 0.01 # for soft update of target parameters - 0.01
# Score tracker and learning parameters
self.best_w = None
self.best_score = -np.inf
self.score = -np.inf
class DDPG(object):
def __init__(self, nb_status, nb_actions, args, writer):
self.clip_actor_grad = args.clip_actor_grad
self.nb_status = nb_status * args.window_length
self.nb_actions = nb_actions
self.writer = writer
self.select_time = 0
# Create Actor and Critic Network
net_cfg = {
'hidden1':args.hidden1,
'hidden2':args.hidden2,
'init_method':args.init_method
}
self.actor = Actor(self.nb_status, self.nb_actions, **net_cfg)
self.actor_target = Actor(self.nb_status, self.nb_actions, **net_cfg)
self.actor_optim = Adam(self.actor.parameters(), lr=args.prate)
self.critic = Critic(self.nb_status, self.nb_actions, **net_cfg)
self.critic_target = Critic(self.nb_status, self.nb_actions, **net_cfg)
self.critic_optim = Adam(self.critic.parameters(), lr=args.rate)
hard_update(self.actor_target, self.actor) # Make sure target is with the same weight
hard_update(self.critic_target, self.critic)
#Create replay buffer
self.memory = rpm(args.rmsize)
self.random_process = Myrandom(size=nb_actions)
# Hyper-parameters
self.batch_size = args.batch_size
self.tau = args.tau
self.discount = args.discount
self.depsilon = 1.0 / args.epsilon
#
self.epsilon = 1.0
self.s_t = None # Most recent state
self.a_t = None # Most recent action
self.use_cuda = args.cuda
#
if self.use_cuda: self.cuda()
def update_policy(self, train_actor = True):
# Sample batch
state_batch, action_batch, reward_batch, \
next_state_batch, terminal_batch = self.memory.sample_batch(self.batch_size)
# Prepare for the target q batch
next_q_values = self.critic_target([
to_tensor(next_state_batch, volatile=True),
self.actor_target(to_tensor(next_state_batch, volatile=True)),
])
# print('batch of picture is ok')
next_q_values.volatile = False
target_q_batch = to_tensor(reward_batch) + \
self.discount * to_tensor((1 - terminal_batch.astype(np.float))) * next_q_values
# Critic update
self.critic.zero_grad()
q_batch = self.critic([to_tensor(state_batch), to_tensor(action_batch)])
# print(reward_batch, next_q_values*self.discount, target_q_batch, terminal_batch.astype(np.float))
value_loss = nn.MSELoss()(q_batch, target_q_batch)
value_loss.backward()
self.critic_optim.step()
self.actor.zero_grad()
policy_loss = -self.critic([
to_tensor(state_batch),
self.actor(to_tensor(state_batch))
])
policy_loss = policy_loss.mean()
policy_loss.backward()
if self.clip_actor_grad is not None:
torch.nn.utils.clip_grad_norm(self.actor.parameters(), float(self.clip_actor_grad))
if self.writer != None:
mean_policy_grad = np.array(np.mean([np.linalg.norm(p.grad.data.cpu().numpy().ravel()) for p in self.actor.parameters()]))
#print(mean_policy_grad)
self.writer.add_scalar('train/mean_policy_grad', mean_policy_grad, self.select_time)
if train_actor:
self.actor_optim.step()
# Target update
soft_update(self.actor_target, self.actor, self.tau)
soft_update(self.critic_target, self.critic, self.tau)
return -policy_loss, value_loss
def eval(self):
self.actor.eval()
self.actor_target.eval()
self.critic.eval()
self.critic_target.eval()
def train(self):
self.actor.train()
self.actor_target.train()
self.critic.train()
self.critic_target.train()
def cuda(self):
self.actor.cuda()
self.actor_target.cuda()
self.critic.cuda()
self.critic_target.cuda()
def observe(self, r_t, s_t1, done):
self.memory.append([self.s_t, self.a_t, r_t, s_t1, done])
self.s_t = s_t1
def random_action(self):
action = np.random.uniform(-1.,1.,self.nb_actions)
self.a_t = action
return action
def select_action(self, s_t, decay_epsilon=True, return_fix=False, noise_level=0):
self.eval()
# print(s_t.shape)
action = to_numpy(
self.actor(to_tensor(np.array([s_t])))
).squeeze(0)
self.train()
noise_level = noise_level * max(self.epsilon, 0)
action = action * (1 - noise_level) + (self.random_process.sample() * noise_level)
action = np.clip(action, -1., 1.)
if decay_epsilon:
self.epsilon -= self.depsilon
self.a_t = action
return action
def reset(self, obs):
self.s_t = obs
self.random_process.reset_status()
def load_weights(self, output, num=1):
if output is None: return
self.actor.load_state_dict(
torch.load('{}/actor{}.pkl'.format(output, num))
)
self.actor_target.load_state_dict(
torch.load('{}/actor{}.pkl'.format(output, num))
)
self.critic.load_state_dict(
torch.load('{}/critic{}.pkl'.format(output, num))
)
self.critic_target.load_state_dict(
torch.load('{}/critic{}.pkl'.format(output, num))
)
def save_model(self, output, num):
if self.use_cuda:
self.actor.cpu()
self.critic.cpu()
torch.save(
self.actor.state_dict(),
'{}/actor{}.pkl'.format(output, num)
)
torch.save(
self.critic.state_dict(),
'{}/critic{}.pkl'.format(output, num)
)
if self.use_cuda:
self.actor.cuda()
self.critic.cuda()
class Agent():
"""Interacts with and learns from the environment."""
def __init__(self, state_size, action_size, num_agents, random_seed):
"""Initialize an Agent object.
Params
======
state_size (int): dimension of each state
action_size (int): dimension of each action
random_seed (int): random seed
"""
self.state_size = state_size
self.action_size = action_size
self.num_agents = num_agents
self.seed = random.seed(random_seed)
# Actor Network (w/ Target Network)
self.actor_local = Actor(state_size, action_size,
random_seed).to(device)
self.actor_target = Actor(state_size, action_size,
random_seed).to(device)
self.actor_optimizer = optim.Adam(self.actor_local.parameters(),
lr=LR_ACTOR)
# Critic Network (w/ Target Network)
self.critic_local = Critic(state_size, action_size,
random_seed).to(device)
self.critic_target = Critic(state_size, action_size,
random_seed).to(device)
self.critic_optimizer = optim.Adam(self.critic_local.parameters(),
lr=LR_CRITIC,
weight_decay=WEIGHT_DECAY)
# Noise process
self.noise = OUNoise(action_size, random_seed)
self.noise_eps = NOISE_EPS
# Replay memory
self.memory = ReplayBuffer(action_size, BUFFER_SIZE, BATCH_SIZE,
random_seed)
self.hard_update(self.critic_target, self.critic_local)
self.hard_update(self.actor_target, self.actor_local)
def step(self, state, action, reward, next_state, done, timesetep):
"""Save experience in replay memory, and use random sample from buffer to learn."""
# Save experience / reward
self.memory.add(state, action, reward, next_state, done)
# Learn, if enough samples are available in memory
#print(len(self.memory))
if len(self.memory) > BATCH_SIZE and timesetep % UPDATE_EVERY == 0:
for _ in range(UPDATE_ONLY):
experiences = self.memory.sample()
self.learn(experiences, GAMMA)
def act(self, state, add_noise=True):
"""Returns actions for given state as per current policy."""
state = torch.from_numpy(state).float().to(device)
self.actor_local.eval()
with torch.no_grad():
action = self.actor_local(state).cpu().data.numpy()
self.actor_local.train()
if add_noise:
action += self.noise_eps * np.random.normal(
0, 0.1
) # Noise from normal distribution.OU Noise seems to not explore
#np.random.randn(self.num_agents,self.action_size)# self.noise.sample()
return np.clip(action, -1, 1)
def reset(self):
self.noise.reset()
def learn(self, experiences, gamma):
"""Update policy and value parameters using given batch of experience tuples.
Q_targets = r + γ * critic_target(next_state, actor_target(next_state))
where:
actor_target(state) -> action
critic_target(state, action) -> Q-value
Params
======
experiences (Tuple[torch.Tensor]): tuple of (s, a, r, s', done) tuples
gamma (float): discount factor
"""
states, actions, rewards, next_states, dones = experiences
# ---------------------------- update critic ---------------------------- #
# Get predicted next-state actions and Q values from target models
actions_next = self.actor_target(next_states)
Q_targets_next = self.critic_target(next_states, actions_next)
# Compute Q targets for current states (y_i)
Q_targets = rewards + (gamma * Q_targets_next * (1 - dones))
# Compute critic loss
Q_expected = self.critic_local(states, actions)
critic_loss = F.mse_loss(Q_expected, Q_targets)
# Minimize the loss
self.critic_optimizer.zero_grad()
critic_loss.backward()
torch.nn.utils.clip_grad_norm_(self.critic_local.parameters(), 1.)
self.critic_optimizer.step()
# ---------------------------- update actor ---------------------------- #
# Compute actor loss
actions_pred = self.actor_local(states)
actor_loss = -self.critic_local(states, actions_pred).mean()
# Minimize the loss
self.actor_optimizer.zero_grad()
actor_loss.backward()
self.actor_optimizer.step()
# ----------------------- update target networks ----------------------- #
self.soft_update(self.critic_local, self.critic_target, TAU)
self.soft_update(self.actor_local, self.actor_target, TAU)
self.noise_eps -= NOISE_EPS_DECAY
def soft_update(self, local_model, target_model, tau):
"""Soft update model parameters.
θ_target = τ*θ_local + (1 - τ)*θ_target
Params
======
local_model: PyTorch model (weights will be copied from)
target_model: PyTorch model (weights will be copied to)
tau (float): interpolation parameter
"""
for target_param, local_param in zip(target_model.parameters(),
local_model.parameters()):
target_param.data.copy_(tau * local_param.data +
(1.0 - tau) * target_param.data)
def hard_update(self, target, source):
"""
Copy network parameters from source to target
Inputs:
target (torch.nn.Module): Net to copy parameters to
source (torch.nn.Module): Net whose parameters to copy
"""
for target_param, param in zip(target.parameters(),
source.parameters()):
target_param.data.copy_(param.data)
def main():
env = gym.make(args.env_name)
env.seed(args.seed)
torch.manual_seed(args.seed)
num_inputs = env.observation_space.shape[0]
num_actions = env.action_space.shape[0]
running_state = ZFilter((num_inputs,), clip=5)
print('state size:', num_inputs)
print('action size:', num_actions)
actor = Actor(num_inputs, num_actions, args)
critic = Critic(num_inputs, args)
actor_optim = optim.Adam(actor.parameters(), lr=args.learning_rate)
critic_optim = optim.Adam(critic.parameters(), lr=args.learning_rate,
weight_decay=args.l2_rate)
writer = SummaryWriter(comment="-ppo_iter-" + str(args.max_iter_num))
if args.load_model is not None:
saved_ckpt_path = os.path.join(os.getcwd(), 'save_model', str(args.load_model))
ckpt = torch.load(saved_ckpt_path)
actor.load_state_dict(ckpt['actor'])
critic.load_state_dict(ckpt['critic'])
running_state.rs.n = ckpt['z_filter_n']
running_state.rs.mean = ckpt['z_filter_m']
running_state.rs.sum_square = ckpt['z_filter_s']
print("Loaded OK ex. Zfilter N {}".format(running_state.rs.n))
episodes = 0
for iter in range(args.max_iter_num):
actor.eval(), critic.eval()
memory = deque()
steps = 0
scores = []
while steps < args.total_sample_size:
state = env.reset()
score = 0
state = running_state(state)
for _ in range(10000):
if args.render:
env.render()
steps += 1
mu, std = actor(torch.Tensor(state).unsqueeze(0))
action = get_action(mu, std)[0]
next_state, reward, done, _ = env.step(action)
if done:
mask = 0
else:
mask = 1
memory.append([state, action, reward, mask])
next_state = running_state(next_state)
state = next_state
score += reward
if done:
break
episodes += 1
scores.append(score)
score_avg = np.mean(scores)
print('{}:: {} episode score is {:.2f}'.format(iter, episodes, score_avg))
writer.add_scalar('log/score', float(score_avg), iter)
actor.train(), critic.train()
train_model(actor, critic, memory, actor_optim, critic_optim, args)
if iter % 100:
score_avg = int(score_avg)
model_path = os.path.join(os.getcwd(),'save_model')
if not os.path.isdir(model_path):
os.makedirs(model_path)
ckpt_path = os.path.join(model_path, 'ckpt_'+ str(score_avg)+'.pth.tar')
save_checkpoint({
'actor': actor.state_dict(),
'critic': critic.state_dict(),
'z_filter_n':running_state.rs.n,
'z_filter_m': running_state.rs.mean,
'z_filter_s': running_state.rs.sum_square,
'args': args,
'score': score_avg
}, filename=ckpt_path)
class DDPG_Agent():
"""
The agent uses experiences from a single or multiple agents to train the agents
using the Deep Deterministic Policy Gradient (DDPG) algorithm.
Code is taken from the 'ddpg-pendulum' example provided by Udacity
and modified to learn from the shared experiences of multiple agents.
"""
def __init__(self, state_size, action_size, num_agents, seed=31415):
"""
Initialize a DDPG_Agent object.
Arguments
state_size (int) : dimension of each state
action_size (int): dimension of each action
num_agents (int) : number of agents in the environment
seed (int) : seed for random generator
"""
self.state_size = state_size
self.action_size = action_size
self.num_agents = num_agents
random.seed(seed)
self.steps = 0 # to track number of steps
# Actor Network (w/ Target Network)
self.actor_local = Actor(state_size,
action_size,
hidden_size=HIDDEN_SIZE_ACTOR).to(device)
self.actor_target = Actor(state_size,
action_size,
hidden_size=HIDDEN_SIZE_ACTOR).to(device)
self.actor_optimizer = optim.Adam(self.actor_local.parameters(),
lr=LR_ACTOR)
# Critic Network (w/ Target Network)
self.critic_local = Critic(state_size,
action_size,
hidden_size=HIDDEN_SIZE_CRITIC).to(device)
self.critic_target = Critic(state_size,
action_size,
hidden_size=HIDDEN_SIZE_CRITIC).to(device)
self.critic_optimizer = optim.Adam(self.critic_local.parameters(),
lr=LR_CRITIC)
# Noise process
self.noise = OUNoise(action_size, seed)
# Replay memory
self.memory = ReplayBuffer(action_size, BUFFER_SIZE, BATCH_SIZE, seed)
def step(self, states, actions, rewards, next_states, dones):
"""Save experiences of all agents in replay memory, and
use batch of random samples from memory to perform training step."""
# Increment step count
self.steps += 1
# Save experience to the replay buffer
for i in range(self.num_agents):
self.memory.add(states[i], actions[i], rewards[i], next_states[i],
dones[i])
# Learn if enough samples are available in the replay buffer
if (self.steps % 20 == 0) and (len(self.memory) > BATCH_SIZE):
for _ in range(10):
experiences = self.memory.sample()
self.learn(experiences, GAMMA)
def act(self, states, add_noise=True):
"""Return an action taken by each agent using current policy
given the state of each agent's environment.
Returns numpy array with shape [num_agents, action_size]
Arguments:
states: numpy array of shape [num_agents, state_size]
add_noise: boolean, True if noise should be added to actions
"""
states = torch.from_numpy(states).float().to(device)
self.actor_local.eval()
with torch.no_grad():
actions = self.actor_local(states).cpu().data.numpy()
self.actor_local.train()
if add_noise:
actions += [self.noise.sample() for _ in range(self.num_agents)]
return np.clip(actions, -1, 1)
def reset(self):
self.noise.reset()
def learn(self, experiences, gamma):
"""Update policy and value parameters using given batch of experience tuples.
Q_targets = r + γ * critic_target(next_state, actor_target(next_state))
where:
actor_target(state) -> action
critic_target(state, action) -> Q-value
Arguments:
experiences (Tuple[torch.Tensor]): tuple of (s, a, r, s', done) tuples
gamma (float): discount factor
"""
states, actions, rewards, next_states, dones = experiences
# ---------------------------- update critic ---------------------------- #
# Get predicted next-state actions and Q values from target models
actions_next = self.actor_target(next_states)
Q_targets_next = self.critic_target(next_states, actions_next)
# Compute Q targets for current states (y_i)
Q_targets = rewards + (gamma * Q_targets_next * (1 - dones))
# Compute critic loss
Q_expected = self.critic_local(states, actions)
critic_loss = F.mse_loss(Q_expected, Q_targets)
# Minimize the loss
self.critic_optimizer.zero_grad()
critic_loss.backward()
self.critic_optimizer.step()
# ---------------------------- update actor ---------------------------- #
# Compute actor loss
actions_pred = self.actor_local(states)
actor_loss = -self.critic_local(states, actions_pred).mean()
# Minimize the loss
self.actor_optimizer.zero_grad()
actor_loss.backward()
self.actor_optimizer.step()
# ----------------------- update target networks ----------------------- #
self.soft_update(self.critic_local, self.critic_target, TAU)
self.soft_update(self.actor_local, self.actor_target, TAU)
def soft_update(self, local_model, target_model, tau):
"""Soft update model parameters.
θ_target = τ*θ_local + (1 - τ)*θ_target
Arguments:
local_model: PyTorch model (weights will be copied from)
target_model: PyTorch model (weights will be copied to)
tau (float): interpolation parameter
"""
for target_param, local_param in zip(target_model.parameters(),
local_model.parameters()):
target_param.data.copy_(tau * local_param.data +
(1.0 - tau) * target_param.data)
class DeepDeterministicPolicyGradient:
"""
Interacts with and learns from the environment.
Deep Deterministic Policy Gradient algorithm.
"""
def __init__(self, observation_dim: int, action_dim: int, num_agents: int,
idx: int, seed: int):
"""
Initialize an Agent object.
:param observation_dim: observation dimension per agent;
:param action_dim: action dimension per agent;
:param num_agents: number of agents;
:param idx: agent's index;
:param seed: random seed.
"""
random.seed(seed)
self.idx = idx
self.num_agents = num_agents
self.observation_dim = observation_dim
self.action_dim = action_dim
self.state_dim = num_agents * observation_dim
self.full_action_dim = num_agents * action_dim
self.eps = EPS_START
# Initialize networks and optimizers
self.actor_local = Actor(self.observation_dim,
self.action_dim,
seed=seed).to(DEVICE)
self.actor_target = Actor(self.observation_dim,
self.action_dim,
seed=seed).to(DEVICE)
self.hard_update(self.actor_local, self.actor_target)
self.actor_optim = Adam(self.actor_local.parameters(), lr=ACTOR_LR)
self.critic_local = Critic(self.state_dim,
self.full_action_dim,
seed=seed).to(DEVICE)
self.critic_target = Critic(self.state_dim,
self.full_action_dim,
seed=seed).to(DEVICE)
self.hard_update(self.critic_local, self.critic_target)
self.critic_optim = Adam(self.critic_local.parameters(),
lr=CRITIC_LR,
weight_decay=CRITIC_WD)
self.noise = OrnsteinUhlenbeckActionNoise(self.action_dim,
seed,
theta=0.15,
sigma=0.2)
def act(self, observation, explore=True):
"""
Returns actions for given state as per current policy.
:param observation: (array_like) current observation;
:param explore: (bool) explore or exploit flag.
"""
observation = torch.from_numpy(observation).float().to(DEVICE)
self.actor_local.eval()
with torch.no_grad():
action = self.actor_local(
observation.unsqueeze(0)).cpu().data.numpy()
self.actor_local.train()
# Add noise for exploration
if explore:
action += self.eps * self.noise()
self.eps = max(EPS_MIN, self.eps * EPS_DECAY)
return np.clip(action, -1, 1)
def update_critic(self, states, actions, rewards, dones, next_states,
next_actions):
Q_targets_next = self.critic_target(next_states, next_actions)
Q_targets = rewards + (DISCOUNT_FACTOR * Q_targets_next *
(1 - dones)).detach()
Q_expected = self.critic_local(states, actions)
critic_loss = F.mse_loss(Q_expected, Q_targets)
self.critic_optim.zero_grad()
critic_loss.backward()
torch.nn.utils.clip_grad_norm_(self.critic_local.parameters(), 1)
self.critic_optim.step()
def update_actor(self, states, action_predictions):
# Update Actor
actor_loss = -self.critic_local(states, action_predictions).mean()
self.actor_optim.zero_grad()
actor_loss.backward()
self.actor_optim.step()
def soft_update(self):
self._soft_update(self.critic_local, self.critic_target, SOFT_UPDATE)
self._soft_update(self.actor_local, self.actor_target, SOFT_UPDATE)
# def learn(self, states, actions, rewards, next_states, dones, next_actions, action_predictions):
# # Update Critic
# Q_targets_next = self.critic_target(next_states, next_actions)
# Q_targets = rewards + (DISCOUNT_FACTOR * Q_targets_next * (1 - dones)).detach()
# Q_expected = self.critic_local(states, actions)
# critic_loss = F.mse_loss(Q_expected, Q_targets)
# self.critic_optim.zero_grad()
# critic_loss.backward()
# torch.nn.utils.clip_grad_norm_(self.critic_local.parameters(), 1)
# self.critic_optim.step()
#
# # Update Actor
# actor_loss = -self.critic_local(states, action_predictions).mean()
# self.actor_optim.zero_grad()
# actor_loss.backward()
# self.actor_optim.step()
#
# # Target network soft update
# self.soft_update(self.critic_local, self.critic_target, SOFT_UPDATE)
# self.soft_update(self.actor_local, self.actor_target, SOFT_UPDATE)
def reset(self):
self.noise.reset()
def make_checkpoint(self):
torch.save(self.actor_local.state_dict(), 'checkpoint_actor.pth')
torch.save(self.critic_local.state_dict(), 'checkpoint_critic.pth')
@staticmethod
def _soft_update(local_model, target_model, tau: float):
"""
Soft update model parameters:
θ_target = τ * θ_local + (1 - τ) * θ_target.
:param local_model: (PyTorch model) weights will be copied from;
:param target_model: (PyTorch model) weights will be copied to;
:param tau: interpolation parameter.
"""
for target_param, local_param in zip(target_model.parameters(),
local_model.parameters()):
target_param.data.copy_(tau * local_param.data +
(1.0 - tau) * target_param.data)
@staticmethod
def hard_update(local_model, target_model):
"""
Hard update model parameters.
:param local_model: (PyTorch model) weights will be copied from;
:param target_model: (PyTorch model) weights will be copied to;
"""
for target_param, local_param in zip(target_model.parameters(),
local_model.parameters()):
target_param.data.copy_(local_param.data)
class Agent():
def __init__(self, state_size, action_size, n_agents, random_seed):
self.state_size = state_size
self.action_size = action_size
self.n_agents = n_agents
self.seed = random.seed(random_seed)
#Actor Network
self.actor_local = Actor(state_size, action_size,
random_seed).to(device)
self.actor_target = Actor(state_size, action_size,
random_seed).to(device)
self.actor_optimizer = optim.Adam(self.actor_local.parameters(),
lr=LR_ACTOR)
#Critic Network
self.critic_local = Critic(state_size, action_size,
random_seed).to(device)
self.critic_target = Critic(state_size, action_size,
random_seed).to(device)
self.critic_optimizer = optim.Adam(self.critic_local.parameters(),
lr=LR_CRITIC,
weight_decay=WEIGHT_DECAY)
#Noise Process
self.noise = OUNoise((n_agents, action_size), random_seed)
#Replay Memory
self.memory = ReplayBuffer(action_size, BUFFER_SIZE, BATCH_SIZE,
random_seed)
def step(self, state, action, reward, next_state, done, timestep):
#Save Memory
for state, action, reward, next_state, done in zip(
state, action, reward, next_state, done):
self.memory.add(state, action, reward, next_state, done)
if timestep % N_LEARN_TIMESTEPS != 0:
return
#IF enough samples in memory
if len(self.memory) > BATCH_SIZE:
for i in range(N_LEARN_UPDATES):
#Load sample of tuples from memory
experiences = self.memory.sample()
#Learn from a randomly selected sample
self.learn(experiences, GAMMA)
def act(self, state, add_noise=True):
state = torch.from_numpy(state).float().to(device)
self.actor_local.eval()
with torch.no_grad():
action = self.actor_local(state).cpu().data.numpy()
self.actor_local.train()
if add_noise:
action += self.noise.sample()
#Return action
return np.clip(action, -1, 1)
def reset(self):
self.noise.reset()
def learn(self, experiences, gamma):
states, actions, rewards, next_states, dones = experiences
# ---------------------------- update critic ---------------------------- #
#Get predicted actions + Q values from target models
actions_next = self.actor_target(next_states)
Q_targets_next = self.critic_target(next_states, actions_next)
#Q targets for current states
Q_targets = rewards + (gamma * Q_targets_next * (1 - dones))
#Critic loss
Q_expected = self.critic_local(states, actions)
critic_loss = F.mse_loss(Q_expected, Q_targets)
#Minimize loss
self.critic_optimizer.zero_grad()
critic_loss.backward()
torch.nn.utils.clip_grad_norm_(self.critic_local.parameters(), 1)
self.critic_optimizer.step()
# ---------------------------- update actor ---------------------------- #
#Actor Loss
actions_pred = self.actor_local(states)
#Negative sign for gradient ascent
actor_loss = -self.critic_local(states, actions_pred).mean()
#Minimize Loss
self.actor_optimizer.zero_grad()
actor_loss.backward()
self.actor_optimizer.step()
# ----------------------- update target networks ----------------------- #
self.soft_update(self.critic_local, self.critic_target, TAU)
self.soft_update(self.actor_local, self.actor_target, TAU)
def soft_update(self, local_model, target_model, tau):
for local_param, target_param in zip(local_model.parameters(),
target_model.parameters()):
target_param.data.copy_(tau * local_param.data +
(1.0 - tau) * target_param.data)
class DDPGAgent():
"""Interacts with and learns from the environment."""
def __init__(self,
state_size,
action_size,
n_agents,
seed,
pretrainedWeightsFile='checkpoint_actor.pth',
train=True):
"""Initialize an Agent object.
Params
======
state_size (int): dimension of each state
action_size (int): dimension of each action
n_agents (int): number of agents in the multi-agent env
pretrainedWeightsFile (string): filename for pretrained weights when running in test mode
train (bool): True when training, False when Testing
seed (int): random seed
"""
self.state_size = state_size
self.action_size = action_size
self.n_agents = n_agents
self.seed = random.seed(seed)
self.train = train
if self.train:
self.actor = Actor(state_size, action_size,
seed).to(device) # Actor Q network
self.critic = Critic(state_size, action_size,
seed).to(device) # Critic Q network
self.actor_tgt = Actor(state_size, action_size,
seed).to(device) # Target Actor Q network
self.critic_tgt = Critic(state_size, action_size, seed).to(
device) # Target Critic Q network
self.optimizer_actor = optim.Adam(
self.actor.parameters(),
lr=LR_ACTOR) # Optimizer for training the actor
self.optimizer_critic = optim.Adam(
self.critic.parameters(),
lr=LR_CRITIC,
weight_decay=WEIGHT_DECAY) # Optimizer for training the critic
self.memory = ReplayBuffer(action_size, BUFFER_SIZE, BATCH_SIZE,
seed) # Replay memory
self.t_step = 0 # Initialize time step (for updating every UPDATE_EVERY steps)
self.noise = OUNoise(action_size, seed) # Noise Process
else:
self.actor = Actor(state_size, action_size,
seed).to(device) # Local Q network
self.actor.load_state_dict(
torch.load(pretrainedWeightsFile)
) # Load pre trained weights for Q network from file if testing
def step(self, states, actions, rewards, next_states, dones):
"""
Define step behavior of agent
Params
======
states (array of array): current state(s) of the agent(s)
actions (array of array): action(s) taken
rewards (array_like): reward(s) procured
next_state (array of array): transitioned state(s)
dones (array_like): indicates whether the episode has ended
"""
# Save experience in replay memory
self.t_step += 1
for i in range(self.n_agents):
self.memory.add(states[i], actions[i], rewards[i], next_states[i],
dones[i])
if (len(self.memory) > BATCH_SIZE) and (self.t_step % UPDATE_EVERY
== 0):
# if (len(self.memory) > BATCH_SIZE):
self.t_step = 0
for _ in range(NUM_UPDATES):
experiences = self.memory.sample()
self.learn(experiences, GAMMA)
def act(self, states, add_noise=True):
"""Returns actions for given state as per current policy.
Params
======
state (array of array): current state(s)
add_noise(bool): indicates whether to add random noise to the actions
"""
states = torch.from_numpy(states).float().to(device)
self.actor.eval()
with torch.no_grad():
action_values = self.actor(states).cpu().data.numpy()
if self.train:
self.actor.train()
if self.train and add_noise:
action_values += [
self.noise.sample() for _ in range(self.n_agents)
]
return np.clip(action_values, -1, 1)
def reset(self):
self.noise.reset()
def learn(self, experiences, gamma):
"""Update value parameters using given batch of experience tuples.
Params
======
experiences (Tuple[torch.Variable]): tuple of (s, a, r, s', done) tuples
gamma (float): discount factor
"""
states, actions, rewards, next_states, dones = experiences
## ------------------- Update Critic ----------------------- #
next_actions = self.actor_tgt(next_states)
critic_tgt_next = self.critic_tgt(next_states, next_actions)
critic_tgt = rewards + (gamma * critic_tgt_next * (1 - dones))
# print(actions.size())
critic_exp = self.critic(states, actions)
critic_loss = F.mse_loss(critic_exp, critic_tgt)
self.optimizer_critic.zero_grad()
critic_loss.backward()
self.optimizer_critic.step()
## -------------------- Update Actor ----------------------- #
predicted_actions = self.actor(states)
actor_loss = -self.critic(states, predicted_actions).mean()
self.optimizer_actor.zero_grad()
actor_loss.backward()
self.optimizer_actor.step()
# ------------------- update target network ------------------- #
self.soft_update(self.critic, self.critic_tgt, TAU)
self.soft_update(self.actor, self.actor_tgt, TAU)
def soft_update(self, local_model, target_model, tau):
"""Soft update model parameters.
θ_target = τ*θ_local + (1 - τ)*θ_target
Params
======
local_model (PyTorch model): weights will be copied from
target_model (PyTorch model): weights will be copied to
tau (float): interpolation parameter
"""
for target_param, local_param in zip(target_model.parameters(),
local_model.parameters()):
target_param.data.copy_(tau * local_param.data +
(1.0 - tau) * target_param.data)
class Agent():
"""Interacts with and learns from the environment."""
def __init__(self, state_size, action_size, num_agents, random_seed):
"""Initialize an Agent object.
Params
======
state_size (int): dimension of each state
action_size (int): dimension of each action
random_seed (int): random seed
"""
#self.state_size = state_size
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
self.action_size = action_size
self.seed = random.seed(random_seed)
self.num_agents = num_agents
# Actor Network (w/ Target Network)
self.actor_local = Actor(state_size, action_size,
random_seed).to(device)
self.actor_target = Actor(state_size, action_size,
random_seed).to(device)
self.actor_optimizer = optim.Adam(self.actor_local.parameters(),
lr=LR_ACTOR)
# Critic Network (w/ Target Network)
self.critic_local = Critic(state_size, action_size,
random_seed).to(device)
self.critic_target = Critic(state_size, action_size,
random_seed).to(device)
self.critic_optimizer = optim.Adam(self.critic_local.parameters(),
lr=LR_CRITIC,
weight_decay=WEIGHT_DECAY)
# Add noise with inertia
self.noise = OUNoise((num_agents, action_size), random_seed)
# Replay memory
self.memory = ReplayBuffer(action_size, BUFFER_SIZE, BATCH_SIZE,
random_seed, device)
def step(self,
states,
actions,
rewards,
next_states,
dones,
timestep=0,
episode=999):
"""Save experience in replay memory, and use random sample from buffer to learn."""
# Save experience / reward
for i in range(self.num_agents):
self.memory.add(states[i, :], actions[i, :], rewards[i],
next_states[i, :], dones[i])
# Learn, if enough samples are available in memory and it is time for a soft update
if len(self.memory
) > BATCH_SIZE and timestep % UPDATE_TARGET_EVERY == 0:
for i in range(AGENT_LEARN_COUNT):
#experiences = self.memory.sample()
experiences = self.memory.sampleByRewards()
self.learn(experiences, GAMMA)
return
def act(self, states, add_noise=True):
"""Returns actions for given state as per current policy."""
states = torch.from_numpy(states).float().to(device)
actions = np.zeros((self.num_agents, self.action_size))
self.actor_local.eval()
with torch.no_grad():
actions = self.actor_local(states).cpu().data.numpy()
self.actor_local.train()
if add_noise:
actions += self.noise.sample()
return np.clip(actions, -1, 1)
def reset(self):
self.noise.reset()
def learn(self, experiences, gamma):
"""Update policy and value parameters using given batch of experience tuples.
Q_targets = r + γ * critic_target(next_state, actor_target(next_state))
where:
actor_target(state) -> action
critic_target(state, action) -> Q-value
Params
======
experiences (Tuple[torch.Tensor]): tuple of (s, a, r, s', done) tuples
gamma (float): discount factor
"""
states, actions, rewards, next_states, dones = experiences
# ---------------------------- update critic ---------------------------- #
# Get predicted next-state actions and Q values from target models
actions_next = self.actor_target(next_states)
Q_targets_next = self.critic_target(next_states, actions_next)
# Compute Q targets for current states (y_i)
Q_targets = rewards + (gamma * Q_targets_next * (1 - dones))
# Compute critic loss
Q_expected = self.critic_local(states, actions)
critic_loss = F.mse_loss(Q_expected, Q_targets)
# Minimize the loss
self.critic_optimizer.zero_grad()
critic_loss.backward()
#torch.nn.utils.clip_grad_norm_(self.critic_local.parameters(), 1)
self.critic_optimizer.step()
# ---------------------------- update actor ---------------------------- #
# Compute actor loss
actions_pred = self.actor_local(states)
actor_loss = -self.critic_local(states, actions_pred).mean()
# Minimize the loss
self.actor_optimizer.zero_grad()
actor_loss.backward()
self.actor_optimizer.step()
# ----------------------- update target networks ----------------------- #
self.soft_update(self.critic_local, self.critic_target, TAU)
self.soft_update(self.actor_local, self.actor_target, TAU)
def soft_update(self, local_model, target_model, tau):
"""Soft update model parameters.
θ_target = τ*θ_local + (1 - τ)*θ_target
Params
======
local_model: PyTorch model (weights will be copied from)
target_model: PyTorch model (weights will be copied to)
tau (float): Decides how much local values should be updated
"""
for target_param, local_param in zip(target_model.parameters(),
local_model.parameters()):
target_param.data.copy_(tau * local_param.data +
(1.0 - tau) * target_param.data)
class DDPG(object):
def __init__(self, nb_states, nb_actions, args, discrete, use_cuda=False):
if args.seed > 0:
self.seed(args.seed)
self.nb_states = nb_states
self.nb_actions = nb_actions
self.discrete = discrete
# Create Actor and Critic Network
net_cfg = {
'hidden1': args.hidden1,
'hidden2': args.hidden2,
'init_w': args.init_w
}
self.actor = Actor(self.nb_states * args.window_length,
self.nb_actions, **net_cfg)
self.actor_target = Actor(self.nb_states * args.window_length,
self.nb_actions, **net_cfg)
self.actor_optim = Adam(self.actor.parameters(), lr=args.prate)
self.critic = Critic(self.nb_states * args.window_length,
self.nb_actions, **net_cfg)
self.critic_target = Critic(self.nb_states * args.window_length,
self.nb_actions, **net_cfg)
self.critic_optim = Adam(self.critic.parameters(), lr=args.rate)
hard_update(self.actor_target,
self.actor) # Make sure target is with the same weight
hard_update(self.critic_target, self.critic)
#Create replay buffer
self.memory = rpm(
args.rmsize
) # SequentialMemory(limit=args.rmsize, window_length=args.window_length)
self.random_process = Myrandom(size=nb_actions)
# Hyper-parameters
self.batch_size = args.batch_size
self.tau = args.tau
self.discount = args.discount
self.depsilon = 1.0 / args.epsilon
#
self.epsilon = 1.0
self.s_t = None # Most recent state
self.a_t = None # Most recent action
self.use_cuda = use_cuda
#
if self.use_cuda: self.cuda()
def update_policy(self, train_actor=True):
# Sample batch
state_batch, action_batch, reward_batch, \
next_state_batch, terminal_batch = self.memory.sample_batch(self.batch_size)
# state_batch, action_batch, reward_batch, \
# next_state_batch, terminal_batch = self.memory.sample_and_split(self.batch_size)
# Prepare for the target q batch
next_q_values = self.critic_target([
to_tensor(next_state_batch, volatile=True),
self.actor_target(to_tensor(next_state_batch, volatile=True)),
])
next_q_values.volatile = False
target_q_batch = to_tensor(reward_batch) + \
self.discount * to_tensor((1 - terminal_batch.astype(np.float))) * next_q_values
# Critic update
self.critic.zero_grad()
q_batch = self.critic(
[to_tensor(state_batch),
to_tensor(action_batch)])
value_loss = criterion(q_batch, target_q_batch)
value_loss.backward()
self.critic_optim.step()
self.actor.zero_grad()
policy_loss = -self.critic(
[to_tensor(state_batch),
self.actor(to_tensor(state_batch))])
policy_loss = policy_loss.mean()
policy_loss.backward()
if train_actor == True:
self.actor_optim.step()
# Target update
soft_update(self.actor_target, self.actor, self.tau)
soft_update(self.critic_target, self.critic, self.tau)
return -policy_loss, value_loss
def eval(self):
self.actor.eval()
self.actor_target.eval()
self.critic.eval()
self.critic_target.eval()
def cuda(self):
print("use cuda")
self.actor.cuda()
self.actor_target.cuda()
self.critic.cuda()
self.critic_target.cuda()
def observe(self, r_t, s_t1, done):
self.memory.append([self.s_t, self.a_t, r_t, s_t1, done])
self.s_t = s_t1
def random_action(self):
action = np.random.uniform(-1., 1., self.nb_actions)
self.a_t = action
if self.discrete:
return action.argmax()
else:
return action
def select_action(self,
s_t,
decay_epsilon=True,
return_fix=False,
noise_level=1):
action = to_numpy(self.actor(to_tensor(np.array([s_t])))).squeeze(0)
# print(self.random_process.sample(), action)
noise_level = noise_level * max(self.epsilon, 0)
action = action * (1 - noise_level) + (self.random_process.sample() *
noise_level)
# print(max(self.epsilon, 0) * self.random_process.sample() * noise_level, noise_level)
action = np.clip(action, -1., 1.)
# print(action)
if decay_epsilon:
self.epsilon -= self.depsilon
self.a_t = action
if return_fix:
return action
if self.discrete:
return action.argmax()
else:
return action
def reset(self, obs):
self.s_t = obs
self.random_process.reset_states()
def load_weights(self, output):
if output is None: return
self.actor.load_state_dict(torch.load('{}/actor.pkl'.format(output)))
self.actor_target.load_state_dict(
torch.load('{}/actor_target.pkl'.format(output)))
self.critic.load_state_dict(torch.load('{}/critic.pkl'.format(output)))
self.critic_target.load_state_dict(
torch.load('{}/critic_target.pkl'.format(output)))
def save_model(self, output):
if self.use_cuda:
self.actor.cpu()
self.actor_target.cpu()
self.critic.cpu()
self.critic_target.cpu()
torch.save(self.actor.state_dict(), '{}/actor.pkl'.format(output))
torch.save(self.actor_target.state_dict(),
'{}/actor_target.pkl'.format(output))
torch.save(self.critic.state_dict(), '{}/critic.pkl'.format(output))
torch.save(self.critic_target.state_dict(),
'{}/critic_target.pkl'.format(output))
if self.use_cuda:
self.actor.cuda()
self.actor_target.cuda()
self.critic.cuda()
self.critic_target.cuda()
def seed(self, s):
torch.manual_seed(s)
if self.use_cuda:
torch.cuda.manual_seed(s)
class Agent():
"""Interacts with and learns from the environment."""
def __init__(self,
state_size,
action_size,
random_seed,
buffer_size,
batch_size,
learning_rate_actor,
learning_rate_critic,
gamma,
tau):
"""Initialize an Agent object.
Params
======
state_size (int): dimension of each state
action_size (int): dimension of each action
random_seed (int): random seed
"""
self.state_size = state_size
self.action_size = action_size
self.buffer_size = buffer_size
self.batch_size = batch_size
self.gamma = gamma
self.tau = tau
# Actor Network (w/ Target Network)
self.actor_local = Actor(state_size, action_size, random_seed).to(device)
self.actor_target = Actor(state_size, action_size, random_seed).to(device)
self.actor_optimizer = optim.Adam(self.actor_local.parameters(), lr=learning_rate_actor)
# Critic Network (w/ Target Network)
self.critic_local = Critic(state_size, action_size, random_seed).to(device)
self.critic_target = Critic(state_size, action_size, random_seed).to(device)
self.critic_optimizer = optim.Adam(self.critic_local.parameters(), lr=learning_rate_critic)
# initializing the target networks
self.soft_update(self.critic_local, self.critic_target, 1)
self.soft_update(self.actor_local, self.actor_target, 1)
# Noise process
self.noise = OUNoise(action_size, random_seed)
# Replay memory
self.memory = ReplayBuffer(action_size, self.buffer_size, self.batch_size, random_seed)
def step(self, state, action, reward, next_state, done):
"""Save experience in replay memory, and use random sample from buffer to learn."""
# Save experience / reward
self.memory.add(state, action, reward, next_state, done)
# Learn, if enough samples are available in memory
if len(self.memory) > self.batch_size:
experiences = self.memory.sample()
self.learn(experiences, self.gamma)
def act(self, state, add_noise=True):
"""Returns actions for given state as per current policy."""
state = torch.from_numpy(state).float().to(device)
self.actor_local.eval()
with torch.no_grad():
action = self.actor_local(state).cpu().data.numpy()
self.actor_local.train()
if add_noise:
action += self.noise.sample()
return np.clip(action, -1, 1)
def reset(self):
self.noise.reset()
def learn(self, experiences, gamma):
"""Update policy and value parameters using given batch of experience tuples.
Q_targets = r + γ * critic_target(next_state, actor_target(next_state))
where:
actor_target(state) -> action
critic_target(state, action) -> Q-value
Params
======
experiences (Tuple[torch.Tensor]): tuple of (s, a, r, s', done) tuples
gamma (float): discount factor
"""
states, actions, rewards, next_states, dones = experiences
self.update_critic(actions, dones, gamma, next_states, rewards, states)
self.update_actor(states)
self.update_target_networks()
def update_target_networks(self):
self.soft_update(self.critic_local, self.critic_target, self.tau)
self.soft_update(self.actor_local, self.actor_target, self.tau)
def update_actor(self, states):
# Compute actor loss
actions_pred = self.actor_local(states)
actor_loss = -self.critic_local(states, actions_pred).mean()
# Minimize the loss
self.actor_optimizer.zero_grad()
actor_loss.backward()
self.actor_optimizer.step()
def update_critic(self, actions, dones, gamma, next_states, rewards, states):
# Get predicted next-state actions and Q values from target models
actions_next = self.actor_target(next_states)
Q_targets_next = self.critic_target(next_states, actions_next)
# Compute Q targets for current states (y_i)
Q_targets = rewards + (gamma * Q_targets_next * (1 - dones))
# Compute critic loss
Q_expected = self.critic_local(states, actions)
critic_loss = F.mse_loss(Q_expected, Q_targets)
# Minimize the loss
self.critic_optimizer.zero_grad()
critic_loss.backward()
self.critic_optimizer.step()
def soft_update(self, local_model, target_model, tau):
"""Soft update model parameters.
θ_target = τ*θ_local + (1 - τ)*θ_target
Params
======
local_model: PyTorch model (weights will be copied from)
target_model: PyTorch model (weights will be copied to)
tau (float): interpolation parameter
"""
for target_param, local_param in zip(target_model.parameters(), local_model.parameters()):
target_param.data.copy_(tau * local_param.data + (1.0 - tau) * target_param.data)
class Agent():
"""Reinforcement Learning agent that learns using DDPG."""
def __init__(self, task):
self.task = task
self.state_size = task.state_size
self.action_size = task.action_size
self.action_low = task.action_low
self.action_high = task.action_high
# Actor (Policy) Model
self.actor_local = Actor(self.state_size, self.action_size,
self.action_low, self.action_high)
self.actor_target = Actor(self.state_size, self.action_size,
self.action_low, self.action_high)
# Critic (Value) Model
self.critic_local = Critic(self.state_size, self.action_size)
self.critic_target = Critic(self.state_size, self.action_size)
# Initialize target model parameters with local model parameters
self.critic_target.model.set_weights(
self.critic_local.model.get_weights())
self.actor_target.model.set_weights(
self.actor_local.model.get_weights())
# Noise process
self.exploration_mu = 0
self.exploration_theta = 0.15
self.exploration_sigma = 0.2
self.noise = OUNoise(self.action_size, self.exploration_mu,
self.exploration_theta, self.exploration_sigma)
# Replay memory
self.buffer_size = 100000
self.batch_size = 64
self.memory = ReplayBuffer(self.buffer_size, self.batch_size)
# Algorithm parameters
self.gamma = 0.99 # discount factor
self.tau = 0.001 # for soft update of target parameters
def reset_episode(self):
self.noise.reset()
state = self.task.reset()
self.last_state = state
return state
def step(self, action, reward, next_state, done):
# Save experience / reward
self.memory.add(self.last_state, action, reward, next_state, done)
# Learn, if enough samples are available in memory
if len(self.memory) > self.batch_size:
experiences = self.memory.sample()
self.learn(experiences)
# Roll over last state and action
self.last_state = next_state
def act(self, state):
"""Returns actions for given state(s) as per current policy."""
state = np.reshape(state, [-1, self.state_size])
action = self.actor_local.model.predict(state)[0]
return list(action +
self.noise.sample()) # add some noise for exploration
def learn(self, experiences):
"""Update policy and value parameters using given batch of experience tuples."""
# Convert experience tuples to separate arrays for each element (states, actions, rewards, etc.)
states = np.vstack([e.state for e in experiences if e is not None])
actions = np.array([e.action for e in experiences
if e is not None]).astype(np.float32).reshape(
-1, self.action_size)
rewards = np.array([e.reward for e in experiences if e is not None
]).astype(np.float32).reshape(-1, 1)
dones = np.array([e.done for e in experiences
if e is not None]).astype(np.uint8).reshape(-1, 1)
next_states = np.vstack(
[e.next_state for e in experiences if e is not None])
# Get predicted next-state actions and Q values from target models
# Q_targets_next = critic_target(next_state, actor_target(next_state))
actions_next = self.actor_target.model.predict_on_batch(next_states)
Q_targets_next = self.critic_target.model.predict_on_batch(
[next_states, actions_next])
# Compute Q targets for current states and train critic model (local)
Q_targets = rewards + self.gamma * Q_targets_next * (1 - dones)
self.critic_local.model.train_on_batch(x=[states, actions],
y=Q_targets)
# Train actor model (local)
action_gradients = np.reshape(
self.critic_local.get_action_gradients([states, actions, 0]),
(-1, self.action_size))
self.actor_local.train_fn([states, action_gradients,
1]) # custom training function
# Soft-update target models
self.soft_update(self.critic_local.model, self.critic_target.model)
self.soft_update(self.actor_local.model, self.actor_target.model)
def soft_update(self, local_model, target_model):
"""Soft update model parameters."""
local_weights = np.array(local_model.get_weights())
target_weights = np.array(target_model.get_weights())
assert len(local_weights) == len(
target_weights
), "Local and target model parameters must have the same size"
new_weights = self.tau * local_weights + (1 -
self.tau) * target_weights
target_model.set_weights(new_weights)
class DDPGAgent:
def __init__(self,
dimS,
dimA,
gamma=0.99,
actor_lr=1e-4,
critic_lr=1e-3,
tau=1e-3,
sigma=0.1,
hidden_size1=400,
hidden_size2=300,
buffer_size=int(1e6),
batch_size=128,
render=False):
self.dimS = dimS
self.dimA = dimA
self.gamma = gamma
self.pi_lr = actor_lr
self.q_lr = critic_lr
self.tau = tau
self.sigma = sigma
self.batch_size = batch_size
# networks definition
# pi : actor network, Q : critic network
self.pi = Actor(dimS, dimA, hidden_size1, hidden_size2)
self.Q = Critic(dimS, dimA, hidden_size1, hidden_size2)
# target networks
self.targ_pi = copy.deepcopy(self.pi)
self.targ_Q = copy.deepcopy(self.Q)
self.buffer = ReplayBuffer(dimS, dimA, limit=buffer_size)
self.Q_optimizer = torch.optim.Adam(self.Q.parameters(), lr=self.q_lr)
self.pi_optimizer = torch.optim.Adam(self.pi.parameters(), lr=self.pi_lr)
self.render = render
def target_update(self):
# soft-update for both actors and critics
# \theta^\prime = \tau * \theta + (1 - \tau) * \theta^\prime
for th, targ_th in zip(self.pi.parameters(), self.targ_pi.parameters()): # th : theta
targ_th.data.copy_(self.tau * th.data + (1.0 - self.tau) * targ_th.data)
for th, targ_th in zip(self.Q.parameters(), self.targ_Q.parameters()):
targ_th.data.copy_(self.tau * th.data + (1.0 - self.tau) * targ_th.data)
def get_action(self, state, eval=False):
state = torch.tensor(state, dtype=torch.float)
with torch.no_grad():
action = self.pi(state)
action = action.numpy()
if not eval:
# for exploration, we use a behavioral policy of the form
# \beta(s) = \pi(s) + N(0, \sigma^2)
noise = self.sigma * np.random.randn(self.dimA)
return action + noise
else:
return action
def train(self):
"""
train actor-critic network using DDPG
"""
batch = self.buffer.sample_batch(batch_size=self.batch_size)
# unroll batch
observations = torch.tensor(batch['state'], dtype=torch.float)
actions = torch.tensor(batch['action'], dtype=torch.float)
rewards = torch.tensor(batch['reward'], dtype=torch.float)
next_observations = torch.tensor(batch['next_state'], dtype=torch.float)
terminal_flags = torch.tensor(batch['done'], dtype=torch.float)
mask = torch.tensor([1.]) - terminal_flags
# compute TD targets based on target networks
# if done, set target value to reward
target = rewards + self.gamma * mask * self.targ_Q(next_observations, self.targ_pi(next_observations))
out = self.Q(observations, actions)
loss_ftn = MSELoss()
loss = loss_ftn(out, target)
self.Q_optimizer.zero_grad()
loss.backward()
self.Q_optimizer.step()
pi_loss = - torch.mean(self.Q(observations, self.pi(observations)))
self.pi_optimizer.zero_grad()
pi_loss.backward()
self.pi_optimizer.step()
self.target_update()
def save_model(self, path):
checkpoint_path = path + 'model.pth.tar'
torch.save(
{'actor': self.pi.state_dict(),
'critic': self.Q.state_dict(),
'target_actor': self.targ_pi.state_dict(),
'target_critic': self.targ_Q.state_dict(),
'actor_optimizer': self.pi_optimizer.state_dict(),
'critic_optimizer': self.Q_optimizer.state_dict()
},
checkpoint_path)
return
def load_model(self, path):
checkpoint = torch.load(path)
self.pi.load_state_dict(checkpoint['actor'])
self.Q.load_state_dict(checkpoint['critic'])
self.targ_pi.load_state_dict(checkpoint['target_actor'])
self.targ_Q.load_state_dict(checkpoint['target_critic'])
self.pi_optimizer.load_state_dict(checkpoint['actor_optimizer'])
self.Q_optimizer.load_state_dict(checkpoint['critic_optimizer'])
return
class Agent():
def __init__(self, state_size, action_size, random_seed):
self.state_size = state_size
self.action_size = action_size
self.seed = random.seed(random_seed)
# Actor w/ target
self.actor_local = Actor(state_size, action_size,
seed=random_seed).to(device)
self.actor_target = Actor(state_size, action_size,
seed=random_seed).to(device)
self.actor_opt = optim.Adam(self.actor_local.parameters(), lr=LR_ACTOR)
# Critic w/ target
self.critic_local = Critic(state_size, action_size,
seed=random_seed).to(device)
self.critic_target = Critic(state_size, action_size,
seed=random_seed).to(device)
self.critic_opt = optim.Adam(self.critic_local.parameters(),
lr=LR_CRITIC,
weight_decay=WEIGHT_DECAY)
# Misc
self.noise = OUNoise(action_size, random_seed)
self.memory = ReplayBuffer(BUFFER_SIZE, BATCH_SIZE, random_seed)
def step(self, state, action, reward, next_state, done):
self.memory.add(state, action, reward, next_state, done)
if len(self.memory) > BATCH_SIZE:
experiences = self.memory.sample()
self.learn(experiences, GAMMA)
def act(self, state, add_noise=True):
state = torch.from_numpy(state).float().to(device)
self.actor_local.eval()
with torch.no_grad():
action = self.actor_local(state).cpu().data.numpy()
self.actor_local.train()
if add_noise:
action += self.noise.sample()
return np.clip(action, -1, +1)
def reset(self):
self.noise.reset()
def learn(self, experiences, gamma):
states, actions, rewards, next_states, dones = experiences
# update critic
actions_next = self.actor_target(next_states)
Q_targets_next = self.critic_target(next_states, actions_next)
Q_targets = rewards + gamma * Q_targets_next * (1 - dones)
Q_expected = self.critic_local(states, actions)
critic_loss = F.mse_loss(Q_expected, Q_targets)
self.critic_opt.zero_grad()
critic_loss.backward()
self.critic_opt.step()
# update actor
actions_pred = self.actor_local(states)
actor_loss = -self.critic_local(states, actions_pred).mean()
self.actor_opt.zero_grad()
actor_loss.backward()
self.actor_opt.step()
# target network upates
self.soft_update(self.actor_local, self.actor_target, TAU)
self.soft_update(self.critic_local, self.critic_target, TAU)
def soft_update(self, local_model, target_model, tau):
for target_param, local_param in zip(target_model.parameters(),
local_model.parameters()):
mixed_param = tau * local_param.data + (1 -
tau) * target_param.data
target_param.data.copy_(mixed_param)
class DDPGAgent:
def __init__(self, total_agents, state_size, action_size, seed):
self.device = 'cuda:0' if torch.cuda.is_available() else 'cpu'
#self.device = 'cpu'
self.total_agents = total_agents
self.state_size = state_size
self.action_size = action_size
self.seed = random.seed(seed)
self.actor_local = Actor(self.state_size, self.action_size,
seed).to(self.device)
self.actor_target = Actor(self.state_size, self.action_size,
seed).to(self.device)
self.actor_optimizer = optim.Adam(self.actor_local.parameters(),
lr=LR_ACTOR)
self.critic_local = Critic(self.state_size, self.action_size,
seed).to(self.device)
self.critic_target = Critic(self.state_size, self.action_size,
seed).to(self.device)
self.critic_optimizer = optim.Adam(self.critic_local.parameters(),
lr=LR_CRITIC,
weight_decay=WEIGHT_DECAY)
#self.noise = OrnsteinUhlenbeckNoise(action_size, seed)
self.noise = OrnsteinUhlenbeckProcess((self.total_agents, action_size),
std=LinearSchedule(0.2))
self.replay_buffer = UniformReplayBuffer(
BUFFER_SIZE, BATCH_SIZE * self.total_agents, seed, self.device)
#self.replay_buffer = PrioritizedReplay(BUFFER_SIZE, self.device)
print('Device used: {}'.format(self.device))
print('Actor Local DDPG ->', self.actor_local)
print('Actor Target DDPG ->', self.actor_target)
print('Critic Local DDPG ->', self.critic_local)
print('Critic Target DDPG ->', self.critic_target)
def reset(self):
self.noise.reset()
def act(self, states, add_noise=False):
states = torch.from_numpy(states).float().to(self.device)
self.actor_local.eval()
with torch.no_grad():
actions = self.actor_local(states).cpu().data.numpy()
self.actor_local.train()
return np.clip(actions +
self.noise.sample(), -1, 1) if add_noise else actions
def step(self, states, actions, rewards, next_states, dones):
for state, action, reward, next_state, done in zip(
states, actions, rewards, next_states, dones):
self.replay_buffer.add(state, action, reward, next_state, done)
#for _ in range(self.total_agents): TOO SLOW
#if len(self.replay_buffer) > BATCH_SIZE:
return self._learn(self.replay_buffer.sample(), GAMMA)
#return (None,None)
def _learn(self, experiences, gamma):
states, actions, rewards, next_states, dones = experiences
# ---------- CRITIC UPDATE --------------------
next_actions = self.actor_target(next_states)
next_rewards = self.critic_target(next_states, next_actions)
target_rewards = rewards + gamma * next_rewards * (1 - dones)
predicted_rewards = self.critic_local(states, actions)
critic_loss = F.mse_loss(predicted_rewards, target_rewards)
self.critic_optimizer.zero_grad()
critic_loss.backward()
torch.nn.utils.clip_grad_norm_(self.critic_local.parameters(), 1)
self.critic_optimizer.step()
# ---------- ACTOR UPDATE --------------------
predicted_actions = self.actor_local(states)
actor_loss = -self.critic_local(states, predicted_actions).mean()
#print('\rActor Loss: {:.6f} - Critic Loss: {:.6f}'.format(actor_loss, critic_loss), end='')
self.actor_optimizer.zero_grad()
actor_loss.backward()
self.actor_optimizer.step()
self._soft_update(self.critic_local, self.critic_target, TAU)
self._soft_update(self.actor_local, self.actor_target, TAU)
return critic_loss.cpu().data.numpy(), actor_loss.cpu().data.numpy()
def _soft_update(self, local_model, target_model, tau):
for local_parameter, target_parameter in zip(
local_model.parameters(), target_model.parameters()):
target_parameter.data.copy_((1.0 - tau) * target_parameter +
(tau * local_parameter))
class DDPG_Agent():
"""Interacts with and learns from the environment."""
def __init__(self, state_size, action_size, idx, random_seed=0):
"""Initialize an Agent object.
Params
======
state_size (int): dimension of each state
action_size (int): dimension of each action
random_seed (int): random seed
"""
self.state_size = state_size
self.action_size = action_size
self.seed = random.seed(random_seed)
self.idx = idx
# Actor Network (w/ Target Network)
self.actor_local = Actor(state_size, action_size,
random_seed).to(device)
self.actor_target = Actor(state_size, action_size,
random_seed).to(device)
self.actor_optimizer = optim.Adam(self.actor_local.parameters(),
lr=LR_ACTOR)
# Critic Network (w/ Target Network)
self.critic_local = Critic(state_size, action_size,
random_seed).to(device)
self.critic_target = Critic(state_size, action_size,
random_seed).to(device)
self.critic_optimizer = optim.Adam(self.critic_local.parameters(),
lr=LR_CRITIC,
weight_decay=WEIGHT_DECAY)
# Noise process
self.noise = OUNoise(action_size, random_seed)
def act(self, state, add_noise=True, nu=1.0):
"""Returns actions for given state as per current policy."""
state = torch.from_numpy(state).float().to(device)
self.actor_local.eval()
with torch.no_grad():
action = self.actor_local(state).cpu().data.numpy()
self.actor_local.train()
if add_noise:
action += nu * self.noise.sample()
return np.clip(action, -1, 1)
def reset(self):
self.noise.reset()
def learn(self, experiences, actions_next, actions_pred, freq):
"""Update policy and value parameters using given batch of experience tuples.
Q_targets = r + γ * critic_target(next_state, actor_target(next_state))
where:
actor_target(next_state) -> action
critic_target(next_state, next_action) -> Q-value
Params
======
experiences (Tuple[torch.Tensor]): tuple of (s, a, r, s', done) tuples
next_actions (list): next actions computed from each agent
actions_pred (list): prediction for actions for current states from each agent
"""
states, actions, rewards, next_states, dones = experiences
idxt = torch.tensor([self.idx - 1]).to(device)
# ---------------------------- update critic ---------------------------- #
# Get predicted next-state actions and Q values from target model
Q_targets_next = self.critic_target(next_states, actions_next)
# Compute Q targets for current states (y_i)
Q_targets = rewards.index_select(
1, idxt) + (GAMMA * Q_targets_next *
(1 - dones.index_select(1, idxt)))
# Compute critic loss
Q_expected = self.critic_local(states, actions)
critic_loss = F.mse_loss(Q_expected, Q_targets)
# Minimize the loss
self.critic_optimizer.zero_grad()
critic_loss.backward()
self.critic_optimizer.step()
# ---------------------------- update actor ---------------------------- #
# Compute actor loss
actor_loss = -self.critic_local(states, actions_pred).mean()
# Minimize the loss
self.actor_optimizer.zero_grad()
actor_loss.backward()
self.actor_optimizer.step()
# ----------------------- update target networks ----------------------- #
self.soft_update(self.critic_local, self.critic_target, TAU)
self.soft_update(self.actor_local, self.actor_target, TAU)
def soft_update(self, local_model, target_model, tau):
"""Soft update model parameters.
θ_target = τ*θ_local + (1 - τ)*θ_target
Params
======
local_model: PyTorch model (weights will be copied from)
target_model: PyTorch model (weights will be copied to)
tau (float): interpolation parameter
"""
for target_param, local_param in zip(target_model.parameters(),
local_model.parameters()):
target_param.data.copy_(tau * local_param.data +
(1.0 - tau) * target_param.data)
class UADDPG(object):
def __init__(self, nb_states, nb_actions, args):
if args.seed > 0:
self.seed(args.seed)
self.nb_states = nb_states
self.nb_actions = nb_actions
self.train_with_dropout = args.train_with_dropout
self.dropout_p = args.dropout_p
self.dropout_n = args.dropout_n
self.print_var_count = 0
self.action_std = np.array([])
self.save_dir = args.output
self.episode = 0
# self.save_file = open(self.save_dir + '/std.txt', "a")
print("train_with_dropout : " + str(self.train_with_dropout))
print("Dropout p : " + str(self.dropout_p))
print("Dropout n : " + str(self.dropout_n))
# Create Actor and Critic Network
net_cfg_actor = {
'dropout_n': args.dropout_n,
'dropout_p': args.dropout_p,
'hidden1': args.hidden1,
'hidden2': args.hidden2,
'init_w': args.init_w
}
net_cfg_critic = {
'dropout_n': args.dropout_n,
'dropout_p': args.dropout_p,
'hidden1': args.hidden1,
'hidden2': args.hidden2,
'init_w': args.init_w
}
self.actor = Actor(self.nb_states, self.nb_actions, **net_cfg_actor)
self.actor_target = Actor(self.nb_states, self.nb_actions,
**net_cfg_actor)
self.actor_optim = Adam(self.actor.parameters(), lr=args.prate)
self.critic = Critic(self.nb_states, self.nb_actions, **net_cfg_critic)
self.critic_target = Critic(self.nb_states, self.nb_actions,
**net_cfg_critic)
self.critic_optim = Adam(self.critic.parameters(), lr=args.rate)
hard_update(self.actor_target, self.actor)
hard_update(self.critic_target, self.critic)
# Create replay buffer
self.memory = SequentialMemory(limit=args.rmsize,
window_length=args.window_length)
self.random_process = OrnsteinUhlenbeckProcess(size=nb_actions,
theta=args.ou_theta,
mu=args.ou_mu,
sigma=args.ou_sigma)
# Hyper-parameters
self.batch_size = args.bsize
self.tau = args.tau
self.discount = args.discount
self.depsilon = 1.0 / args.epsilon
#
self.epsilon = 1.0
self.s_t = None # Most recent state
self.a_t = None # Most recent action
self.is_training = True
#
if USE_CUDA:
self.cuda()
def update_policy(self):
# Sample batch
state_batch, action_batch, reward_batch, next_state_batch, terminal_batch = self.memory.sample_and_split(
self.batch_size)
# Prepare for the target q batch
# TODO : (1) Also apply epistemic and aleatoric uncertainty to both actor and critic target network
# TOOD : (2) Is it proper to apply epistemic uncertainty to target network? If then, how to apply? Which network to choose for target? Let's think more about it after July.
next_q_values = self.critic_target([
to_tensor(next_state_batch, volatile=True),
self.actor_target(to_tensor(next_state_batch, volatile=True))
])[:
-1] # x : next_state_batch, a : self.actor_target(next_state_batch)
target_q_batch = to_tensor(reward_batch) + self.discount * to_tensor(
terminal_batch.astype(np.float)) * next_q_values
#########################
# Critic update
#########################
self.critic.zero_grad()
# TODO : (Completed) Add epistemic uncertainty for critic network
q_batch = self.critic(
[to_tensor(state_batch),
to_tensor(action_batch)])
# q_batch_mean, q_batch_var = select_q_with_dropout(state_batch, action_batch)
# q_batch = self.critic.foward_with_dropout([to_tensor(state_batch), to_tensor(action_batch)])
# TODO : (Completed) Add aleatoric uncertainty term from aleatoric uncertainty output of critic network (Add aleatoric uncertainty term in criterion)
value_loss = criterion(q_batch, target_q_batch)
# value_loss = AULoss(q_batch, target_q_batch)
value_loss.backward()
self.critic_optim.step()
#########################
# Actor update
#########################
self.actor.zero_grad()
# policy loss
# TODO : (Completed) Add epistemic certainty term from aleatoric certainty output of policy network
policy_loss = -self.critic(
[to_tensor(state_batch),
self.actor(to_tensor(state_batch))])
policy_loss = policy_loss.mean()
# policy_loss = policy_loss.mean() + 1 / self.actor(to_tensor(state_batch)[-1])
policy_loss.backward()
self.actor_optim.step()
#########################
# Target soft update
#########################
soft_update(self.actor_target, self.actor, self.tau)
soft_update(self.critic_target, self.critic, self.tau)
def eval(self):
self.actor.eval()
self.actor_target.eval()
self.critic.eval()
self.critic_target.eval()
def cuda(self):
self.actor.cuda()
self.actor_target.cuda()
self.critic.cuda()
self.critic_target.cuda()
def observe(self, r_t, s_t1, done):
if self.is_training:
self.memory.append(self.s_t, self.a_t, r_t, done)
self.s_t = s_t1
def random_action(self):
action = np.random.uniform(-1., 1., self.nb_actions)
self.a_t = action
return action
# def select_action(self, s_t, decay_epsilon=True):
# action = to_numpy(self.actor(to_tensor(np.array([s_t])))).squeeze(0)
# action += self.is_training*max(self.epsilon, 0)*self.random_process.sample()
#
# if decay_epsilon:
# self.epsilon -= self.depsilon
#
# self.a_t = action
# return action
def select_q_with_dropout(self, s_t, a_t):
dropout_qs = np.arrary([])
with torch.no_grad():
for i in range(self.dropout_n):
q_batch = to_numpy(
self.critic.forward_with_dropout([
to_tensor(s_t), to_tensor(a_t)
]).squeeze(0)[:-1]) # ignore aleatoric variance term
dropout_qs = np.append(dropout_qs, [q_batch])
q_mean = torch.mean(dropout_qs)
q_var = torch.var(dropout_qs)
return q_mean, q_var
def select_action_with_dropout(self, s_t, decay_epsilon=True):
dropout_actions = np.array([])
with torch.no_grad():
for i in range(self.dropout_n):
action = to_numpy(
self.actor.forward_with_dropout(to_tensor(np.array(
[s_t])))).squeeze(0)
dropout_actions = np.append(dropout_actions, [action])
if self.train_with_dropout:
plt_action = to_numpy(
self.actor.forward_with_dropout(to_tensor(np.array(
[s_t])))).squeeze(0)
plt_action += self.is_training * max(
self.epsilon, 0) * self.random_process.sample()
else:
plt_action = to_numpy(self.actor(to_tensor(np.array(
[s_t])))).squeeze(0)
plt_action += self.is_training * max(
self.epsilon, 0) * self.random_process.sample()
"""
UNFIXED RESET POINT for Mujoco
"""
if self.print_var_count != 0 and (self.print_var_count + 1) % 999 == 0:
# self.action_std = np.append(self.action_std, [np.std(dropout_actions)])
with open(self.save_dir + "/std.txt", "a") as myfile:
myfile.write(str(np.std(dropout_actions)) + '\n')
with open(self.save_dir + "/mean.txt", "a") as myfile:
myfile.write(str(np.mean(dropout_actions)) + '\n')
if self.print_var_count % (1000 * 5) == 0:
print("dropout actions std", np.std(dropout_actions),
" ", "dir : ", str(self.save_dir))
"""
FIXED RESET POINT for MCC
"""
# if s_t[0] == -0.5 and s_t[1] == 0:
# # print("fixed dropout actions std", np.std(dropout_actions), " ", "dir : ", str(self.save_dir))
# self.action_std = np.append(self.action_std, [np.std(dropout_actions)])
# # np.savetxt(self.save_dir + '/std.txt', self.action_std, fmt='%4.10f', delimiter=' ')
# with open(self.save_dir + "/std.txt", "a") as myfile:
# myfile.write(str(np.std(dropout_actions))+'\n')
# with open(self.save_dir + "/mean.txt", "a") as myfile:
# myfile.write(str(np.mean(dropout_actions))+'\n')
if not (os.path.isdir(self.save_dir + "/episode/" +
str(self.episode))):
os.makedirs(
os.path.join(self.save_dir + "/episode/" + str(self.episode)))
self.action_std = np.append(self.action_std, [np.std(dropout_actions)])
with open(self.save_dir + "/episode/" + str(self.episode) + "/std.txt",
"a") as myfile:
myfile.write(str(np.std(dropout_actions)) + '\n')
with open(
self.save_dir + "/episode/" + str(self.episode) + "/mean.txt",
"a") as myfile:
myfile.write(str(np.mean(dropout_actions)) + '\n')
self.print_var_count = self.print_var_count + 1
if decay_epsilon:
self.epsilon -= self.depsilon
# dropout_action = np.array([np.mean(dropout_actions)])
self.a_t = plt_action
return plt_action
def reset(self, obs):
self.s_t = obs
self.random_process.reset_states()
def load_weights(self, output):
if output is None: return
self.actor.load_state_dict(torch.load('{}/actor.pkl'.format(output)))
self.critic.load_state_dict(torch.load('{}/critic.pkl'.format(output)))
def save_model(self, output):
torch.save(self.actor.state_dict(), '{}/actor.pkl'.format(output))
torch.save(self.critic.state_dict(), '{}/critic.pkl'.format(output))
def seed(self, s):
torch.manual_seed(s)
if USE_CUDA:
torch.cuda.manual_seed(s)
class DDPG(object):
def __init__(self, memory, nb_status, nb_actions, action_noise=None,
gamma=0.99, tau=0.001, normalize_observations=True,
batch_size=128, observation_range=(-5., 5.), action_range=(-1., 1.),
actor_lr=1e-4, critic_lr=1e-3):
self.nb_status = nb_status
self.nb_actions = nb_actions
self.action_range = action_range
self.observation_range = observation_range
self.normalize_observations = normalize_observations
self.actor = Actor(self.nb_status, self.nb_actions)
self.actor_target = Actor(self.nb_status, self.nb_actions)
self.actor_optim = Adam(self.actor.parameters(), lr=actor_lr)
self.critic = Critic(self.nb_status, self.nb_actions)
self.critic_target = Critic(self.nb_status, self.nb_actions)
self.critic_optim = Adam(self.critic.parameters(), lr=critic_lr)
# Create replay buffer
self.memory = memory # SequentialMemory(limit=args.rmsize, window_length=args.window_length)
self.action_noise = action_noise
# Hyper-parameters
self.batch_size = batch_size
self.tau = tau
self.discount = gamma
if self.normalize_observations:
self.obs_rms = RunningMeanStd()
else:
self.obs_rms = None
def pi(self, obs, apply_noise=True, compute_Q=True):
obs = np.array([obs])
action = to_numpy(self.actor(to_tensor(obs))).squeeze(0)
if compute_Q:
q = self.critic([to_tensor(obs), to_tensor(action)]).cpu().data
else:
q = None
if self.action_noise is not None and apply_noise:
noise = self.action_noise()
assert noise.shape == action.shape
action += noise
action = np.clip(action, self.action_range[0], self.action_range[1])
return action, q[0][0]
def store_transition(self, obs0, action, reward, obs1, terminal1):
self.memory.append(obs0, action, reward, obs1, terminal1)
if self.normalize_observations:
self.obs_rms.update(np.array([obs0]))
def train(self):
# Get a batch.
batch = self.memory.sample(batch_size=self.batch_size)
next_q_values = self.critic_target([
to_tensor(batch['obs1'], volatile=True),
self.actor_target(to_tensor(batch['obs1'], volatile=True))])
next_q_values.volatile = False
target_q_batch = to_tensor(batch['rewards']) + \
self.discount * to_tensor(1 - batch['terminals1'].astype('float32')) * next_q_values
self.critic.zero_grad()
q_batch = self.critic([to_tensor(batch['obs0']), to_tensor(batch['actions'])])
value_loss = criterion(q_batch, target_q_batch)
value_loss.backward()
self.critic_optim.step()
self.actor.zero_grad()
policy_loss = -self.critic([to_tensor(batch['obs0']), self.actor(to_tensor(batch['obs0']))]).mean()
policy_loss.backward()
self.actor_optim.step()
# Target update
soft_update(self.actor_target, self.actor, self.tau)
soft_update(self.critic_target, self.critic, self.tau)
return value_loss.cpu().data[0], policy_loss.cpu().data[0]
def initialize(self):
hard_update(self.actor_target, self.actor) # Make sure target is with the same weight
hard_update(self.critic_target, self.critic)
def update_target_net(self):
soft_update(self.actor_target, self.actor, self.tau)
soft_update(self.critic_target, self.critic, self.tau)
def reset(self):
if self.action_noise is not None:
self.action_noise.reset()
def cuda(self):
self.actor.cuda()
self.actor_target.cuda()
self.critic.cuda()
self.critic_target.cuda()
def __init__(self, nb_states, nb_actions, args):
if args.seed > 0:
self.seed(args.seed)
self.nb_states = nb_states
self.nb_actions = nb_actions
self.train_with_dropout = args.train_with_dropout
self.dropout_p = args.dropout_p
self.dropout_n = args.dropout_n
self.print_var_count = 0
self.action_std = np.array([])
self.save_dir = args.output
self.episode = 0
# self.save_file = open(self.save_dir + '/std.txt', "a")
print("train_with_dropout : " + str(self.train_with_dropout))
print("Dropout p : " + str(self.dropout_p))
print("Dropout n : " + str(self.dropout_n))
# Create Actor and Critic Network
net_cfg_actor = {
'dropout_n': args.dropout_n,
'dropout_p': args.dropout_p,
'hidden1': args.hidden1,
'hidden2': args.hidden2,
'init_w': args.init_w
}
net_cfg_critic = {
'dropout_n': args.dropout_n,
'dropout_p': args.dropout_p,
'hidden1': args.hidden1,
'hidden2': args.hidden2,
'init_w': args.init_w
}
self.actor = Actor(self.nb_states, self.nb_actions, **net_cfg_actor)
self.actor_target = Actor(self.nb_states, self.nb_actions,
**net_cfg_actor)
self.actor_optim = Adam(self.actor.parameters(), lr=args.prate)
self.critic = Critic(self.nb_states, self.nb_actions, **net_cfg_critic)
self.critic_target = Critic(self.nb_states, self.nb_actions,
**net_cfg_critic)
self.critic_optim = Adam(self.critic.parameters(), lr=args.rate)
hard_update(self.actor_target, self.actor)
hard_update(self.critic_target, self.critic)
# Create replay buffer
self.memory = SequentialMemory(limit=args.rmsize,
window_length=args.window_length)
self.random_process = OrnsteinUhlenbeckProcess(size=nb_actions,
theta=args.ou_theta,
mu=args.ou_mu,
sigma=args.ou_sigma)
# Hyper-parameters
self.batch_size = args.bsize
self.tau = args.tau
self.discount = args.discount
self.depsilon = 1.0 / args.epsilon
#
self.epsilon = 1.0
self.s_t = None # Most recent state
self.a_t = None # Most recent action
self.is_training = True
#
if USE_CUDA:
self.cuda()
class Agent():
"""Interacts with and learns from the environment."""
def __init__(self, state_size, action_size, random_seed):
"""Initialize an Agent object.
Params
======
state_size (int): dimension of each state
action_size (int): dimension of each action
random_seed (int): random seed
"""
self.state_size = state_size
self.action_size = action_size
self.seed = random.seed(random_seed)
self.epsilon = EPSILON
# Actor Network (w/ Target Network)
self.actor_local = Actor(state_size, action_size,
random_seed).to(device)
self.actor_target = Actor(state_size, action_size,
random_seed).to(device)
self.actor_optimizer = optim.Adam(self.actor_local.parameters(),
lr=LR_ACTOR)
# Critic Network (w/ Target Network)
self.critic_local = Critic(state_size, action_size,
random_seed).to(device)
self.critic_target = Critic(state_size, action_size,
random_seed).to(device)
self.critic_optimizer = optim.Adam(self.critic_local.parameters(),
lr=LR_CRITIC,
weight_decay=WEIGHT_DECAY)
# Noise process
self.noise = OUNoise(action_size, random_seed)
# Replay memory
self.memory = ReplayBuffer(action_size, BUFFER_SIZE, BATCH_SIZE,
random_seed)
# Make sure target is with the same weight as the source
self.hard_copy(self.actor_target, self.actor_local)
self.hard_copy(self.critic_target, self.critic_local)
def add_to_memory(self, state, action, reward, next_state, done):
self.memory.add(state, action, reward, next_state, done)
def step(self):
if len(
self.memory
) > BATCH_SIZE: # Learn, if enough samples are available in memory
experiences = self.memory.sample()
self.learn(experiences, GAMMA)
def act(self, state, add_noise=True):
"""Returns actions for given state as per current policy."""
state = torch.from_numpy(state).float().to(device)
self.actor_local.eval()
with torch.no_grad():
action = self.actor_local(state).cpu().data.numpy()
self.actor_local.train()
if add_noise:
action += self.epsilon * self.noise.sample()
return action
def reset(self):
self.noise.reset()
def learn(self, experiences, gamma):
"""Update policy and value parameters using given batch of experience tuples.
Q_targets = r + ? * critic_target(next_state, actor_target(next_state))
where:
actor_target(state) -> action
critic_target(state, action) -> Q-value
Params
======
experiences (Tuple[torch.Tensor]): tuple of (s, a, r, s', done) tuples
gamma (float): discount factor
"""
states, actions, rewards, next_states, dones = experiences
# ---------------------------- update critic ---------------------------- #
# Get predicted next-state actions and Q values from target models
actions_next = self.actor_target(next_states)
Q_targets_next = self.critic_target(next_states, actions_next)
# Compute Q targets for current states (y_i)
Q_targets = rewards + (gamma * Q_targets_next * (1 - dones))
# Compute critic loss
Q_expected = self.critic_local(states, actions)
critic_loss = F.mse_loss(Q_expected, Q_targets)
# Minimize the loss
self.critic_optimizer.zero_grad()
critic_loss.backward()
torch.nn.utils.clip_grad_norm_(self.critic_local.parameters(), 1)
self.critic_optimizer.step()
# ---------------------------- update actor ---------------------------- #
# Compute actor loss
actions_pred = self.actor_local(states)
actor_loss = -self.critic_local(states, actions_pred).mean()
# Minimize the loss
self.actor_optimizer.zero_grad()
actor_loss.backward()
self.actor_optimizer.step()
# ----------------------- update target networks ----------------------- #
self.soft_update(self.critic_local, self.critic_target, TAU)
self.soft_update(self.actor_local, self.actor_target, TAU)
# ---------------------------- update noise ---------------------------- #
self.epsilon -= EPSILON_DECAY
self.noise.reset()
def soft_update(self, local_model, target_model, tau):
"""Soft update model parameters.
?_target = t*?_local + (1 - t)*?_target
Params
======
local_model: PyTorch model (weights will be copied from)
target_model: PyTorch model (weights will be copied to)
tau (float): interpolation parameter
"""
for target_param, local_param in zip(target_model.parameters(),
local_model.parameters()):
target_param.data.copy_(tau * local_param.data +
(1.0 - tau) * target_param.data)
def hard_copy(self, target, source):
for target_param, param in zip(target.parameters(),
source.parameters()):
target_param.data.copy_(param.data)
class DDPG(object):
def __init__(self, nb_status, nb_actions, args, writer):
self.clip_actor_grad = args.clip_actor_grad
self.nb_status = nb_status * args.window_length
self.nb_actions = nb_actions
self.discrete = args.discrete
self.pic = args.pic
self.writer = writer
self.select_time = 0
if self.pic:
self.nb_status = args.pic_status
# Create Actor and Critic Network
net_cfg = {
'hidden1': args.hidden1,
'hidden2': args.hidden2,
'use_bn': args.bn,
'init_method': args.init_method
}
if args.pic:
self.cnn = CNN(1, args.pic_status)
self.cnn_target = CNN(1, args.pic_status)
self.cnn_optim = Adam(self.cnn.parameters(), lr=args.crate)
self.actor = Actor(self.nb_status, self.nb_actions, **net_cfg)
self.actor_target = Actor(self.nb_status, self.nb_actions, **net_cfg)
self.actor_optim = Adam(self.actor.parameters(), lr=args.prate)
self.critic = Critic(self.nb_status, self.nb_actions, **net_cfg)
self.critic_target = Critic(self.nb_status, self.nb_actions, **net_cfg)
self.critic_optim = Adam(self.critic.parameters(), lr=args.rate)
hard_update(self.actor_target,
self.actor) # Make sure target is with the same weight
hard_update(self.critic_target, self.critic)
if args.pic:
hard_update(self.cnn_target, self.cnn)
#Create replay buffer
self.memory = rpm(
args.rmsize
) # SequentialMemory(limit=args.rmsize, window_length=args.window_length)
self.random_process = Myrandom(size=nb_actions)
# Hyper-parameters
self.batch_size = args.batch_size
self.tau = args.tau
self.discount = args.discount
self.depsilon = 1.0 / args.epsilon
#
self.epsilon = 1.0
self.s_t = None # Most recent state
self.a_t = None # Most recent action
self.use_cuda = args.cuda
#
if self.use_cuda: self.cuda()
def normalize(self, pic):
pic = pic.swapaxes(0, 2).swapaxes(1, 2)
return pic
def update_policy(self):
# Sample batch
state_batch, action_batch, reward_batch, \
next_state_batch, terminal_batch = self.memory.sample_batch(self.batch_size)
# Prepare for the target q batch
if self.pic:
state_batch = np.array([self.normalize(x) for x in state_batch])
state_batch = to_tensor(state_batch, volatile=True)
state_batch = self.cnn(state_batch)
next_state_batch = np.array(
[self.normalize(x) for x in next_state_batch])
next_state_batch = to_tensor(next_state_batch, volatile=True)
next_state_batch = self.cnn_target(next_state_batch)
next_q_values = self.critic_target(
[next_state_batch,
self.actor_target(next_state_batch)])
else:
next_q_values = self.critic_target([
to_tensor(next_state_batch, volatile=True),
self.actor_target(to_tensor(next_state_batch, volatile=True)),
])
# print('batch of picture is ok')
next_q_values.volatile = False
target_q_batch = to_tensor(reward_batch) + \
self.discount * to_tensor((1 - terminal_batch.astype(np.float))) * next_q_values
# Critic update
self.critic.zero_grad()
if self.pic: self.cnn.zero_grad()
if self.pic:
state_batch.volatile = False
q_batch = self.critic([state_batch, to_tensor(action_batch)])
else:
q_batch = self.critic(
[to_tensor(state_batch),
to_tensor(action_batch)])
# print(reward_batch, next_q_values*self.discount, target_q_batch, terminal_batch.astype(np.float))
value_loss = criterion(q_batch, target_q_batch)
value_loss.backward()
self.critic_optim.step()
if self.pic: self.cnn_optim.step()
self.actor.zero_grad()
if self.pic:
state_batch.volatile = False
policy_loss = -self.critic([state_batch, self.actor(state_batch)])
else:
policy_loss = -self.critic(
[to_tensor(state_batch),
self.actor(to_tensor(state_batch))])
policy_loss = policy_loss.mean()
policy_loss.backward()
if self.clip_actor_grad is not None:
torch.nn.utils.clip_grad_norm(self.actor.parameters(),
float(self.clip_actor_grad))
if self.writer != None:
mean_policy_grad = np.array(
np.mean([
np.linalg.norm(p.grad.data.cpu().numpy().ravel())
for p in self.actor.parameters()
]))
#print(mean_policy_grad)
self.writer.add_scalar('train/mean_policy_grad',
mean_policy_grad, self.select_time)
self.actor_optim.step()
# Target update
soft_update(self.actor_target, self.actor, self.tau)
soft_update(self.critic_target, self.critic, self.tau)
if self.pic:
soft_update(self.cnn_target, self.cnn, self.tau)
return -policy_loss, value_loss
def eval(self):
self.actor.eval()
self.actor_target.eval()
self.critic.eval()
self.critic_target.eval()
if (self.pic):
self.cnn.eval()
self.cnn_target.eval()
def train(self):
self.actor.train()
self.actor_target.train()
self.critic.train()
self.critic_target.train()
if (self.pic):
self.cnn.eval()
self.cnn_target.eval()
def cuda(self):
self.cnn.cuda()
self.cnn_target.cuda()
self.actor.cuda()
self.actor_target.cuda()
self.critic.cuda()
self.critic_target.cuda()
def observe(self, r_t, s_t1, done):
self.memory.append([self.s_t, self.a_t, r_t, s_t1, done])
self.s_t = s_t1
def random_action(self):
action = np.random.uniform(-1., 1., self.nb_actions)
action = np.concatenate((softmax(action[:84]), softmax(action[84:])))
self.a_t = action
if self.discrete:
return action.argmax()
else:
return action
def select_action(self,
s_t,
decay_epsilon=True,
return_fix=False,
noise_level=0):
self.eval()
if self.pic:
s_t = self.normalize(s_t)
s_t = self.cnn(to_tensor(np.array([s_t])))
if self.pic:
action = to_numpy(self.actor_target(s_t)).squeeze(0)
else:
action = to_numpy(self.actor(to_tensor(np.array([s_t
])))).squeeze(0)
self.train()
noise_level = noise_level * max(self.epsilon, 0)
if np.random.uniform(0, 1) < noise_level:
action = self.random_process.sample() # episilon greedy
if decay_epsilon:
self.epsilon -= self.depsilon
self.a_t = action
if return_fix:
return action
if self.discrete:
return action.argmax()
else:
return action
def reset(self, obs):
self.s_t = obs
self.random_process.reset_status()
def load_weights(self, output, num=1):
if output is None: return
self.actor.load_state_dict(
torch.load('{}/actor{}.pkl'.format(output, num)))
self.actor_target.load_state_dict(
torch.load('{}/actor{}.pkl'.format(output, num)))
self.critic.load_state_dict(
torch.load('{}/critic{}.pkl'.format(output, num)))
self.critic_target.load_state_dict(
torch.load('{}/critic{}.pkl'.format(output, num)))
def save_model(self, output, num):
if self.use_cuda:
self.cnn.cpu()
self.cnn_target.cpu()
self.actor.cpu()
self.critic.cpu()
torch.save(self.actor.state_dict(),
'{}/actor{}.pkl'.format(output, num))
torch.save(self.critic.state_dict(),
'{}/critic{}.pkl'.format(output, num))
if self.use_cuda:
self.cnn.cuda()
self.cnn_target.cuda()
self.actor.cuda()
self.critic.cuda()
class Agent():
"""Interacts with and learns from the environment."""
def __init__(self, state_size, action_size, random_seed):
"""Initialize an Agent object.
Params
======
state_size (int): dimension of each state
action_size (int): dimension of each action
random_seed (int): random seed
"""
self.state_size = state_size
self.action_size = action_size
self.seed = random.seed(random_seed)
self.gradient_clip = GRADIENT_CLIP
# Actor Network (w/ Target Network)
self.actor_local = Actor(state_size,
action_size,
random_seed,
fc1_units=ACTOR_FC1_UNITS,
fc2_units=ACTOR_FC2_UNITS).to(device)
self.actor_target = Actor(state_size,
action_size,
random_seed,
fc1_units=ACTOR_FC1_UNITS,
fc2_units=ACTOR_FC2_UNITS).to(device)
self.actor_optimizer = optim.Adam(self.actor_local.parameters(),
lr=LR_ACTOR)
# Critic Network (w/ Target Network)
self.critic_local = Critic(state_size,
action_size,
random_seed,
fcs1_units=CRITIC_FCS1_UNITS,
fc2_units=CRITIC_FC2_UNITS).to(device)
self.critic_target = Critic(state_size,
action_size,
random_seed,
fcs1_units=CRITIC_FCS1_UNITS,
fc2_units=CRITIC_FC2_UNITS).to(device)
self.critic_optimizer = optim.Adam(self.critic_local.parameters(),
lr=LR_CRITIC,
weight_decay=WEIGHT_DECAY)
# Copy weights to the target networks
self.soft_update(self.critic_local, self.critic_target, 1)
self.soft_update(self.actor_local, self.actor_target, 1)
# Noise process
self.noise = OUNoiseWrapper(action_size, random_seed, NUM_AGENTS)
# self.noise = OUNoise(action_size, random_seed)
self.noise_factor = 1
self.noise_decay = NOISE_DECAY
# Replay memory
self.memory = ReplayBuffer(action_size, BUFFER_SIZE, BATCH_SIZE,
random_seed)
# Initialize time step (for updating every UPDATE_EVERY steps)
self.t_step = 0
def step(self, state, action, reward, next_state, done):
"""Save experience in replay memory, and use random sample from buffer to learn."""
# Save experience / reward
self.memory.add(state, action, reward, next_state, done)
# Learn every UPDATE_EVERY time steps.
self.t_step = (self.t_step + 1) % UPDATE_EVERY
if self.t_step == 0:
self.noise_decay *= self.noise_decay
# If enough samples are available in memory, get random subset and learn
# Learn, if enough samples are available in memory
if len(self.memory) > BATCH_SIZE:
for _ in range(LEARN_PASS):
experiences = self.memory.sample()
self.learn(experiences, GAMMA)
def act(self, state, add_noise=True):
"""Returns actions for given state as per current policy."""
state = torch.from_numpy(state).float().to(device)
self.actor_local.eval()
with torch.no_grad():
action = self.actor_local(state).cpu().data.numpy()
self.actor_local.train()
if add_noise:
action += self.noise_factor * self.noise.sample()
self.noise_factor *= self.noise_decay
return np.clip(action, -1, 1)
def reset(self):
self.noise.reset()
def learn(self, experiences, gamma):
"""Update policy and value parameters using given batch of experience tuples.
Q_targets = r + γ * critic_target(next_state, actor_target(next_state))
where:
actor_target(state) -> action
critic_target(state, action) -> Q-value
Params
======
experiences (Tuple[torch.Tensor]): tuple of (s, a, r, s', done) tuples
gamma (float): discount factor
"""
states, actions, rewards, next_states, dones = experiences
# ---------------------------- update critic ---------------------------- #
# Get predicted next-state actions and Q values from target models
actions_next = self.actor_target(next_states)
Q_targets_next = self.critic_target(next_states, actions_next)
# Compute Q targets for current states (y_i)
Q_targets = rewards + (gamma * Q_targets_next * (1 - dones))
# Compute critic loss
Q_expected = self.critic_local(states, actions)
critic_loss = F.mse_loss(Q_expected, Q_targets)
# Minimize the loss
self.critic_optimizer.zero_grad()
critic_loss.backward()
if self.gradient_clip:
torch.nn.utils.clip_grad_norm_(self.critic_local.parameters(), 1)
self.critic_optimizer.step()
# ---------------------------- update actor ---------------------------- #
# Compute actor loss
actions_pred = self.actor_local(states)
actor_loss = -self.critic_local(states, actions_pred).mean()
# Minimize the loss
self.actor_optimizer.zero_grad()
actor_loss.backward()
self.actor_optimizer.step()
# ----------------------- update target networks ----------------------- #
self.soft_update(self.critic_local, self.critic_target, TAU)
self.soft_update(self.actor_local, self.actor_target, TAU)
def soft_update(self, local_model, target_model, tau):
"""Soft update model parameters.
θ_target = τ*θ_local + (1 - τ)*θ_target
Params
======
local_model: PyTorch model (weights will be copied from)
target_model: PyTorch model (weights will be copied to)
tau (float): interpolation parameter
"""
for target_param, local_param in zip(target_model.parameters(),
local_model.parameters()):
target_param.data.copy_(tau * local_param.data +
(1.0 - tau) * target_param.data)
class Agent():
def __init__(self, state_size, action_size, random_seed):
self.state_size = state_size
self.action_size = action_size
self.seed = random.seed(random_seed)
# Actor Network (w/ Target Network)
self.actor_local = Actor(state_size, action_size,
random_seed).to(device)
self.actor_target = Actor(state_size, action_size,
random_seed).to(device)
self.actor_optimizer = optim.Adam(self.actor_local.parameters(),
lr=LR_ACTOR)
# Critic Network (w/ Target Network)
self.critic_local = Critic(state_size, action_size,
random_seed).to(device)
self.critic_target = Critic(state_size, action_size,
random_seed).to(device)
self.critic_optimizer = optim.Adam(self.critic_local.parameters(),
lr=LR_CRITIC,
weight_decay=WEIGHT_DECAY)
# Noise process
self.noise = OUNoise(action_size, random_seed)
# Replay memory
self.memory = ReplayBuffer(action_size, BUFFER_SIZE, BATCH_SIZE,
random_seed)
def step(self, state, action, reward, next_state, done, timestep):
# Save experience / reward
self.memory.add(state, action, reward, next_state, done)
# Learn at defined interval, if enough samples are available in memory
if len(self.memory) > BATCH_SIZE and timestep % LEARN_EVERY == 0:
for _ in range(N_LEARN_UPDATES):
experiences = self.memory.sample()
self.learn(experiences, GAMMA)
def act(self, state, add_noise=True):
"""Returns actions for given state as per current policy."""
state = torch.from_numpy(state).float().to(device)
self.actor_local.eval()
with torch.no_grad():
action = self.actor_local(state).cpu().data.numpy()
self.actor_local.train()
if add_noise:
action += self.noise.sample()
return np.clip(action, -1, 1)
def reset(self):
self.noise.reset()
def learn(self, experiences, gamma):
states, actions, rewards, next_states, dones = experiences
# ---------------------------- update critic ---------------------------- #
# Get predicted next-state actions and Q values from target models
actions_next = self.actor_target(next_states)
Q_targets_next = self.critic_target(next_states, actions_next)
# Compute Q targets for current states (y_i)
Q_targets = rewards + (gamma * Q_targets_next * (1 - dones))
# Compute critic loss
Q_expected = self.critic_local(states, actions)
critic_loss = F.mse_loss(Q_expected, Q_targets)
# Minimize the loss
self.critic_optimizer.zero_grad()
critic_loss.backward()
#torch.nn.utils.clip_grad_norm_(self.critic_local.parameters(), 1)
self.critic_optimizer.step()
# ---------------------------- update actor ---------------------------- #
# Compute actor loss
actions_pred = self.actor_local(states)
actor_loss = -self.critic_local(states, actions_pred).mean()
# Minimize the loss
self.actor_optimizer.zero_grad()
actor_loss.backward()
self.actor_optimizer.step()
# ----------------------- update target networks ----------------------- #
self.soft_update(self.critic_local, self.critic_target, TAU)
self.soft_update(self.actor_local, self.actor_target, TAU)
# ---------------------------- update noise ---------------------------- #
self.noise.reset()
def soft_update(self, local_model, target_model, tau):
for target_param, local_param in zip(target_model.parameters(),
local_model.parameters()):
target_param.data.copy_(tau * local_param.data +
(1.0 - tau) * target_param.data)
class Agent():
"""Interacts with and learns from the environment."""
#critic_local = None
#critic_target = None
#critic_optimizer = None
def __init__(self, state_size, action_size, num_agents, random_seed):
"""Initialize an Agent object.
Params
======
state_size (int): dimension of each state
action_size (int): dimension of each action
num_agents (int): number of agents
random_seed (int): random seed
"""
self.state_size = state_size
self.action_size = action_size
self.num_agents = num_agents
self.random_seed = random_seed
self.eps = eps_start
self.t_step = 0
# Actor Network (w/ Target Network)
self.actor_local = Actor(state_size, action_size,
random_seed).to(device)
self.actor_target = Actor(state_size, action_size,
random_seed).to(device)
self.actor_optimizer = optim.Adam(self.actor_local.parameters(),
lr=LR_ACTOR)
# Critic Network (w/ Target Network)
self.critic_local = Critic(self.state_size, self.action_size,
self.random_seed).to(device)
self.critic_target = Critic(self.state_size, self.action_size,
self.random_seed).to(device)
self.critic_optimizer = optim.Adam(self.critic_local.parameters(),
lr=LR_CRITIC,
weight_decay=WEIGHT_DECAY)
# Noise process
self.noise = OUNoise((num_agents, action_size), random_seed)
# Replay memory
self.memory = ReplayBuffer(action_size, BUFFER_SIZE, BATCH_SIZE,
random_seed)
def step(self, state, action, reward, next_state, done, agent_number):
"""Save experience in replay memory, and use random sample from buffer to learn."""
self.t_step += 1
# Save experience / reward
self.memory.add(state, action, reward, next_state, done)
# Learn, if enough samples are available in memory
if len(self.memory) > BATCH_SIZE:
if self.t_step % UPDATE_EVERY == 0:
for _ in range(N_UPDATES):
experiences = self.memory.sample()
self.learn(experiences, GAMMA, agent_number)
def act(self, states, add_noise=True):
"""Returns actions for given state as per current policy."""
states = torch.from_numpy(states).float().to(device)
actions = np.zeros((self.num_agents, self.action_size))
self.actor_local.eval()
with torch.no_grad():
for agent_num, state in enumerate(states):
action = self.actor_local(state).cpu().data.numpy()
actions[agent_num, :] = action
self.actor_local.train()
if add_noise:
actions += self.noise.sample()
return np.clip(actions, -1, 1)
def reset(self):
self.noise.reset()
def learn(self, experiences, gamma, agent_number):
"""Update policy and value parameters using given batch of experience tuples.
Q_targets = r + γ * critic_target(next_state, actor_target(next_state))
where:
actor_target(state) -> action
critic_target(state, action) -> Q-value
Params
======
experiences (Tuple[torch.Tensor]): tuple of (s, a, r, s', done) tuples
gamma (float): discount factor
"""
states, actions, rewards, next_states, dones = experiences
# ---------------------------- update critic ---------------------------- #
# Get predicted next-state actions and Q values from target models
actions_next = self.actor_target(next_states)
if agent_number == 0:
actions_next = torch.cat((actions_next, actions[:, 2:]), dim=1)
else:
actions_next = torch.cat((actions[:, :2], actions_next), dim=1)
Q_targets_next = self.critic_target(next_states, actions_next)
# Compute Q targets for current states (y_i)
Q_targets = rewards + (gamma * Q_targets_next * (1 - dones))
# Compute critic loss
Q_expected = self.critic_local(states, actions)
critic_loss = F.mse_loss(Q_expected, Q_targets)
# Minimize the loss
self.critic_optimizer.zero_grad()
critic_loss.backward()
self.critic_optimizer.step()
# ---------------------------- update actor ---------------------------- #
# Compute actor loss
actions_pred = self.actor_local(states)
if agent_number == 0:
actions_pred = torch.cat((actions_pred, actions[:, 2:]), dim=1)
else:
actions_pred = torch.cat((actions[:, :2], actions_pred), dim=1)
actor_loss = -self.critic_local(states, actions_pred).mean()
# Minimize the loss
self.actor_optimizer.zero_grad()
actor_loss.backward()
self.actor_optimizer.step()
# ----------------------- update target networks ----------------------- #
self.soft_update(self.critic_local, self.critic_target, TAU)
self.soft_update(self.actor_local, self.actor_target, TAU)
# Update epsilon noise value
self.eps = self.eps - (1 / eps_decay)
if self.eps < eps_end:
self.eps = eps_end
def soft_update(self, local_model, target_model, tau):
"""Soft update model parameters.
θ_target = τ*θ_local + (1 - τ)*θ_target
Params
======
local_model: PyTorch model (weights will be copied from)
target_model: PyTorch model (weights will be copied to)
tau (float): interpolation parameter
"""
for target_param, local_param in zip(target_model.parameters(),
local_model.parameters()):
target_param.data.copy_(tau * local_param.data +
(1.0 - tau) * target_param.data)
n_critic_updates = 5 # N critic updates per generator update
lp_coeff = 10 # Lipschitz penalty coefficient
train_batch_size = 64
test_batch_size = 64
lr = 1e-4
beta1 = 0.5
beta2 = 0.9
z_dim = 100
log_every = 500
save_images_every = 5000
train_loader, _, _ = svhn_sampler(data_root, train_batch_size,
test_batch_size)
generator = Generator(z_dim=z_dim).to(device)
critic = Critic().to(device)
optim_critic = optim.Adam(critic.parameters(), lr=lr, betas=(beta1, beta2))
optim_generator = optim.Adam(generator.parameters(),
lr=lr,
betas=(beta1, beta2))
# Define dataloader
dataloader_iter = iter(cycle(train_loader))
### TRAINING LOOP ###
loss_critic_cum = 0
loss_generator_cum = 0
for i in range(n_iter * n_critic_updates):
########### UPDATE CRITIC - every 1 iteration ###########
class DDPG:
def __init__(self, actor_state_size, actor_action_size, critic_state_size, critic_action_size, **kwargs):
if 'filename' in kwargs.keys():
data= torch.load(kwargs['filename'])
self.config= data["config"]
self.scores= data["scores"]
elif 'config' in kwargs.keys():
self.config= kwargs['config']
data= {}
self.scores= []
else:
raise OSError('DDPG: no configuration parameter in class init')
self.actor_state_size = actor_state_size
self.actor_action_size = actor_action_size
self.critic_state_size = critic_state_size
self.critic_action_size = critic_action_size
memory_size = self.config.get("memory_size", 100000)
actor_lr = self.config.get("actor_lr", 1e-3)
critic_lr = self.config.get("critic_lr", 1e-3)
self.batch_size = self.config.get("batch_size", 256)
self.discount = self.config.get("discount", 0.9)
sigma = self.config.get("sigma", 0.2)
self.tau= self.config.get("tau", 0.001)
self.seed = self.config.get("seed", 0)
self.action_noise= self.config.get("action_noise", "No")
self.critic_l2_reg= self.config.get("critic_l2_reg", 0.0)
random.seed(self.seed)
torch.manual_seed(self.seed)
param_noise= False
if self.action_noise== "Param": param_noise= True
self.actor = Actor(actor_state_size, actor_action_size, nodes= self.config["actor_nodes"], seed= self.seed, param_noise= param_noise).to(device)
self.critic = Critic(critic_state_size, critic_action_size, nodes= self.config["critic_nodes"], seed= self.seed).to(device)
self.targetActor = Actor(actor_state_size, actor_action_size, nodes= self.config["actor_nodes"], seed= self.seed, param_noise= param_noise).to(device)
self.targetCritic = Critic(critic_state_size, critic_action_size, nodes= self.config["critic_nodes"], seed= self.seed).to(device)
# Initialize parameters
self.hard_update(self.actor, self.targetActor)
self.hard_update(self.critic, self.targetCritic)
self.actor_optimizer = optim.Adam(self.actor.parameters(), lr= actor_lr)
self.critic_optimizer = optim.Adam(self.critic.parameters(), lr= critic_lr, weight_decay= self.critic_l2_reg)
self.criticLoss = nn.MSELoss() #nn.SmoothL1Loss()
#self.criticLoss = nn.SmoothL1Loss()
#self.noise= None
self.noise = NoNoise()
if self.action_noise== "OU":
self.noise = OUNoise(np.zeros(actor_action_size), sigma= sigma)
elif self.action_noise== "No":
self.noise = NoNoise()
elif self.action_noise== "Normal":
self.noise = NormalActionNoise(np.zeros(actor_action_size), sigma= sigma)
self.memory = Memory(memory_size, self.batch_size, self.seed)
def hard_update(self, source, target):
for target_param, param in zip(target.parameters(), source.parameters()):
target_param.data.copy_(param.data)
def soft_update(self, local_model, target_model, tau):
"""Soft update model parameters.
θ_target = τ*θ_local + (1 - τ)*θ_target
Params
======
local_model: PyTorch model (weights will be copied from)
target_model: PyTorch model (weights will be copied to)
tau (float): interpolation parameter
"""
for target_param, local_param in zip(target_model.parameters(), local_model.parameters()):
target_param.data.copy_(tau*local_param.data + (1.0-tau)*target_param.data)
def act(self, state, add_noise= True):
"""Returns actions for given state as per current policy."""
self.actor.resample()
#state = torch.from_numpy(state).float().to(device)
#state= torch.FloatTensor(state).view(1, -1).to(device)
#state= torch.FloatTensor(state).unsqueeze(0).to(device)
state= torch.FloatTensor(state).to(device)
if len(state.size())== 1:
state= state.unsqueeze(0)
self.actor.eval()
with torch.no_grad():
action = self.actor(state).cpu().data.numpy()
self.actor.train()
if add_noise and self.noise:
action += self.noise()
return np.clip(action, -1, 1)
def step(self, state, action, reward, next_state, done):
"""Save experience in replay memory, and use random sample from buffer to learn."""
self.memory.add((state, action, reward, next_state, done))
if len(self.memory) >= self.batch_size:
self.learn()
def learn_critic(self, states, actions, rewards, next_states, dones, actions_next):
# ---------------------------- update critic ---------------------------- #
# Get predicted next-state actions and Q values from target models
#actions_next = self.targetActor(next_states)
Q_targets_next = self.targetCritic(next_states, actions_next)
# Compute Q targets for current states (y_i)
Q_targets = rewards + (self.discount * Q_targets_next * (1 - dones))
Q_targets = Variable(Q_targets.data, requires_grad=False)
# Compute critic loss
Q_expected = self.critic(states, actions)
critic_loss = self.criticLoss(Q_expected, Q_targets)
# Minimize the loss
self.critic_optimizer.zero_grad()
critic_loss.backward()
self.critic_optimizer.step()
# ----------------------- update target networks ----------------------- #
self.soft_update(self.critic, self.targetCritic, self.tau)
#def learn_actor(self, states, actions, rewards, next_states, dones, actions_pred):
def learn_actor(self, states, actions_pred):
# ---------------------------- update actor ---------------------------- #
# Compute actor loss
#actions_pred = self.actor(states)
actor_loss = -self.critic(states, actions_pred).mean()
# Minimize the loss
self.actor_optimizer.zero_grad()
actor_loss.backward()
self.actor_optimizer.step()
# ----------------------- update target networks ----------------------- #
self.soft_update(self.actor, self.targetActor, self.tau)
def learn(self):
states, actions, rewards, next_states, dones = self.memory.sample()
self.learn_critic(states, actions, rewards, next_states, dones, self.targetActor(next_states))
self.learn_actor( states, self.actor(states))
def reset(self):
self.noise.reset()
def update(self, score= None):
if score: self.scores.append(score)
def save(self, filename= None):
data= {"config": self.config, "actor": self.actor.state_dict(), "scores": self.scores,}
if not filename:
filename= self.__class__.__name__+ '_'+ datetime.now().strftime("%Y-%m-%d_%H:%M:%S")+ '.data'
torch.save(data, filename)
torch.save(self.actor.state_dict(), "last_actor.pth")
class DDPGAgent:
def __init__(self, state_size, action_size, random_seed):
self.state_size = state_size
self.action_size = action_size
self.seed = random_seed
# ------------------ actor ------------------ #
self.actor_local = Actor(state_size, action_size,
random_seed).to(device)
self.actor_target = Actor(state_size, action_size,
random_seed).to(device)
# ------------------ critic ----------------- #
self.critic_local = Critic(state_size, action_size,
random_seed).to(device)
self.critic_target = Critic(state_size, action_size,
random_seed).to(device)
# ------------------ optimizers ------------- #
self.actor_optimizer = optim.Adam(self.actor_local.parameters(),
lr=LR_ACTOR)
self.critic_optimizer = optim.Adam(self.critic_local.parameters(),
lr=LR_CRITIC)
# ----------------------- initialize target networks ----------------------- #
self.soft_update(self.critic_local, self.critic_target, 1)
self.soft_update(self.actor_local, self.actor_target, 1)
# Noise process
self.noise = OUNoise(action_size, random_seed)
def act(self, state, add_noise=True):
"""Returns actions for given state as per current policy."""
state = torch.from_numpy(state).float().to(device)
self.actor_local.eval()
with torch.no_grad():
action = self.actor_local(state).cpu().data.numpy()
self.actor_local.train()
if add_noise:
action += self.noise.sample()
return np.clip(action, -1, 1)
def reset(self):
self.noise.reset()
def learn(self, experiences, gamma):
states, actions, rewards, next_states, dones = experiences
# ---------------------------- update critic ---------------------------- #
# Get predicted next-state actions and Q values from target models
actions_next = self.actor_target(next_states)
Q_targets_next = self.critic_target(next_states, actions_next)
# Compute Q targets for current states (y_i)
Q_targets = rewards + (gamma * Q_targets_next * (1 - dones))
# Compute critic loss
Q_expected = self.critic_local(states, actions)
critic_loss = F.mse_loss(Q_expected, Q_targets)
# Minimize the loss
self.critic_optimizer.zero_grad()
critic_loss.backward()
torch.nn.utils.clip_grad_norm_(self.critic_local.parameters(), 1)
self.critic_optimizer.step()
# ---------------------------- update actor ---------------------------- #
# Compute actor loss
actions_pred = self.actor_local(states)
actor_loss = -self.critic_local(states, actions_pred).mean()
# Minimize the loss
self.actor_optimizer.zero_grad()
actor_loss.backward()
self.actor_optimizer.step()
# ----------------------- update target networks ----------------------- #
self.soft_update(self.critic_local, self.critic_target, TAU)
self.soft_update(self.actor_local, self.actor_target, TAU)
def soft_update(self, local_model, target_model, tau):
"""Soft update model parameters.
θ_target = τ*θ_local + (1 - τ)*θ_target
Params
======
local_model: PyTorch model (weights will be copied from)
target_model: PyTorch model (weights will be copied to)
tau (float): interpolation parameter
"""
for target_param, local_param in zip(target_model.parameters(),
local_model.parameters()):
target_param.data.copy_(tau * local_param.data +
(1.0 - tau) * target_param.data)
class Agent():
"""Interacts with and learns from the environment."""
def __init__(self, state_size, action_size, n_agents, random_seed):
"""Initialize an Agent object.
Params
======
state_size (int): dimension of each state
action_size (int): dimension of each action
random_seed (int): random seed
"""
self.state_size = state_size
self.action_size = action_size
self.n_agents = n_agents
self.seed = random.seed(random_seed)
print("LR_Actor", LR_ACTOR)
# Actor Network (w/ Target Network)
self.actor_local = Actor(state_size, action_size, random_seed).to(device)
self.actor_target = Actor(state_size, action_size, random_seed).to(device)
self.actor_optimizer = optim.Adam(self.actor_local.parameters(), lr=LR_ACTOR)
# Critic Network (w/ Target Network)
self.critic_local = Critic(state_size, action_size, random_seed).to(device)
self.critic_target = Critic(state_size, action_size, random_seed).to(device)
self.critic_optimizer = optim.Adam(self.critic_local.parameters(), lr=LR_CRITIC, weight_decay=WEIGHT_DECAY)
# Noise process
#self.noise = OUNoise(action_size, random_seed)
self.noise = OUNoise((n_agents, action_size), random_seed)
# Replay memory
self.memory = ReplayBuffer(action_size, BUFFER_SIZE, BATCH_SIZE, random_seed)
# update back prop time
def step(self, state, action, reward, next_state, done, upate_backprop_time):
"""Save experience in replay memory, and use random sample from buffer to learn."""
# Save experience / reward
for i in range(self.n_agents):
self.memory.add(state[i], action[i], reward[i], next_state[i], done[i])
#self.memory.add(state, action, reward, next_state, done)
# self.upate_backprop_time += 1
# print("upate_backprop_time:", upate_backprop_time)
# Learn, if enough samples are available in memory
if len(self.memory) > BATCH_SIZE:# and upate_backprop_time%20==0:
experiences = self.memory.sample()
self.learn(experiences, GAMMA)
#print("Update! upate_backprop_time:", upate_backprop_time)
def act(self, state, add_noise=True):
"""Returns actions for given state as per current policy."""
state = torch.from_numpy(state).float().to(device)
self.actor_local.eval()
with torch.no_grad():
action = self.actor_local(state).cpu().data.numpy()
self.actor_local.train()
if add_noise:
action += self.noise.sample()
return np.clip(action, -1, 1)
def reset(self):
self.noise.reset()
def learn(self, experiences, gamma):
"""Update policy and value parameters using given batch of experience tuples.
Q_targets = r + γ * critic_target(next_state, actor_target(next_state))
where:
actor_target(state) -> action
critic_target(state, action) -> Q-value
Params
======
experiences (Tuple[torch.Tensor]): tuple of (s, a, r, s', done) tuples
gamma (float): discount factor
"""
states, actions, rewards, next_states, dones = experiences
# ---------------------------- update critic ---------------------------- #
# Get predicted next-state actions and Q values from target models
actions_next = self.actor_target(next_states)
Q_targets_next = self.critic_target(next_states, actions_next)
# Compute Q targets for current states (y_i)
Q_targets = rewards + (gamma * Q_targets_next * (1 - dones))
# Compute critic loss
Q_expected = self.critic_local(states, actions)
critic_loss = F.mse_loss(Q_expected, Q_targets)
# Minimize the loss
self.critic_optimizer.zero_grad()
critic_loss.backward()
self.critic_optimizer.step()
# ---------------------------- update actor ---------------------------- #
# Compute actor loss
actions_pred = self.actor_local(states)
actor_loss = -self.critic_local(states, actions_pred).mean()
# Minimize the loss
self.actor_optimizer.zero_grad()
actor_loss.backward()
self.actor_optimizer.step()
# ----------------------- update target networks ----------------------- #
self.soft_update(self.critic_local, self.critic_target, TAU)
self.soft_update(self.actor_local, self.actor_target, TAU)
def soft_update(self, local_model, target_model, tau):
"""Soft update model parameters.
θ_target = τ*θ_local + (1 - τ)*θ_target
Params
======
local_model: PyTorch model (weights will be copied from)
target_model: PyTorch model (weights will be copied to)
tau (float): interpolation parameter
"""
for target_param, local_param in zip(target_model.parameters(), local_model.parameters()):
target_param.data.copy_(tau*local_param.data + (1.0-tau)*target_param.data)
class DDPG(object):
def __init__(self, nb_status, nb_actions, args, writer):
self.clip_actor_grad = args.clip_actor_grad
self.nb_status = nb_status * args.window_length
self.nb_actions = nb_actions
self.discrete = args.discrete
self.pic = args.pic
self.writer = writer
self.select_time = 0
if self.pic:
self.nb_status = args.pic_status
# Create Actor and Critic Network
net_cfg = {
'hidden1':args.hidden1,
'hidden2':args.hidden2,
'use_bn':args.bn,
'init_method':args.init_method
}
if args.pic:
self.cnn = CNN(1, args.pic_status)
self.cnn_target = CNN(1, args.pic_status)
self.cnn_optim = Adam(self.cnn.parameters(), lr=args.crate)
self.actor = Actor(self.nb_status, self.nb_actions, **net_cfg)
self.actor_target = Actor(self.nb_status, self.nb_actions, **net_cfg)
self.actor_optim = Adam(self.actor.parameters(), lr=args.prate)
self.critic = Critic(self.nb_status, self.nb_actions, **net_cfg)
self.critic_target = Critic(self.nb_status, self.nb_actions, **net_cfg)
self.critic_optim = Adam(self.critic.parameters(), lr=args.rate)
hard_update(self.actor_target, self.actor) # Make sure target is with the same weight
hard_update(self.critic_target, self.critic)
if args.pic:
hard_update(self.cnn_target, self.cnn)
#Create replay buffer
self.memory = rpm(args.rmsize) # SequentialMemory(limit=args.rmsize, window_length=args.window_length)
self.random_process = Myrandom(size=nb_actions)
# Hyper-parameters
self.batch_size = args.batch_size
self.tau = args.tau
self.discount = args.discount
self.depsilon = 1.0 / args.epsilon
#
self.epsilon = 1.0
self.s_t = None # Most recent state
self.a_t = None # Most recent action
self.use_cuda = args.cuda
#
if self.use_cuda: self.cuda()
def normalize(self, pic):
pic = pic.swapaxes(0, 2).swapaxes(1, 2)
return pic
def update_policy(self):
# Sample batch
state_batch, action_batch, reward_batch, \
next_state_batch, terminal_batch = self.memory.sample_batch(self.batch_size)
# Prepare for the target q batch
if self.pic:
state_batch = np.array([self.normalize(x) for x in state_batch])
state_batch = to_tensor(state_batch, volatile=True)
state_batch = self.cnn(state_batch)
next_state_batch = np.array([self.normalize(x) for x in next_state_batch])
next_state_batch = to_tensor(next_state_batch, volatile=True)
next_state_batch = self.cnn_target(next_state_batch)
next_q_values = self.critic_target([
next_state_batch,
self.actor_target(next_state_batch)
])
else:
next_q_values = self.critic_target([
to_tensor(next_state_batch, volatile=True),
self.actor_target(to_tensor(next_state_batch, volatile=True)),
])
# print('batch of picture is ok')
next_q_values.volatile = False
target_q_batch = to_tensor(reward_batch) + \
self.discount * to_tensor((1 - terminal_batch.astype(np.float))) * next_q_values
# Critic update
self.critic.zero_grad()
if self.pic: self.cnn.zero_grad()
if self.pic:
state_batch.volatile = False
q_batch = self.critic([state_batch, to_tensor(action_batch)])
else:
q_batch = self.critic([to_tensor(state_batch), to_tensor(action_batch)])
# print(reward_batch, next_q_values*self.discount, target_q_batch, terminal_batch.astype(np.float))
value_loss = criterion(q_batch, target_q_batch)
value_loss.backward()
self.critic_optim.step()
if self.pic: self.cnn_optim.step()
self.actor.zero_grad()
if self.pic: self.cnn.zero_grad()
if self.pic:
state_batch.volatile = False
policy_loss = -self.critic([
state_batch,
self.actor(state_batch)
])
else:
policy_loss = -self.critic([
to_tensor(state_batch),
self.actor(to_tensor(state_batch))
])
policy_loss = policy_loss.mean()
policy_loss.backward()
if self.clip_actor_grad is not None:
torch.nn.utils.clip_grad_norm(self.actor.parameters(), float(self.clip_actor_grad))
if self.writer != None:
mean_policy_grad = np.array(np.mean([np.linalg.norm(p.grad.data.cpu().numpy().ravel()) for p in self.actor.parameters()]))
#print(mean_policy_grad)
self.writer.add_scalar('train/mean_policy_grad', mean_policy_grad, self.select_time)
self.actor_optim.step()
if self.pic: self.cnn_optim.step()
# Target update
soft_update(self.actor_target, self.actor, self.tau)
soft_update(self.critic_target, self.critic, self.tau)
if self.pic:
soft_update(self.cnn_target, self.cnn, self.tau)
return -policy_loss, value_loss
def eval(self):
self.actor.eval()
self.actor_target.eval()
self.critic.eval()
self.critic_target.eval()
if(self.pic):
self.cnn.eval()
self.cnn_target.eval()
def train(self):
self.actor.train()
self.actor_target.train()
self.critic.train()
self.critic_target.train()
if(self.pic):
self.cnn.train()
self.cnn_target.train()
def cuda(self):
self.cnn.cuda()
self.cnn_target.cuda()
self.actor.cuda()
self.actor_target.cuda()
self.critic.cuda()
self.critic_target.cuda()
def observe(self, r_t, s_t1, done):
self.memory.append([self.s_t, self.a_t, r_t, s_t1, done])
self.s_t = s_t1
def random_action(self, fix=False):
action = np.random.uniform(-1.,1.,self.nb_actions)
self.a_t = action
if self.discrete and fix == False:
action = action.argmax()
# if self.pic:
# action = np.concatenate((softmax(action[:16]), softmax(action[16:])))
return action
def select_action(self, s_t, decay_epsilon=True, return_fix=False, noise_level=0):
self.eval()
if self.pic:
s_t = self.normalize(s_t)
s_t = self.cnn(to_tensor(np.array([s_t])))
if self.pic:
action = to_numpy(
self.actor_target(s_t)
).squeeze(0)
else:
action = to_numpy(
self.actor(to_tensor(np.array([s_t])))
).squeeze(0)
self.train()
noise_level = noise_level * max(self.epsilon, 0)
if np.random.uniform(0, 1) < noise_level:
action = self.random_action(fix=True) # episilon greedy
if decay_epsilon:
self.epsilon -= self.depsilon
self.a_t = action
if return_fix:
return action
if self.discrete:
return action.argmax()
else:
return action
def reset(self, obs):
self.s_t = obs
self.random_process.reset_status()
def load_weights(self, output, num=1):
if output is None: return
self.actor.load_state_dict(
torch.load('{}/actor{}.pkl'.format(output, num))
)
self.actor_target.load_state_dict(
torch.load('{}/actor{}.pkl'.format(output, num))
)
self.critic.load_state_dict(
torch.load('{}/critic{}.pkl'.format(output, num))
)
self.critic_target.load_state_dict(
torch.load('{}/critic{}.pkl'.format(output, num))
)
def save_model(self, output, num):
if self.use_cuda:
self.cnn.cpu()
self.actor.cpu()
self.critic.cpu()
torch.save(
self.actor.state_dict(),
'{}/actor{}.pkl'.format(output, num)
)
torch.save(
self.critic.state_dict(),
'{}/critic{}.pkl'.format(output, num)
)
if self.use_cuda:
self.cnn.cuda()
self.actor.cuda()
self.critic.cuda()
def main():
env = gym.make(args.env_name)
env.seed(args.seed)
torch.manual_seed(args.seed)
num_inputs = env.observation_space.shape[0]
num_actions = env.action_space.shape[0]
running_state = ZFilter((num_inputs,), clip=5)
print('state size:', num_inputs)
print('action size:', num_actions)
actor = Actor(num_inputs, num_actions, args)
critic = Critic(num_inputs, args)
discrim = Discriminator(num_inputs + num_actions, args)
actor_optim = optim.Adam(actor.parameters(), lr=args.learning_rate)
critic_optim = optim.Adam(critic.parameters(), lr=args.learning_rate,
weight_decay=args.l2_rate)
discrim_optim = optim.Adam(discrim.parameters(), lr=args.learning_rate)
# load demonstrations
expert_demo, _ = pickle.load(open('./expert_demo/expert_demo.p', "rb"))
demonstrations = np.array(expert_demo)
print("demonstrations.shape", demonstrations.shape)
writer = SummaryWriter(args.logdir)
if args.load_model is not None:
saved_ckpt_path = os.path.join(os.getcwd(), 'save_model', str(args.load_model))
ckpt = torch.load(saved_ckpt_path)
actor.load_state_dict(ckpt['actor'])
critic.load_state_dict(ckpt['critic'])
discrim.load_state_dict(ckpt['discrim'])
running_state.rs.n = ckpt['z_filter_n']
running_state.rs.mean = ckpt['z_filter_m']
running_state.rs.sum_square = ckpt['z_filter_s']
print("Loaded OK ex. Zfilter N {}".format(running_state.rs.n))
episodes = 0
train_discrim_flag = True
for iter in range(args.max_iter_num):
actor.eval(), critic.eval()
memory = deque()
steps = 0
scores = []
while steps < args.total_sample_size:
state = env.reset()
score = 0
state = running_state(state)
for _ in range(10000):
if args.render:
env.render()
steps += 1
mu, std = actor(torch.Tensor(state).unsqueeze(0))
action = get_action(mu, std)[0]
next_state, reward, done, _ = env.step(action)
irl_reward = get_reward(discrim, state, action)
if done:
mask = 0
else:
mask = 1
memory.append([state, action, irl_reward, mask])
next_state = running_state(next_state)
state = next_state
score += reward
if done:
break
episodes += 1
scores.append(score)
score_avg = np.mean(scores)
print('{}:: {} episode score is {:.2f}'.format(iter, episodes, score_avg))
writer.add_scalar('log/score', float(score_avg), iter)
actor.train(), critic.train(), discrim.train()
if train_discrim_flag:
expert_acc, learner_acc = train_discrim(discrim, memory, discrim_optim, demonstrations, args)
print("Expert: %.2f%% | Learner: %.2f%%" % (expert_acc * 100, learner_acc * 100))
if expert_acc > args.suspend_accu_exp and learner_acc > args.suspend_accu_gen:
train_discrim_flag = False
train_actor_critic(actor, critic, memory, actor_optim, critic_optim, args)
if iter % 100:
score_avg = int(score_avg)
model_path = os.path.join(os.getcwd(),'save_model')
if not os.path.isdir(model_path):
os.makedirs(model_path)
ckpt_path = os.path.join(model_path, 'ckpt_'+ str(score_avg)+'.pth.tar')
save_checkpoint({
'actor': actor.state_dict(),
'critic': critic.state_dict(),
'discrim': discrim.state_dict(),
'z_filter_n':running_state.rs.n,
'z_filter_m': running_state.rs.mean,
'z_filter_s': running_state.rs.sum_square,
'args': args,
'score': score_avg
}, filename=ckpt_path)
class Agent():
"""Interacts with and learns from the environment."""
def __init__(self, params, device = DEVICE, critic_input_size = None):
"""Initialize an Agent object.
"""
self.params = params
self.state_size = params.STATE_SIZE
self.action_size = params.ACTION_SIZE
self.seed = params.SEED
self.tau = params.TAU
self.device = device
if critic_input_size is None:
critic_input_size = 2 * (self.state_size + self.action_size)
# Actor Network (w/ Target Network)
self.actor_local = Actor(self.state_size, self.action_size, self.seed).to(device)
self.actor_target = Actor(self.state_size, self.action_size, self.seed).to(device)
self.actor_optimizer = optim.Adam(self.actor_local.parameters(),
lr=params.LR_ACTOR, weight_decay=params.WEIGHT_DECAY_ACTOR)
# Critic Network (w/ Target Network)
self.critic_local = Critic(critic_input_size, self.seed).to(device)
self.critic_target = Critic(critic_input_size, self.seed).to(device)
self.critic_optimizer = optim.Adam(self.critic_local.parameters(),
lr=params.LR_CRITIC, weight_decay=params.WEIGHT_DECAY_CRITIC)
# Noise process
self.noise = OUNoise(self.action_size, self.seed,
mu=0., theta=params.NOISE_THETA, sigma=params.NOISE_SIGMA)
# Parameters for learning
self.gamma = params.GAMMA
self.learning_step = 0 # Counter for learning steps
def act(self, state, add_noise=False, sigma = 0.1):
"""
Returns actions for given state as per current policy.
Arguments:
state - input state
add_noise - can be:
False - No nose added (default)
'OU' - Ornstein-Uhlenbeck noise added
'rand' - uniformly random noise added
'sigma' - noise is scaled from -simga/2 to sigma/2. Works with 'rand' noise
"""
state = torch.from_numpy(state).float().to(self.device)
self.actor_local.eval()
with torch.no_grad():
action = self.actor_local(state).cpu().data.numpy()
self.actor_local.train()
if add_noise:
if add_noise == 'OU':
action += self.noise.sample()
else:
action += sigma * np.random.rand(len(action)) - sigma / 2
return np.clip(action, -1, 1) # Clipping is necessary if we are adding noise
else:
return action
def reset(self):
self.noise.reset()
def learn(self,
states, actions, rewards, next_states, dones,
next_actions,
ag2_states, ag2_actions, ag2_next_states,
ag2_next_actions):
"""Update policy and value parameters using given batch of experience tuples.
Q_targets = r + γ * critic_target(next_state, actor_target(next_state))
where:
actor_target(state) -> action
critic_target(state, action) -> Q-value
Params
======
states, actions, rewards, next_states, dones - parameters for agent
next_actions - actions produced by target network
ag2_states, ag2_actions, ag2_next_states - parameters for the other agent
ag2_next_actions - actions produced by target network of the other agent
"""
# ---------------------------- update critic ---------------------------- #
# Get predicted next-state actions and Q values from target models
with torch.no_grad():
Q_targets_next = self.critic_target(next_states, next_actions, ag2_next_states, ag2_next_actions)
# Compute Q targets for current states
Q_targets = rewards + (self.gamma * Q_targets_next * (1 - dones))
# Compute critic loss
Q_expected = self.critic_local(states, actions, ag2_states, ag2_actions)
critic_loss = F.mse_loss(Q_expected, Q_targets)
# Minimize the loss
self.critic_optimizer.zero_grad()
critic_loss.backward()
self.critic_optimizer.step()
# ---------------------------- update actor ---------------------------- #
# Compute actor loss
pred_actions = self.actor_local(states)
actor_loss = -self.critic_local(states, pred_actions, ag2_states, ag2_next_actions).mean()
# Minimize the loss
self.actor_optimizer.zero_grad()
actor_loss.backward()
self.actor_optimizer.step()
# ----------------------- update target networks ----------------------- #
self.soft_update(self.critic_local, self.critic_target, self.tau)
self.soft_update(self.actor_local, self.actor_target, self.tau)
def soft_update(self, local_model, target_model, tau):
"""Soft update model parameters.
θ_target = τ*θ_local + (1 - τ)*θ_target
Params
======
local_model: PyTorch model (weights will be copied from)
target_model: PyTorch model (weights will be copied to)
tau (float): interpolation parameter
"""
for target_param, local_param in zip(target_model.parameters(), local_model.parameters()):
target_param.data.copy_(tau*local_param.data + (1.0-tau)*target_param.data)
class Agent():
"""Interacts with and learns from the environment."""
def __init__(self, state_size, action_size, random_seed):
"""Initialize an Agent object.
Params
======
state_size (int): dimension of each state
action_size (int): dimension of each action
random_seed (int): random seed
"""
self.state_size = state_size
self.action_size = action_size
self.seed = random.seed(random_seed)
# Actor Network (w/ Target Network)
self.actor_local = Actor(state_size, action_size, random_seed).to(device)
self.actor_target = Actor(state_size, action_size, random_seed).to(device)
self.actor_optimizer = optim.Adam(self.actor_local.parameters(), lr=LR_ACTOR)
# Critic Network (w/ Target Network)
self.critic_local = Critic(state_size, action_size, random_seed).to(device)
self.critic_target = Critic(state_size, action_size, random_seed).to(device)
self.critic_optimizer = optim.Adam(self.critic_local.parameters(), lr=LR_CRITIC, weight_decay=WEIGHT_DECAY)
# Noise process
self.noise = OUNoise(action_size, random_seed)
# Replay memory
self.memory = ReplayBuffer(action_size, BUFFER_SIZE, BATCH_SIZE, random_seed)
def step(self, state, action, reward, next_state, done):
"""Save experience in replay memory, and use random sample from buffer to learn."""
# Save experience / reward
self.memory.add(state, action, reward, next_state, done)
# Learn, if enough samples are available in memory
if len(self.memory) > BATCH_SIZE:
experiences = self.memory.sample()
self.learn(experiences, GAMMA)
def act(self, state, add_noise=True):
"""Returns actions for given state as per current policy."""
state = torch.from_numpy(state).float().unsqueeze(0).to(device)
self.actor_local.eval()
with torch.no_grad():
action = self.actor_local(state).cpu().data.numpy()
self.actor_local.train()
if add_noise:
action += self.noise.sample()
return np.clip(action, -1, 1)
def reset(self):
self.noise.reset()
def learn(self, experiences, gamma):
"""Update policy and value parameters using given batch of experience tuples.
Q_targets = r + γ * critic_target(next_state, actor_target(next_state))
where:
actor_target(state) -> action
critic_target(state, action) -> Q-value
Params
======
experiences (Tuple[torch.Tensor]): tuple of (s, a, r, s', done) tuples
gamma (float): discount factor
"""
states, actions, rewards, next_states, dones = experiences
# ---------------------------- update critic ---------------------------- #
# Get predicted next-state actions and Q values from target models
actions_next = self.actor_target(next_states)
Q_targets_next = self.critic_target(next_states, actions_next)
# Compute Q targets for current states (y_i)
Q_targets = rewards + (gamma * Q_targets_next * (1 - dones))
# Compute critic loss
Q_expected = self.critic_local(states, actions)
critic_loss = F.mse_loss(Q_expected, Q_targets)
# Minimize the loss
self.critic_optimizer.zero_grad()
critic_loss.backward()
self.critic_optimizer.step()
# ---------------------------- update actor ---------------------------- #
# Compute actor loss
actions_pred = self.actor_local(states)
actor_loss = -self.critic_local(states, actions_pred).mean()
# Minimize the loss
self.actor_optimizer.zero_grad()
actor_loss.backward()
self.actor_optimizer.step()
# ----------------------- update target networks ----------------------- #
self.soft_update(self.critic_local, self.critic_target, TAU)
self.soft_update(self.actor_local, self.actor_target, TAU)
def soft_update(self, local_model, target_model, tau):
"""Soft update model parameters.
θ_target = τ*θ_local + (1 - τ)*θ_target
Params
======
local_model: PyTorch model (weights will be copied from)
target_model: PyTorch model (weights will be copied to)
tau (float): interpolation parameter
"""
for target_param, local_param in zip(target_model.parameters(), local_model.parameters()):
target_param.data.copy_(tau*local_param.data + (1.0-tau)*target_param.data)
class Agent():
"""Interacts with and learns from the environment."""
def __init__(self, state_size, action_size,num_agents ,random_seed,OU_mu,OU_theta, OU_sigma, weight_decay=WEIGHT_DECAY, LR_actor=LR_ACTOR, LR_critic=LR_CRITIC, tau=TAU, gamma=GAMMA, noise_decay=NOISE_DECAY,noise_min=NOISE_MIN ):
"""Initialize an Agent object.
Params
======
state_size (int): dimension of each state
action_size (int): dimension of each action
random_seed (int): random seed
"""
self.state_size = state_size
self.action_size = action_size
self.seed = random.seed(random_seed)
# init local and target actor Networks
self.actor_local = Actor(state_size, action_size, random_seed).to(device)
self.actor_target = Actor(state_size, action_size, random_seed).to(device)
self.actor_optimizer = optim.Adam(self.actor_local.parameters(), lr=LR_actor)
# init local and target critic Networks
self.critic_local = Critic(state_size*num_agents, action_size*num_agents, random_seed).to(device)
self.critic_target = Critic(state_size*num_agents, action_size*num_agents, random_seed).to(device)
self.critic_optimizer = optim.Adam(self.critic_local.parameters(), lr=LR_critic, weight_decay=weight_decay)
# init params of noise process
self.noise = OUNoise(action_size, random_seed, mu=OU_mu, theta=OU_theta, sigma=OU_sigma)
self.noise_decay = noise_decay
self.noise_min = noise_min
self.step_count = 0
def act(self, state, i_episode, add_noise=True):
"""Uses policy to map states to action"""
state = torch.from_numpy(state).float().to(device)
self.actor_local.eval()
with torch.no_grad():
action = self.actor_local(state).cpu().data.numpy()
self.actor_local.train()
if add_noise:
action += max(self.noise_decay, self.noise_min )*self.noise.sample()
self.noise_decay*=self.noise_decay
return np.clip(action, -1, 1)
def act_inference(self, state):
"""Uses policy to map states to action( no grad accumulation and no noise)"""
state = torch.from_numpy(state).float().to(device)
self.actor_local.eval()
with torch.no_grad():
action = self.actor_local(state).cpu().data.numpy()
return np.clip(action, -1, 1)
def reset(self):
self.noise.reset()
def learn(self, experiences, gamma):
full_states, actions, actor_local_actions, actor_target_actions, agent_state, agent_action, agent_reward, agent_done, next_states, next_full_states = experiences
# ---------------------------- update critic ---------------------------- #
# Get predicted next-state actions and Q values from target models
# actions_next = self.actor_target(next_states)
Q_targets_next = self.critic_target(next_full_states, actor_target_actions)
# Compute Q targets for current states (y_i)
Q_targets = agent_reward + (gamma * Q_targets_next * (1 - agent_done))
# Compute critic loss
Q_expected = self.critic_local(full_states, actions)
critic_loss = F.mse_loss(Q_expected, Q_targets)
# Minimize the loss
self.critic_optimizer.zero_grad()
critic_loss.backward()
# torch.nn.utils.clip_grad_norm_(self.critic_local.parameters(), 1)
self.critic_optimizer.step()
# ---------------------------- update actor ---------------------------- #
actor_loss = -self.critic_local(full_states, actor_local_actions).mean()
# Minimize the loss
self.actor_optimizer.zero_grad()
actor_loss.backward()
self.actor_optimizer.step()
# ----------------------- update target networks ----------------------- #
self.soft_update(self.critic_local, self.critic_target, TAU)
self.soft_update(self.actor_local, self.actor_target, TAU)
def hard_copy_weights(self, target, source):
""" copy weights from source to target network (part of initialization)"""
for target_param, param in zip(target.parameters(), source.parameters()):
target_param.data.copy_(param.data)
def soft_update(self, local_model, target_model, tau):
"""Soft update model parameters.
θ_target = τ*θ_local + (1 - τ)*θ_target
Params
======
local_model: PyTorch model (weights will be copied from)
target_model: PyTorch model (weights will be copied to)
tau (float): interpolation parameter
"""
for target_param, local_param in zip(target_model.parameters(), local_model.parameters()):
target_param.data.copy_(tau*local_param.data + (1.0-tau)*target_param.data)
class DDPG(object):
def __init__(self, nb_status, nb_actions, args):
self.num_actor = 3
self.nb_status = nb_status * args.window_length
self.nb_actions = nb_actions
self.discrete = args.discrete
# Create Actor and Critic Network
net_cfg = {
'hidden1': args.hidden1,
'hidden2': args.hidden2,
'use_bn': args.bn
}
self.actors = [Actor(self.nb_status, self.nb_actions) for _ in range(self.num_actor)]
self.actor_targets = [Actor(self.nb_status, self.nb_actions) for _ in
range(self.num_actor)]
self.actor_optims = [Adam(self.actors[i].parameters(), lr=args.prate) for i in range(self.num_actor)]
self.critic = Critic(self.nb_status, self.nb_actions, **net_cfg)
self.critic_target = Critic(self.nb_status, self.nb_actions, **net_cfg)
self.critic_optim = Adam(self.critic.parameters(), lr=args.rate)
for i in range(self.num_actor):
hard_update(self.actor_targets[i], self.actors[i]) # Make sure target is with the same weight
hard_update(self.critic_target, self.critic)
# Create replay buffer
self.memory = rpm(args.rmsize) # SequentialMemory(limit=args.rmsize, window_length=args.window_length)
self.random_process = Myrandom(size=nb_actions)
# Hyper-parameters
self.batch_size = args.batch_size
self.tau = args.tau
self.discount = args.discount
self.depsilon = 1.0 / args.epsilon
#
self.epsilon = 1.0
self.s_t = None # Most recent state
self.a_t = None # Most recent action
self.use_cuda = args.cuda
#
if self.use_cuda: self.cuda()
def update_policy(self, train_actor=True):
# Sample batch
state_batch, action_batch, reward_batch, \
next_state_batch, terminal_batch = self.memory.sample_batch(self.batch_size)
# Prepare for the target q batch
next_q_values = 0
for i in range(self.num_actor):
next_q_values = next_q_values + self.critic_target([
to_tensor(next_state_batch, volatile=True),
self.actor_targets[i](to_tensor(next_state_batch, volatile=True)),
])
# print('batch of picture is ok')
next_q_values = next_q_values / self.num_actor
next_q_values.volatile = False
target_q_batch = to_tensor(reward_batch) + \
self.discount * to_tensor((1 - terminal_batch.astype(np.float))) * next_q_values
# Critic update
self.critic.zero_grad()
q_batch = self.critic([to_tensor(state_batch), to_tensor(action_batch)])
# print(reward_batch, next_q_values*self.discount, target_q_batch, terminal_batch.astype(np.float))
value_loss = criterion(q_batch, target_q_batch)
value_loss.backward()
self.critic_optim.step()
sum_policy_loss = 0
for i in range(self.num_actor):
self.actors[i].zero_grad()
policy_loss = -self.critic([
to_tensor(state_batch),
self.actors[i](to_tensor(state_batch))
])
policy_loss = policy_loss.mean()
policy_loss.backward()
if train_actor:
self.actor_optims[i].step()
sum_policy_loss += policy_loss
# Target update
soft_update(self.actor_targets[i], self.actors[i], self.tau)
soft_update(self.critic_target, self.critic, self.tau)
return -sum_policy_loss / self.num_actor, value_loss
def cuda(self):
for i in range(self.num_actor):
self.actors[i].cuda()
self.actor_targets[i].cuda()
self.critic.cuda()
self.critic_target.cuda()
def observe(self, r_t, s_t1, done):
self.memory.append([self.s_t, self.a_t, r_t, s_t1, done])
self.s_t = s_t1
def random_action(self):
action = np.random.uniform(-1., 1., self.nb_actions)
self.a_t = action
if self.discrete:
return action.argmax()
else:
return action
def select_action(self, s_t, decay_epsilon=True, return_fix=False, noise_level=0):
actions = []
status = []
tot_score = []
for i in range(self.num_actor):
action = to_numpy(self.actors[i](to_tensor(np.array([s_t]), volatile=True))).squeeze(0)
noise_level = noise_level * max(self.epsilon, 0)
action = action + self.random_process.sample() * noise_level
status.append(s_t)
actions.append(action)
tot_score.append(0.)
scores = self.critic([to_tensor(np.array(status), volatile=True), to_tensor(np.array(actions), volatile=True)])
for j in range(self.num_actor):
tot_score[j] += scores.data[j][0]
best = np.array(tot_score).argmax()
if decay_epsilon:
self.epsilon -= self.depsilon
self.a_t = actions[best]
return actions[best]
def reset(self, obs):
self.s_t = obs
self.random_process.reset_status()
def load_weights(self, output, num=0):
if output is None: return
for i in range(self.num_actor):
actor = self.actors[i]
actor_target = self.actor_targets[i]
actor.load_state_dict(
torch.load('{}/actor{}_{}.pkl'.format(output, num, i))
)
actor_target.load_state_dict(
torch.load('{}/actor{}_{}.pkl'.format(output, num, i))
)
self.critic.load_state_dict(
torch.load('{}/critic{}.pkl'.format(output, num))
)
self.critic_target.load_state_dict(
torch.load('{}/critic{}.pkl'.format(output, num))
)
def save_model(self, output, num):
if self.use_cuda:
for i in range(self.num_actor):
self.actors[i].cpu()
self.critic.cpu()
for i in range(self.num_actor):
torch.save(
self.actors[i].state_dict(),
'{}/actor{}_{}.pkl'.format(output, num, i)
)
torch.save(
self.critic.state_dict(),
'{}/critic{}.pkl'.format(output, num)
)
if self.use_cuda:
for i in range(self.num_actor):
self.actors[i].cuda()
self.critic.cuda()
class Agent():
"""Interacts with and learns from the environment"""
def __init__(self, state_size, action_size, random_seed=0):
"""Initialize an Agent object.
Params
======
state_size (int): dimension of each state
action_size (int): dimension of each action
random_seed (int): random seed
"""
self.state_size = state_size
self.action_size = action_size
self.seed = random.seed(random_seed)
self.epsilon = EPSILON
# Actor network
self.actor_local = Actor(state_size, action_size,
random_seed).to(device)
self.actor_target = Actor(state_size, action_size,
random_seed).to(device)
self.actor_optimizer = optim.Adam(self.actor_local.parameters(),
lr=LR_ACTOR)
# Critic network
self.critic_local = Critic(state_size, action_size,
random_seed).to(device)
self.critic_target = Critic(state_size, action_size,
random_seed).to(device)
self.critic_optimizer = optim.Adam(self.critic_local.parameters(),
lr=LR_CRITIC,
weight_decay=WEIGHT_DECAY)
# Noise process
self.noise = OUNoise(action_size, random_seed)
# Replay memory
self.memory = ReplayBuffer(action_size, BUFFER_SIZE, BATCH_SIZE,
random_seed)
def step(self, state, action, reward, next_state, done, timestep):
"""Save experience in replay memory, and use random sample from buffer to learn"""
# save experience/reward
# if updating in batches, then add the last memory of the agents(e.g. 20 agents) to a buffer
# and if we've met batch size, push to learn in multiples of LEARN_NUM
self.memory.add(state, action, reward, next_state, done)
# Learn, if enough samples are available in memory
if len(self.memory) > BATCH_SIZE and timestep % LEARN_EVERY == 0:
for _ in range(LEARN_NUM):
experiences = self.memory.sample()
self.learn(experiences, GAMMA)
def act(self, state, add_noise=True):
"""Returns actions for given state as per current policy"""
state = torch.from_numpy(state).float().to(device)
self.actor_local.eval()
with torch.no_grad():
action = self.actor_local(state).cpu().data.numpy()
self.actor_local.train()
if add_noise:
action += self.epsilon * self.noise.sample()
return np.clip(action, -1, 1)
def reset(self):
self.noise.reset()
def learn(self, experiences, gamma):
"""Update policy and value parameters using given batch of experience tuples.
Q_targets = r + γ * critic_target(next_state, actor_target(next_state))
where:
actor_target(state) -> action
critic_target(state, action) -> Q-value
Params
======
experiences (Tuple[torch.Tensor]): tuple of (s, a, r, s', done)
gamma (float): discount factor
"""
states, actions, rewards, next_states, dones = experiences
# Update critic
# get predicted next-state actions and Q values from target models
actions_next = self.actor_target(next_states)
Q_targets_next = self.critic_target(next_states, actions_next)
# compute Q targets for current states(y_i)
Q_targets = rewards + (gamma * Q_targets_next * (1 - dones))
# compute critic loss
Q_expected = self.critic_local(states, actions)
critic_loss = F.mse_loss(Q_expected, Q_targets)
# minimize the loss
self.critic_optimizer.zero_grad()
critic_loss.backward()
# gradient clipping for critic
if GRAD_CLIPPING > 0:
torch.nn.utils.clip_grad_norm_(self.critic_local.parameters(),
GRAD_CLIPPING)
self.critic_optimizer.step()
# update actor
# compute actor loss
actions_pred = self.actor_local(states)
actor_loss = -self.critic_local(states, actions_pred).mean()
# minimize the loss
self.actor_optimizer.zero_grad()
actor_loss.backward()
self.actor_optimizer.step()
# update target networks
self.soft_update(self.critic_local, self.critic_target, TAU)
self.soft_update(self.actor_local, self.actor_target, TAU)
# update epsilon decay
if EPSILON_DECAY > 0:
self.epsilon -= EPSILON_DECAY
self.noise.reset()
def soft_update(self, local_model, target_model, tau):
"""Soft update model parameters.
θ_target = τ*θ_local + (1 - τ)*θ_target
Params
======
local_model: PyTorch model from which weights will be copied
target_model: PyToch model to which weights will be copied
tau (float): interpolation parameter
"""
for target_param, local_param in zip(target_model.parameters(),
local_model.parameters()):
target_param.data.copy_(tau * local_param.data +
(1.0 - tau) * target_param.data)
class Agent():
"""Interacts with and learns from the environment."""
def __init__(self, state_size, action_size, seed):
"""Initialize an Agent object.
Params
======
state_size (int): dimension of each state
action_size (int): dimension of each action
seed (int): random seed
"""
self.state_size = state_size
self.action_size = action_size
self.seed = random.seed(seed)
self.epsilon = EPSILON
# Actor Network
self.actor_local = Actor(state_size, action_size, seed).to(device)
self.actor_target = Actor(state_size, action_size, seed).to(device)
self.actor_optimizer = optim.Adam(self.actor_local.parameters(),
lr=LR_ACTOR)
# Critic Network
self.critic_local = Critic(state_size, action_size, seed).to(device)
self.critic_target = Critic(state_size, action_size, seed).to(device)
self.critic_optimizer = optim.Adam(self.critic_local.parameters(),
lr=LR_CRITIC)
self.noise = OUNoise(action_size, seed)
# # Q-Network
# self.qnetwork_local = QNetwork(state_size, action_size, seed).to(device)
# self.qnetwork_target = QNetwork(state_size, action_size, seed).to(device)
# self.optimizer = optim.Adam(self.qnetwork_local.parameters(), lr=LR)
# Replay memory
self.memory = ReplayBuffer(action_size, BUFFER_SIZE, BATCH_SIZE, seed)
# # Initialize time step (for updating every UPDATE_EVERY steps)
# self.t_step = 0
def step(self, state, action, reward, next_state, done, timestep):
# Save experience in replay memory
self.memory.add(state, action, reward, next_state, done)
if len(self.memory) > BATCH_SIZE and timestep % LEARN_EVERY == 0:
for _ in range(LEARN_NUM):
experiences = self.memory.sample()
self.learn(experiences, GAMMA)
# # Learn every UPDATE_EVERY time steps.
# self.t_step = (self.t_step + 1) % UPDATE_EVERY
# if self.t_step == 0:
# # If enough samples are available in memory, get random subset and learn
# if len(self.memory) > BATCH_SIZE:
# experiences = self.memory.sample()
# self.learn(experiences, GAMMA)
def act(self, state, add_noise=True):
state = torch.from_numpy(state).float().to(device)
self.actor_local.eval()
with torch.no_grad():
action = self.actor_local(state).cpu().data.numpy()
self.actor_local.train()
if add_noise:
action += self.epsilon * self.noise.sample()
return np.clip(action, -1, 1)
def reset(self):
self.noise.reset()
def learn(self, experiences, gamma):
"""Update value parameters using given batch of experience tuples.
Params
======
experiences (Tuple[torch.Tensor]): tuple of (s, a, r, s', done) tuples
gamma (float): discount factor
"""
states, actions, rewards, next_states, dones = experiences
actions_next = self.actor_target(next_states)
Q_target_next = self.critic_target(next_states, actions_next)
Q_targets = rewards + (gamma * Q_target_next * (1 - dones))
Q_expected = self.critic_local(states, actions)
critic_loss = F.mse_loss(Q_expected, Q_targets)
self.critic_optimizer.zero_grad()
critic_loss.backward()
if GRAD_CLIPPING > 0:
torch.nn.utils.clip_grad_norm_(self.critic_local.parameters(),
GRAD_CLIPPING)
self.critic_optimizer.step()
actions_pred = self.actor_local(states)
actor_loss = -self.critic_local(states, actions_pred).mean()
self.actor_optimizer.zero_grad()
actor_loss.backward()
self.actor_optimizer.step()
self.soft_update(self.critic_local, self.critic_target, TAU)
self.soft_update(self.actor_local, self.actor_target, TAU)
if EPSILON_DECAY > 0:
self.epsilon -= EPSILON_DECAY
self.noise.reset()
def soft_update(self, local_model, target_model, tau):
"""Soft update model parameters.
θ_target = τ*θ_local + (1 - τ)*θ_target
Params
======
local_model (PyTorch model): weights will be copied from
target_model (PyTorch model): weights will be copied to
tau (float): interpolation parameter
"""
for target_param, local_param in zip(target_model.parameters(),
local_model.parameters()):
target_param.data.copy_(tau * local_param.data +
(1.0 - tau) * target_param.data)
class Agent():
"""Interacts with and learns from the environment."""
def __init__(self,
state_size,
action_size,
num_agents,
seed,
fc1=400,
fc2=300):
"""Initialize an Agent object.
Params
======
state_size (int): dimension of each state
action_size (int): dimension of each action
seed (int): random seed
"""
self.state_size = state_size
self.action_size = action_size
self.seed = random.seed(seed)
self.num_agents = num_agents
self.noise = [
OrnsteinUhlenbeckProcess(size=(action_size, ), std=0.2)
for i in range(num_agents)
]
# actor local and target network (Policy gradient)
self.actor_local = Actor(state_size, action_size, fc1, fc2,
seed).to(device)
self.actor_target = Actor(state_size, action_size, fc1, fc2,
seed).to(device)
# critic local and target network (Q-Learning)
self.critic_local = Critic(state_size, action_size, fc1, fc2,
seed).to(device)
self.critic_target = Critic(state_size, action_size, fc1, fc2,
seed).to(device)
# optimizer for critic and actor network
self.actor_optimizer = optim.Adam(self.actor_local.parameters(),
lr=ACTOR_LR)
self.critic_optimizer = optim.Adam(self.critic_local.parameters(),
lr=CRITIC_LR)
# Replay memory
self.memory = ReplayBuffer(action_size, BUFFER_SIZE, BATCH_SIZE, seed)
# Initialize time step (for updating every UPDATE_EVERY steps)
self.t_step = 0
def step(self, state, action, reward, next_state, done):
# Save experience in replay memory
for i in range(self.num_agents):
self.memory.add(state[i], action[i], reward[i], next_state[i],
done[i])
self.t_step += 1
# If enough samples are available in memory, get random subset and learn
if len(self.memory) > BATCH_SIZE:
if self.t_step % UPDATE_EVERY == 0:
for i in range(UPDATE_TIMES):
experiences = self.memory.sample()
self.learn(experiences, GAMMA)
def act(self, state, training=True):
"""Returns continous actions values for all action for given state as per current policy.
Params
======
state (array_like): current state
"""
state = torch.from_numpy(state).float().detach().to(device)
self.actor_local.eval()
with torch.no_grad():
actions = self.actor_local(state).cpu().data.numpy()
self.actor_local.train()
noise = np.array(
[self.noise[i].sample() for i in range(self.num_agents)])
return np.clip(actions + noise, -1, 1)
def learn(self, experiences, gamma):
"""Update policy and value parameters using given batch of experience tuples.
Q_targets = r + γ * critic_target(next_state, actor_target(next_state))
where:
actor_target(state) -> action
critic_target(state, action) -> Q-value
Params
======
experiences (Tuple[torch.Tensor]): tuple of (s, a, r, s', done) tuples
gamma (float): discount factor
"""
states, actions, rewards, next_states, dones = experiences
# ---------------------------- update critic ---------------------------- #
# Get predicted next-state actions and Q values from target models
actions_next = self.actor_target(next_states)
Q_targets_next = self.critic_target(next_states, actions_next)
# Compute Q targets for current states (y_i)
Q_targets = rewards + (gamma * Q_targets_next * (1 - dones))
# Compute critic loss
Q_expected = self.critic_local(states, actions)
critic_loss = F.mse_loss(Q_expected, Q_targets)
# Minimize the loss
self.critic_optimizer.zero_grad()
critic_loss.backward()
torch.nn.utils.clip_grad_norm_(self.critic_local.parameters(), 1)
self.critic_optimizer.step()
# ---------------------------- update actor ---------------------------- #
# Compute actor loss
actions_pred = self.actor_local(states)
actor_loss = -self.critic_local(states, actions_pred).mean()
# Minimize the loss
self.actor_optimizer.zero_grad()
torch.nn.utils.clip_grad_norm_(self.actor_local.parameters(), 1)
actor_loss.backward()
self.actor_optimizer.step()
# ----------------------- update target networks ----------------------- #
self.soft_update(self.critic_local, self.critic_target, TAU)
self.soft_update(self.actor_local, self.actor_target, TAU)
def soft_update(self, local_model, target_model, tau):
"""Soft update model parameters.
θ_target = τ*θ_local + (1 - τ)*θ_target
Params
======
local_model (PyTorch model): weights will be copied from
target_model (PyTorch model): weights will be copied to
tau (float): interpolation parameter
"""
for target_param, local_param in zip(target_model.parameters(),
local_model.parameters()):
target_param.data.copy_(tau * local_param.data +
(1.0 - tau) * target_param.data)
def reset_random(self):
for i in range(self.num_agents):
self.noise[i].reset_states()
class Agent():
"""Interacts with and learns from the environment."""
def __init__(self, state_size, action_size, random_seed):
"""Initialize an Agent object.
Params
======
state_size (int): dimension of each state
action_size (int): dimension of each action
random_seed (int): random seed
"""
self.state_size = state_size
self.action_size = action_size
self.seed = random.seed(random_seed)
# Actor Network (w/ Target Network)
self.actor_local = Actor(state_size, action_size,
random_seed).to(device)
self.actor_target = Actor(state_size, action_size,
random_seed).to(device)
self.actor_optimizer = optim.Adam(self.actor_local.parameters(),
lr=LR_ACTOR)
# Critic Network (w/ Target Network)
self.critic_local = Critic(state_size, action_size,
random_seed).to(device)
self.critic_target = Critic(state_size, action_size,
random_seed).to(device)
self.critic_optimizer = optim.Adam(self.critic_local.parameters(),
lr=LR_CRITIC,
weight_decay=WEIGHT_DECAY)
# Noise process
self.noise = OUNoise(action_size, random_seed)
# Replay memory
self.memory = ReplayBuffer(action_size, BUFFER_SIZE, BATCH_SIZE,
random_seed)
def step(self, state, action, reward, next_state, done):
"""Save experience in replay memory, and use random sample from buffer to learn."""
# Save experience / reward
self.memory.add(state, action, reward, next_state, done)
#Implement Learning from 10 samples every 20 episodes, Learning method transfered to the main script
# Learn, if enough samples are available in memory
#if len(self.memory) > BATCH_SIZE:
# experiences = self.memory.sample()
# self.learn(experiences, GAMMA)
def act(self, state, add_noise=True):
"""Returns actions for given state as per current policy."""
state = torch.from_numpy(state).float().to(device)
self.actor_local.eval()
with torch.no_grad():
action = self.actor_local(state).cpu().data.numpy()
self.actor_local.train()
if add_noise:
#for i in range(20): # add different random noise per agent
#action[i] += self.noise[i].sample()
action += self.noise.sample()
return np.clip(action, -1, 1)
def reset(self):
self.noise.reset()
# Start Learning from Shared replay buffer from 10 Samples every 20 steps
# From P2 continuous Control Forum
def go_learn(self):
if len(self.memory) > BATCH_SIZE:
experiences = self.memory.sample()
self.learn(experiences, GAMMA)
def learn(self, experiences, gamma):
"""Update policy and value parameters using given batch of experience tuples.
Q_targets = r + γ * critic_target(next_state, actor_target(next_state))
where:
actor_target(state) -> action
critic_target(state, action) -> Q-value
Params
======
experiences (Tuple[torch.Tensor]): tuple of (s, a, r, s', done) tuples
gamma (float): discount factor
"""
states, actions, rewards, next_states, dones = experiences
# ---------------------------- update critic ---------------------------- #
# Get predicted next-state actions and Q values from target models
actions_next = self.actor_target(next_states)
Q_targets_next = self.critic_target(next_states, actions_next)
# Compute Q targets for current states (y_i)
Q_targets = rewards + (gamma * Q_targets_next * (1 - dones))
# Compute critic loss
Q_expected = self.critic_local(states, actions)
critic_loss = F.mse_loss(Q_expected, Q_targets)
# Minimize the loss
self.critic_optimizer.zero_grad()
critic_loss.backward()
torch.nn.utils.clip_grad_norm(
self.critic_local.parameters(), 1
) #Clipping gradiants of the critic local network, P2 Continuous Project instruction - Benchmarking implementation
self.critic_optimizer.step()
# ---------------------------- update actor ---------------------------- #
# Compute actor loss
actions_pred = self.actor_local(states)
actor_loss = -self.critic_local(states, actions_pred).mean()
# Minimize the loss
self.actor_optimizer.zero_grad()
actor_loss.backward()
self.actor_optimizer.step()
# ----------------------- update target networks ----------------------- #
self.soft_update(self.critic_local, self.critic_target, TAU)
self.soft_update(self.actor_local, self.actor_target, TAU)
def soft_update(self, local_model, target_model, tau):
"""Soft update model parameters.
θ_target = τ*θ_local + (1 - τ)*θ_target
Params
======
local_model: PyTorch model (weights will be copied from)
target_model: PyTorch model (weights will be copied to)
tau (float): interpolation parameter
"""
for target_param, local_param in zip(target_model.parameters(),
local_model.parameters()):
target_param.data.copy_(tau * local_param.data +
(1.0 - tau) * target_param.data)
class DDPG():
"""Reinforcement Learning agent that learns using DDPG."""
def __init__(self, task):
self.task = task
self.state_size = task.state_size
self.action_size = task.action_size
self.action_low = task.action_low
self.action_high = task.action_high
# Actor (Policy) Model
self.actor_local = Actor(self.state_size, self.action_size, self.action_low, self.action_high)
self.actor_target = Actor(self.state_size, self.action_size, self.action_low, self.action_high)
# Critic (Value) Model
self.critic_local = Critic(self.state_size, self.action_size)
self.critic_target = Critic(self.state_size, self.action_size)
# Initialize target model parameters with local model parameters
self.critic_target.model.set_weights(self.critic_local.model.get_weights())
self.actor_target.model.set_weights(self.actor_local.model.get_weights())
# Noise process
self.exploration_mu = 0 #0
self.exploration_theta = 0.15
self.exploration_sigma = 0.2
self.noise = OUNoise(self.action_size, self.exploration_mu, self.exploration_theta, self.exploration_sigma)
# Replay memory
self.buffer_size = 100000
self.batch_size = 64
self.memory = ReplayBuffer(self.buffer_size, self.batch_size)
# Algorithm parameters
self.gamma = 0.99 # discount factor - 0.99
self.tau = 0.01 # for soft update of target parameters - 0.01
# Score tracker and learning parameters
self.best_w = None
self.best_score = -np.inf
self.score = -np.inf
def reset_episode(self):
self.total_reward = 0.0
self.count = 0
self.noise.reset()
state = self.task.reset()
self.last_state = state
return state
def step(self, action, reward, next_state, done):
# Save experience / reward
self.total_reward += reward
self.count += 1
self.memory.add(self.last_state, action, reward, next_state, done)
# Learn, if enough samples are available in memory
if len(self.memory) > self.batch_size:
experiences = self.memory.sample()
self.learn(experiences)
# Roll over last state and action
self.last_state = next_state
def act(self, state):
"""Returns actions for given state(s) as per current policy."""
state = np.reshape(state, [-1, self.state_size])
action = self.actor_local.model.predict(state)[0]
return list(action + self.noise.sample()) # add some noise for exploration
def learn(self, experiences):
"""Update policy and value parameters using given batch of experience tuples."""
self.score = self.total_reward / float(self.count) if self.count else 0.0
if self.score > self.best_score:
self.best_score = self.score
# Convert experience tuples to separate arrays for each element (states, actions, rewards, etc.)
states = np.vstack([e.state for e in experiences if e is not None])
actions = np.array([e.action for e in experiences if e is not None]).astype(np.float32).reshape(-1, self.action_size)
rewards = np.array([e.reward for e in experiences if e is not None]).astype(np.float32).reshape(-1, 1)
dones = np.array([e.done for e in experiences if e is not None]).astype(np.uint8).reshape(-1, 1)
next_states = np.vstack([e.next_state for e in experiences if e is not None])
# Get predicted next-state actions and Q values from target models
# Q_targets_next = critic_target(next_state, actor_target(next_state))
actions_next = self.actor_target.model.predict_on_batch(next_states)
Q_targets_next = self.critic_target.model.predict_on_batch([next_states, actions_next])
# Compute Q targets for current states and train critic model (local)
Q_targets = rewards + self.gamma * Q_targets_next * (1 - dones)
self.critic_local.model.train_on_batch(x=[states, actions], y=Q_targets)
# Train actor model (local)
action_gradients = np.reshape(self.critic_local.get_action_gradients([states, actions, 0]), (-1, self.action_size))
self.actor_local.train_fn([states, action_gradients, 1]) # custom training function
# Soft-update target models
self.soft_update(self.critic_local.model, self.critic_target.model)
self.soft_update(self.actor_local.model, self.actor_target.model)
def soft_update(self, local_model, target_model):
"""Soft update model parameters."""
local_weights = np.array(local_model.get_weights())
target_weights = np.array(target_model.get_weights())
assert len(local_weights) == len(target_weights), "Local and target model parameters must have the same size"
new_weights = self.tau * local_weights + (1 - self.tau) * target_weights
target_model.set_weights(new_weights)
