class Preyer:
def __init__(self, s_dim, a_dim, **kwargs):
self.s_dim = s_dim
self.a_dim = a_dim
self.config = kwargs['config']
self.device = 'cuda' if self.config.use_cuda else 'cpu'
self.actor = Actor(s_dim, a_dim)
self.actor_target = Actor(s_dim, a_dim)
self.critic = Critic(s_dim, a_dim, 1)
self.critic_target = Critic(s_dim, a_dim, 1)
self.actor_optimizer = torch.optim.Adam(self.actor.parameters(),
lr=self.config.a_lr)
self.critic_optimizer = torch.optim.Adam(self.critic.parameters(),
lr=self.config.c_lr)
self.c_loss = 0
self.a_loss = 0
if self.config.use_cuda:
self.actor.cuda()
self.actor_target.cuda()
self.critic.cuda()
self.critic_target.cuda()
hard_update(self.actor, self.actor_target)
hard_update(self.critic, self.critic_target)
self.random_process = OrnsteinUhlenbeckProcess(
size=self.a_dim,
theta=self.config.ou_theta,
mu=self.config.ou_mu,
sigma=self.config.ou_sigma)
self.replay_buffer = list()
self.epsilon = 1.
self.depsilon = self.epsilon / self.config.epsilon_decay
def memory(self, s, a, r, s_, done):
self.replay_buffer.append((s, a, r, s_, done))
if len(self.replay_buffer) >= self.config.memory_length:
self.replay_buffer.pop(0)
def get_batches(self):
experiences = random.sample(self.replay_buffer, self.config.batch_size)
state_batches = np.array([_[0] for _ in experiences])
action_batches = np.array([_[1] for _ in experiences])
reward_batches = np.array([_[2] for _ in experiences])
next_state_batches = np.array([_[3] for _ in experiences])
done_batches = np.array([_[4] for _ in experiences])
return state_batches, action_batches, reward_batches, next_state_batches, done_batches
def choose_action(self, s, noisy=True):
if self.config.use_cuda:
s = Variable(torch.cuda.FloatTensor(s))
else:
s = Variable(torch.FloatTensor(s))
a = self.actor.forward(s).cpu().detach().numpy()
if noisy:
a += max(self.epsilon, 0.001) * self.random_process.sample()
self.epsilon -= self.depsilon
a = np.clip(a, -1., 1.)
return np.array([a])
def random_action(self):
action = np.random.uniform(low=-1., high=1., size=(1, self.a_dim))
return action
def reset(self):
self.random_process.reset_states()
def train(self):
state_batches, action_batches, reward_batches, next_state_batches, done_batches = self.get_batches(
)
state_batches = Variable(torch.Tensor(state_batches).to(self.device))
action_batches = Variable(
torch.Tensor(action_batches).reshape(-1, 1).to(self.device))
reward_batches = Variable(
torch.Tensor(reward_batches).reshape(-1, 1).to(self.device))
next_state_batches = Variable(
torch.Tensor(next_state_batches).to(self.device))
done_batches = Variable(
torch.Tensor(
(done_batches == False) * 1).reshape(-1, 1).to(self.device))
target_next_actions = self.actor_target.forward(
next_state_batches).detach()
target_next_q = self.critic_target.forward(
next_state_batches, target_next_actions).detach()
main_q = self.critic(state_batches, action_batches)
# Critic Loss
self.critic.zero_grad()
baselines = reward_batches + done_batches * self.config.gamma * target_next_q
loss_critic = torch.nn.MSELoss()(main_q, baselines)
loss_critic.backward()
self.critic_optimizer.step()
# Actor Loss
self.actor.zero_grad()
clear_action_batches = self.actor.forward(state_batches)
loss_actor = (
-self.critic.forward(state_batches, clear_action_batches)).mean()
loss_actor.backward()
self.actor_optimizer.step()
# This is for logging
self.c_loss = loss_critic.item()
self.a_loss = loss_actor.item()
soft_update(self.actor, self.actor_target, self.config.tau)
soft_update(self.critic, self.critic_target, self.config.tau)
def getLoss(self):
return self.c_loss, self.a_loss
class DDPG(object):
def __init__(self, nb_states, nb_actions, args):
if args.seed > 0:
self.seed(args.seed)
self.nb_states = nb_states
self.nb_actions = nb_actions
# Create Actor and Critic Network
net_cfg = {
"hidden1": args.hidden1,
"hidden2": args.hidden2,
"init_w": args.init_w,
}
self.actor = Actor(self.nb_states, self.nb_actions, **net_cfg)
self.actor_target = Actor(self.nb_states, self.nb_actions, **net_cfg)
self.actor_optim = Adam(self.actor.parameters(), lr=args.prate)
self.critic = Critic(self.nb_states, self.nb_actions, **net_cfg)
self.critic_target = Critic(self.nb_states, self.nb_actions, **net_cfg)
self.critic_optim = Adam(self.critic.parameters(), lr=args.rate)
hard_update(
self.actor_target, self.actor
) # Make sure target is with the same weight
hard_update(self.critic_target, self.critic)
# Create replay buffer
self.memory = SequentialMemory(
limit=args.rmsize, window_length=args.window_length
)
self.random_process = OrnsteinUhlenbeckProcess(
size=nb_actions, theta=args.ou_theta, mu=args.ou_mu, sigma=args.ou_sigma
)
# Hyper-parameters
self.batch_size = args.bsize
self.tau = args.tau
self.discount = args.discount
self.depsilon = 1.0 / args.epsilon
#
self.epsilon = 1.0
self.s_t = None # Most recent state
self.a_t = None # Most recent action
self.is_training = True
#
if USE_CUDA:
self.cuda()
def update_policy(self):
# Sample batch
(
state_batch,
action_batch,
reward_batch,
next_state_batch,
terminal_batch,
) = self.memory.sample_and_split(self.batch_size)
# Prepare for the target q batch
next_q_values = self.critic_target(
[
to_tensor(next_state_batch, volatile=True),
self.actor_target(to_tensor(next_state_batch, volatile=True)),
]
)
# next_q_values.volatile = False
target_q_batch = (
to_tensor(reward_batch)
+ self.discount * to_tensor(terminal_batch.astype(np.float)) * next_q_values
)
# Critic update
self.critic.zero_grad()
q_batch = self.critic([to_tensor(state_batch), to_tensor(action_batch)])
value_loss = criterion(q_batch, target_q_batch)
value_loss.backward()
self.critic_optim.step()
# Actor update
self.actor.zero_grad()
policy_loss = -self.critic(
[to_tensor(state_batch), self.actor(to_tensor(state_batch))]
)
policy_loss = policy_loss.mean()
policy_loss.backward()
self.actor_optim.step()
# Target update
soft_update(self.actor_target, self.actor, self.tau)
soft_update(self.critic_target, self.critic, self.tau)
def eval(self):
self.actor.eval()
self.actor_target.eval()
self.critic.eval()
self.critic_target.eval()
def cuda(self):
self.actor.cuda()
self.actor_target.cuda()
self.critic.cuda()
self.critic_target.cuda()
def observe(self, r_t, s_t1, done):
if self.is_training:
self.memory.append(self.s_t, self.a_t, r_t, done)
self.s_t = s_t1
def random_action(self):
action = np.random.uniform(-1.0, 1.0, self.nb_actions)
self.a_t = action
return action
def select_action(self, s_t, decay_epsilon=True):
action = to_numpy(self.actor(to_tensor(np.array([s_t])))).squeeze(0)
action += self.is_training * max(self.epsilon, 0) * self.random_process.sample()
action = np.clip(action, -1.0, 1.0)
if decay_epsilon:
self.epsilon -= self.depsilon
self.a_t = action
return action
def reset(self, obs):
self.s_t = obs
self.random_process.reset_states()
def load_weights(self, output):
if output is None:
return
self.actor.load_state_dict(torch.load("{}/actor.pkl".format(output)))
self.critic.load_state_dict(torch.load("{}/critic.pkl".format(output)))
def save_model(self, output):
torch.save(self.actor.state_dict(), "{}/actor.pkl".format(output))
torch.save(self.critic.state_dict(), "{}/critic.pkl".format(output))
def seed(self, s):
torch.manual_seed(s)
if USE_CUDA:
torch.cuda.manual_seed(s)
class Agent():
def __init__(self, nb_states, nb_actions):
self.critic = Critic(nb_states, nb_actions) # Q
self.critic_target = Critic(nb_states, nb_actions)
self.actor = Actor(nb_states, nb_actions) # policy mu
self.actor_target = Actor(nb_states, nb_actions)
hard_update(self.critic_target, self.critic)
hard_update(self.actor_target, self.actor)
self.critic_optimizer = torch.optim.Adam(self.critic.parameters(),
lr=0.001)
self.actor_optimizer = torch.optim.Adam(self.actor.parameters(),
lr=0.0001)
self.criterion = nn.MSELoss()
self.random_process = OrnsteinUhlenbeckProcess(size=nb_actions,
theta=0.15,
mu=0,
sigma=0.2)
self.gamma = 0.99
self.batch_size = 64
if USE_CUDA:
self.actor.cuda()
self.actor_target.cuda()
self.critic.cuda()
self.critic_target.cuda()
def act(self, obs, epsilon=0.1): # epsilon -> tunning paramter
if (random.random() < epsilon): # choose random action
action = np.random.uniform(-1., 1., nb_actions)
return action
else: # the action is the output of actor network + Exploration Noise
action = self.actor(obs).cpu().data.numpy()
action += self.random_process.sample()
action = np.clip(action, -1., 1.) # to stay in interval [-1,1]
return action
def backward(self, transitions):
transitions = memory.sample(self.batch_size)
batch = Transition(*zip(*transitions))
state_batch = Variable(torch.cat(batch.state)).type(
FLOAT) # size 64 x 3
action_batch = Variable(torch.cat(batch.action)).type(FLOAT) # size 64
next_state_batch = Variable(torch.cat(batch.next_state)).type(
FLOAT) # size 64 x 3
reward_batch = Variable(torch.cat(batch.reward)).type(FLOAT) # size 64
done_batch = Variable(torch.cat(batch.done)).type(FLOAT)
#### Q - CRITIC UPDATE ####
# Q(s_t,a_t)
action_batch.unsqueeze_(1) # size 64x1
state_action_value = self.critic(state_batch, action_batch) # 64x1
# a_{t+1} = mu_target(s_{t+1})
next_action = self.actor_target(
next_state_batch).detach() # 64 x nb_actions
# Q'(s_{t+1},a_{t+1})
next_state_action_value = self.critic_target(next_state_batch,
next_action).detach()
next_state_action_value.squeeze_() # 64
# mask to consider next_state_values to 0 if state is terminal
mask = Variable(
np.logical_not(done_batch.data).type(
torch.FloatTensor)).type(FLOAT)
# mask = 1,1,1 ..
# Compare Q(s_t,a_t) with r_t + gamma * Q'(s_{t+1},a_{t+1})
expected_state_action_value = reward_batch + (
self.gamma * next_state_action_value * mask)
# Compute Huber loss
# loss = F.smooth_l1_loss(state_action_values, expected_state_action_values)
loss = self.criterion(state_action_value, expected_state_action_value)
# Optimize the nn by updating weights with adam descent
self.critic_optimizer.zero_grad()
loss.backward()
self.critic_optimizer.step()
#### mu - ACTOR UPDATE ####
# a_t = mu(s_t)
action = self.actor(state_batch)
# J = esperance[Q(s_t,mu(s_t))] -> a maximiser
# -J = policy_loss -> a minimiser
policy_loss = -self.critic(state_batch, action)
policy_loss = policy_loss.mean()
self.actor_optimizer.zero_grad()
policy_loss.backward()
self.actor_optimizer.step()
#### update target network with polyak averaging
soft_update(self.critic_target, self.critic, tau=0.001)
soft_update(self.actor_target, self.actor, tau=0.001)
return
class RDPG_v2:
def __init__(self, conf, device):
self.conf = conf
self.state_dim = conf['state_dim']
self.action_dim = conf['action_dim']
self.device = device
# create actor and critic network
self.actor = Actor_RDPG(self.state_dim,
self.action_dim).to(self.device)
self.actor_target = Actor_RDPG(self.state_dim,
self.action_dim).to(self.device)
self.critic = Critic_RDPG(self.state_dim,
self.action_dim).to(self.device)
self.critic_target = Critic_RDPG(self.state_dim,
self.action_dim).to(self.device)
hard_update(self.actor_target,
self.actor) # Make sure target is with the same weight
hard_update(self.critic_target, self.critic)
self.critic_optim = optim.Adam(self.critic.parameters(), lr=q_lr)
self.actor_optim = optim.Adam(self.actor.parameters(), lr=policy_lr)
#Create replay buffer
self.random_process = OrnsteinUhlenbeckProcess(size=self.action_dim,
theta=0.15,
mu=0.0,
sigma=0.2)
# args.ou_theta:0.15 (noise theta), args.ou_sigma:0.2 (noise sigma), args.out_mu:0.0 (noise mu)
self.epsilon = 1.0
self.depsilon = 1.0 / 50000
self.is_training = True
self.tau = 0.001 # moving average for target network
def random_action(self):
action = np.random.uniform(
0., 1., self.action_dim) # [-1,1] select as a number of action_dim
return action
def select_action(self, state, noise_enable=True, decay_epsilon=True):
action, _ = self.actor(
to_tensor(state).reshape(-1).unsqueeze(0)
) # input shape = [batch(=1) X state_dim], action : type (tuple), shape [batch X action_dim]
action = action.cpu().detach().numpy().squeeze(
0) # action shape [action_dim,]
if noise_enable == True:
action += self.is_training * max(self.epsilon,
0) * self.random_process.sample()
action = np.clip(action, 0.,
1.) # input 중 -1~1 을 벗어나는 값에 대해 -1 or 1 로 대체
if decay_epsilon:
self.epsilon -= self.depsilon
return action
def update_policy(self, memory, gamma=0.99):
print("updating...")
# Sample batch
experiences = memory.sample(
self.conf['batch_size']
) # type: list | shape: (max_epi_length(2000)-1 X batch(32) X 5(??))
if len(experiences) == 0: # not enough samples
return
dtype = torch.cuda.FloatTensor
policy_loss_total = 0
value_loss_total = 0
for t in range(len(experiences) - 1): # iterate over episodes
# print("t:", t)
target_cx = Variable(torch.zeros(self.conf['batch_size'],
50)).type(dtype)
target_hx = Variable(torch.zeros(self.conf['batch_size'],
50)).type(dtype)
cx = Variable(torch.zeros(self.conf['batch_size'], 50)).type(dtype)
hx = Variable(torch.zeros(self.conf['batch_size'], 50)).type(dtype)
# we first get the data out of the sampled experience
# shape of state0, action, reward: [batch X state_dim], [batch X 1], [batch X 1]
state0 = np.stack([
trajectory.state0 for trajectory in experiences[t]
]) # batch 개수만큼 각 epi 중 t 시점에서 상태만 추출
# action = np.expand_dims(np.stack((trajectory.action for trajectory in experiences[t])), axis=1)
action = np.stack(
[trajectory.action for trajectory in experiences[t]])
reward = np.expand_dims(np.stack(
[trajectory.reward for trajectory in experiences[t]]),
axis=1)
# reward = np.stack((trajectory.reward for trajectory in experiences[t]))
state1 = np.stack(
[trajectory.state0 for trajectory in experiences[t + 1]])
target_action, (target_hx, target_cx) = self.actor_target(
to_tensor(state1).reshape(self.conf['batch_size'], -1),
(target_hx, target_cx))
next_q_value = self.critic_target([
to_tensor(state1).reshape(self.conf['batch_size'], -1),
target_action
])
target_q = to_tensor(reward) + gamma * next_q_value
# Critic update
current_q = self.critic([
to_tensor(state0).reshape(self.conf['batch_size'], -1),
to_tensor(action)
])
value_loss = F.smooth_l1_loss(current_q, target_q)
value_loss /= len(experiences) # divide by trajectory length
value_loss_total += value_loss
# update per trajectory
self.critic.zero_grad()
value_loss.backward()
# Actor update
action, (hx, cx) = self.actor(
to_tensor(state0).reshape(self.conf['batch_size'], -1),
(hx, cx))
policy_loss = -self.critic([
to_tensor(state0).reshape(self.conf['batch_size'], -1), action
])
policy_loss /= len(experiences) # divide by trajectory length
policy_loss_total += policy_loss.mean()
policy_loss = policy_loss.mean()
self.actor.zero_grad()
policy_loss.backward()
self.critic_optim.step()
self.actor_optim.step()
# Target update
soft_update(self.actor_target, self.actor, self.tau)
soft_update(self.critic_target, self.critic, self.tau)
print("update finish!")
def reset_lstm_hidden_state(self, done=True):
self.actor.reset_lstm_hidden_state(done)
def save_model(self, path):
torch.save(self.critic.state_dict(), path + '_q')
torch.save(self.critic_target.state_dict(), path + '_target_q')
torch.save(self.actor.state_dict(), path + '_policy')
def load_model(self, path):
self.critic.load_state_dict(torch.load(path + '_q'))
self.critic_target.load_state_dict(torch.load(path + '_target_q'))
self.actor.load_state_dict(torch.load(path + '_policy'))
self.critic.eval()
self.critic_target.eval()
self.actor.eval()
class DDPG(object):
def __init__(self,
state_size,
action_size,
memory_size,
batch_size=128,
tan=0.001,
actor_lr=0.001,
critic_lr=0.001,
epsilon=1.):
self.state_size = state_size
self.action_size = action_size
self.batch_size = batch_size
self.tan = tan
self.warmup = WARM_UP
self.epsilon = epsilon
self.epsilon_decay = hyperparameters['D_EPSILON']
self.actor = Actor(state_size, action_size)
self.actor_target = Actor(state_size, action_size)
self.actor_optimizer = optim.Adam(self.actor.parameters(), lr=actor_lr)
self.critic = Critic(state_size, action_size)
self.critic_target = Critic(state_size, action_size)
self.critic_optimizer = optim.Adam(self.critic.parameters(),
lr=critic_lr)
self.memory = Memory(memory_size)
self.criterion = nn.MSELoss()
self.random_process = OrnsteinUhlenbeckProcess(size=action_size,
theta=0.15,
mu=0.,
sigma=0.2)
copy_parameter(self.actor, self.actor_target)
copy_parameter(self.critic, self.critic_target)
def train(self):
# if not warm up
if self.memory.counter < self.warmup:
return
# get batch
state_batch, action_batch, next_state_batch, reward_batch, done_batch = self.memory.sample(
self.batch_size)
action_batch = action_batch.reshape((-1, self.action_size))
reward_batch = reward_batch.reshape((-1, 1))
done_batch = done_batch.reshape((-1, 1))
# update critic
nsb = Variable(torch.from_numpy(next_state_batch).float(),
volatile=True) # next_state_batch
nab = self.actor_target(nsb) # next_action_batch
next_q = self.critic_target(nsb, nab)
next_q.volatile = False
rb = Variable(torch.from_numpy(reward_batch).float()) # reward_batch
db = Variable(torch.from_numpy(done_batch).float(
)) # if next state is None, next_q should be 0, which means q = r
q_target = rb + hyperparameters['GAMMA'] * db * next_q
sb_grad = Variable(torch.from_numpy(state_batch).float()
) # state_batch with grad, mean output need grad
ab = Variable(torch.from_numpy(action_batch).float()) # action_batch
q_eval = self.critic(sb_grad, ab)
value_loss = self.criterion(q_eval, q_target)
self.critic.zero_grad()
value_loss.backward()
# nn.utils.clip_grad_norm(self.critic.parameters(),0.8)
self.critic_optimizer.step()
# update actor
sb_grad = Variable(
torch.from_numpy(state_batch).float()) # state_batch
aab = self.actor(sb_grad) # actor_action_batch
q = self.critic(sb_grad, aab)
policy_loss = torch.mean(-q)
self.actor.zero_grad()
policy_loss.backward()
# nn.utils.clip_grad_norm(self.actor.parameters(),0.8)
self.actor_optimizer.step()
# update parameter between two network
update_parameter(self.critic_target, self.critic, self.tan)
update_parameter(self.actor_target, self.actor, self.tan)
def select_action(self, s, is_train=True, decay_e=True):
if self.memory.counter < self.warmup:
action = env.action_space.sample()[0]
# action = random.uniform(-2.,2.)
return action
state = Variable(torch.FloatTensor([s]).float())
action = self.actor(state).squeeze(1).data.numpy()
action += is_train * max(self.epsilon,
0) * self.random_process.sample()
action = float(np.clip(action, -1., 1.)[0])
if decay_e:
if self.memory.counter > self.warmup:
self.epsilon -= self.epsilon_decay
return action
class DDPG(object):
def __init__(self,
env,
mem_size=7 * int(1e3),
lr_critic=1e-3,
lr_actor=1e-4,
epsilon=1.,
max_epi=1500,
epsilon_decay=1. / (1e5),
gamma=.99,
target_update_frequency=200,
batch_size=64,
random_process=True,
max_step=None):
self.CUDA = torch.cuda.is_available()
self.orig_env = env #for recording
if max_step is not None:
self.orig_env._max_episode_steps = max_step
self.env = self.orig_env
self.N_S = self.env.observation_space.shape[0]
self.N_A = self.env.action_space.shape[0]
self.MAX_EPI = max_epi
self.LOW = self.env.action_space.low
self.HIGH = self.env.action_space.high
self.actor = Actor(self.N_S, self.N_A)
self.critic = Critic(self.N_S, self.N_A)
self.target_actor = Actor(self.N_S, self.N_A)
self.target_critic = Critic(self.N_S, self.N_A)
self.target_actor.eval()
self.target_critic.eval()
self.target_actor.load_state_dict(self.actor.state_dict())
self.target_critic.load_state_dict(self.critic.state_dict())
if self.CUDA:
self.actor.cuda()
self.critic.cuda()
self.target_actor.cuda()
self.target_critic.cuda()
self.exp = Experience(mem_size)
self.optim_critic = optim.Adam(self.critic.parameters(), lr=lr_critic)
self.optim_actor = optim.Adam(self.actor.parameters(), lr=-lr_actor)
self.random_process = OrnsteinUhlenbeckProcess(\
size=self.N_A, theta=.15, mu=0, sigma=.2)
self.EPSILON = epsilon
self.EPSILON_DECAY = epsilon_decay
self.GAMMA = gamma
self.TARGET_UPDATE_FREQUENCY = target_update_frequency
self.BATCH_SIZE = batch_size
title = {common.S_EPI: [], common.S_TOTAL_R: []}
self.data = pd.DataFrame(title)
self.RAND_PROC = random_process
def train(self, dir=None, interval=1000):
if dir is not None:
self.env = wrappers.Monitor(self.orig_env,
'{}/train_record'.format(dir),
force=True)
os.mkdir(os.path.join(dir, 'models'))
update_counter = 0
epsilon = self.EPSILON
for epi in trange(self.MAX_EPI, desc='train epi', leave=True):
self.random_process.reset_states()
o = self.env.reset()
counter = 0
acc_r = 0
while True:
counter += 1
#if dir is not None:
# self.env.render()
a = self.choose_action(o)
if self.RAND_PROC:
a += max(epsilon, 0) * self.random_process.sample()
a = np.clip(a, -1., 1.)
epsilon -= self.EPSILON_DECAY
o_, r, done, info = self.env.step(self.map_to_action(a))
self.exp.push(o, a, r, o_, done)
if epi > 0:
self.update_actor_critic()
update_counter += 1
if update_counter % self.TARGET_UPDATE_FREQUENCY == 0:
self.update_target()
acc_r += r
o = o_
if done:
break
if dir is not None:
if (epi + 1) % interval == 0:
self.save(os.path.join(dir, 'models'),
str(epi + 1),
save_data=False)
s = pd.Series([epi, acc_r], index=[common.S_EPI, common.S_TOTAL_R])
self.data = self.data.append(s, ignore_index=True)
def choose_action(self, state):
self.actor.eval()
s = Variable(torch.Tensor(state)).unsqueeze(0)
if self.CUDA:
s = s.cuda()
a = self.actor(s).data.cpu().numpy()[0].astype('float64')
self.actor.train()
return a
def map_to_action(self, a):
return (self.LOW + self.HIGH) / 2 + a * (self.HIGH - self.LOW) / 2
def update_target(self):
self.target_actor.load_state_dict(self.actor.state_dict())
self.target_critic.load_state_dict(self.critic.state_dict())
def update_actor_critic(self):
# sample minibatch
minibatch = common.Transition(*zip(*self.exp.sample(self.BATCH_SIZE)))
bat_o = Variable(torch.Tensor(minibatch.state))
bat_a = Variable(torch.Tensor(minibatch.action))
bat_r = Variable(torch.Tensor(minibatch.reward)).unsqueeze(1)
bat_o_ = Variable(torch.Tensor(minibatch.next_state))
bat_not_done_mask = list(
map(lambda done: 0 if done else 1, minibatch.done))
bat_not_done_mask = Variable(
torch.ByteTensor(bat_not_done_mask)).unsqueeze(1)
if self.CUDA:
bat_o = bat_o.cuda()
bat_a = bat_a.cuda()
bat_r = bat_r.cuda()
bat_o_ = bat_o_.cuda()
bat_not_done_mask = bat_not_done_mask.cuda()
# update critic
bat_a_o_ = self.target_actor(bat_o_)
Gt = bat_r
Gt[bat_not_done_mask] += self.GAMMA * self.target_critic(
bat_o_, bat_a_o_)[bat_not_done_mask]
Gt.detach_()
eval_o = self.critic(bat_o, bat_a)
criterion = nn.MSELoss()
if self.CUDA:
criterion.cuda()
loss = criterion(eval_o, Gt)
self.optim_critic.zero_grad()
loss.backward()
self.optim_critic.step()
# update actor
self.critic.eval()
bat_a_o = self.actor(bat_o)
obj = torch.mean(self.critic(bat_o, bat_a_o))
self.optim_actor.zero_grad()
obj.backward()
self.optim_actor.step()
self.critic.train()
def test(self, dir=None, n=1):
if dir is not None:
self.env = wrappers.Monitor(self.orig_env,
'{}/test_record'.format(dir),
force=True,
video_callable=lambda episode_id: True)
title = {common.S_EPI: [], common.S_TOTAL_R: []}
df = pd.DataFrame(title)
for epi in trange(n, desc='test epi', leave=True):
o = self.env.reset()
acc_r = 0
while True:
#if dir is not None:
# self.env.render()
a = self.choose_action(o)
o_, r, done, info = self.env.step(self.map_to_action(a))
acc_r += r
o = o_
if done:
break
s = pd.Series([epi, acc_r], index=[common.S_EPI, common.S_TOTAL_R])
df = df.append(s, ignore_index=True)
if dir is not None:
df.to_csv('{}/test_data.csv'.format(dir))
else:
print df
def save(self, dir, suffix='', save_data=True):
torch.save(self.actor.state_dict(),
'{}/actor{}.pt'.format(dir, suffix))
torch.save(self.critic.state_dict(),
'{}/critic{}.pt'.format(dir, suffix))
if save_data:
self.data.to_csv('{}/train_data{}.csv'.format(dir, suffix))
def load_actor(self, dir):
self.actor.load_state_dict(torch.load(dir))
def load_critic(self, dir):
self.critic.load_state_dict(torch.load(dir))
def get_data(self):
return self.data
class DDPG(object):
def __init__(self, args, nb_states, nb_actions):
USE_CUDA = torch.cuda.is_available()
if args.seed > 0:
self.seed(args.seed)
self.nb_states = nb_states
self.nb_actions= nb_actions
self.gpu_ids = [i for i in range(args.gpu_nums)] if USE_CUDA and args.gpu_nums > 0 else [-1]
self.gpu_used = True if self.gpu_ids[0] >= 0 else False
net_cfg = {
'hidden1':args.hidden1,
'hidden2':args.hidden2,
'init_w':args.init_w
}
self.actor = Actor(self.nb_states, self.nb_actions, **net_cfg).double()
self.actor_target = Actor(self.nb_states, self.nb_actions, **net_cfg).double()
self.actor_optim = Adam(self.actor.parameters(), lr=args.p_lr, weight_decay=args.weight_decay)
self.critic = Critic(self.nb_states, self.nb_actions, **net_cfg).double()
self.critic_target = Critic(self.nb_states, self.nb_actions, **net_cfg).double()
self.critic_optim = Adam(self.critic.parameters(), lr=args.c_lr, weight_decay=args.weight_decay)
hard_update(self.actor_target, self.actor) # Make sure target is with the same weight
hard_update(self.critic_target, self.critic)
#Create replay buffer
self.memory = SequentialMemory(limit=args.rmsize, window_length=args.window_length)
self.random_process = OrnsteinUhlenbeckProcess(size=self.nb_actions,
theta=args.ou_theta, mu=args.ou_mu, sigma=args.ou_sigma)
# Hyper-parameters
self.batch_size = args.bsize
self.tau_update = args.tau_update
self.gamma = args.gamma
# Linear decay rate of exploration policy
self.depsilon = 1.0 / args.epsilon
# initial exploration rate
self.epsilon = 1.0
self.s_t = None # Most recent state
self.a_t = None # Most recent action
self.is_training = True
self.continious_action_space = False
def update_policy(self):
pass
def cuda_convert(self):
if len(self.gpu_ids) == 1:
if self.gpu_ids[0] >= 0:
with torch.cuda.device(self.gpu_ids[0]):
print('model cuda converted')
self.cuda()
if len(self.gpu_ids) > 1:
self.data_parallel()
self.cuda()
self.to_device()
print('model cuda converted and paralleled')
def eval(self):
self.actor.eval()
self.actor_target.eval()
self.critic.eval()
self.critic_target.eval()
def cuda(self):
self.actor.cuda()
self.actor_target.cuda()
self.critic.cuda()
self.critic_target.cuda()
def data_parallel(self):
self.actor = nn.DataParallel(self.actor, device_ids=self.gpu_ids)
self.actor_target = nn.DataParallel(self.actor_target, device_ids=self.gpu_ids)
self.critic = nn.DataParallel(self.critic, device_ids=self.gpu_ids)
self.critic_target = nn.DataParallel(self.critic_target, device_ids=self.gpu_ids)
def to_device(self):
self.actor.to(torch.device('cuda:{}'.format(self.gpu_ids[0])))
self.actor_target.to(torch.device('cuda:{}'.format(self.gpu_ids[0])))
self.critic.to(torch.device('cuda:{}'.format(self.gpu_ids[0])))
self.critic_target.to(torch.device('cuda:{}'.format(self.gpu_ids[0])))
def observe(self, r_t, s_t1, done):
if self.is_training:
self.memory.append(self.s_t, self.a_t, r_t, done)
self.s_t = s_t1
def random_action(self):
action = np.random.uniform(-1.,1.,self.nb_actions)
# self.a_t = action
return action
def select_action(self, s_t, decay_epsilon=True):
# proto action
action = to_numpy(
self.actor(to_tensor(np.array([s_t]), gpu_used=self.gpu_used, gpu_0=self.gpu_ids[0])),
gpu_used=self.gpu_used
).squeeze(0)
action += self.is_training * max(self.epsilon, 0) * self.random_process.sample()
action = np.clip(action, -1., 1.)
if decay_epsilon:
self.epsilon -= self.depsilon
# self.a_t = action
return action
def reset(self, s_t):
self.s_t = s_t
self.random_process.reset_states()
def load_weights(self, dir):
if dir is None: return
if self.gpu_used:
# load all tensors to GPU (gpu_id)
ml = lambda storage, loc: storage.cuda(self.gpu_ids)
else:
# load all tensors to CPU
ml = lambda storage, loc: storage
self.actor.load_state_dict(
torch.load('output/{}/actor.pkl'.format(dir), map_location=ml)
)
self.critic.load_state_dict(
torch.load('output/{}/critic.pkl'.format(dir), map_location=ml)
)
print('model weights loaded')
def save_model(self,output):
if len(self.gpu_ids) == 1 and self.gpu_ids[0] > 0:
with torch.cuda.device(self.gpu_ids[0]):
torch.save(
self.actor.state_dict(),
'{}/actor.pt'.format(output)
)
torch.save(
self.critic.state_dict(),
'{}/critic.pt'.format(output)
)
elif len(self.gpu_ids) > 1:
torch.save(self.actor.module.state_dict(),
'{}/actor.pt'.format(output)
)
torch.save(self.actor.module.state_dict(),
'{}/critic.pt'.format(output)
)
else:
torch.save(
self.actor.state_dict(),
'{}/actor.pt'.format(output)
)
torch.save(
self.critic.state_dict(),
'{}/critic.pt'.format(output)
)
def seed(self,seed):
torch.manual_seed(seed)
if len(self.gpu_ids) > 0:
torch.cuda.manual_seed_all(seed)
class DDPG(object):
def __init__(self, env, config):
self.name = 'HierarchicalNet'
self.save_folder = None
self.test_record = {}
self.train_record = {}
self.config = config
self.env = env
self.epsilon = config.EPSILON
self.commander_memory = Commander_Memory(config.MEMORY_SIZE,config.BATCH_SIZE)
self.unit_memory = Unit_Memory(2*config.MEMORY_SIZE,config.UNIT_BATCH_SIZE)
self.commander_actor = Commander_Actor(config.STATE_DIM,config.COMMAND_DIM,config.RNN_INSIZE)
self.commander_actor_target = Commander_Actor(config.STATE_DIM,config.COMMAND_DIM,config.RNN_INSIZE)
self.commander_critic = Commander_Critic(config.STATE_DIM,config.COMMAND_DIM,config.BATCH_SIZE,config.RNN_INSIZE)
self.commander_critic_target = Commander_Critic(config.STATE_DIM,config.COMMAND_DIM,config.BATCH_SIZE,config.RNN_INSIZE)
self.unit_actor = Unit_Actor(config.STATE_DIM,config.COMMAND_DIM,config.ACTION_DIM)
self.unit_actor_target = Unit_Actor(config.STATE_DIM,config.COMMAND_DIM,config.ACTION_DIM)
self.unit_critic = Unit_Critic(config.STATE_DIM,config.COMMAND_DIM,config.ACTION_DIM,config.HIDDEN_SIZE)
self.unit_critic_target = Unit_Critic(config.STATE_DIM,config.COMMAND_DIM,config.ACTION_DIM,config.HIDDEN_SIZE)
self.commander_actor_h0 = Variable(torch.zeros(2, 1, config.RNN_OUTSIZE),requires_grad=False)
if config.GPU >= 0:
self.commander_actor.cuda(device=config.GPU)
self.commander_actor_target.cuda(device=config.GPU)
self.commander_critic.cuda(device=config.GPU)
self.commander_critic_target.cuda(device=config.GPU)
self.unit_actor.cuda(device=config.GPU)
self.unit_actor_target.cuda(device=config.GPU)
self.unit_critic.cuda(device=config.GPU)
self.unit_critic_target.cuda(device=config.GPU)
self.commander_critic.h0 = self.commander_critic.h0.cuda(device=config.GPU)
self.commander_critic_target.h0 = self.commander_critic_target.h0.cuda(device=config.GPU)
self.commander_actor_h0 = self.commander_actor_h0.cuda(device=config.GPU)
copy_parameter(self.commander_actor, self.commander_actor_target)
copy_parameter(self.commander_critic, self.commander_critic_target)
copy_parameter(self.unit_actor, self.unit_actor_target)
copy_parameter(self.unit_critic, self.unit_critic_target)
self.commander_actor_optimizer = optim.Adam(self.commander_actor.parameters(),lr=config.ACTOR_LR)
self.unit_actor_optimizer = optim.Adam(self.unit_actor.parameters(),lr=config.ACTOR_LR)
self.commander_critic_optimizer = optim.Adam(self.commander_critic.parameters(), lr=config.CRITIC_LR)
self.unit_critic_optimizer = optim.Adam(self.unit_critic.parameters(), lr=config.CRITIC_LR)
self.criterion = nn.MSELoss()
self.action_noise = OrnsteinUhlenbeckProcess(size=(config.MYSELF_NUM, config.ACTION_DIM), theta=10, mu=0., sigma=2)
self.command_noise = OrnsteinUhlenbeckProcess(size=(1,config.MYSELF_NUM, config.COMMAND_DIM), theta=10, mu=0., sigma=2)
# self.action_noise = OrnsteinUhlenbeckProcess(size=(config.MYSELF_NUM, config.ACTION_DIM), theta=30, mu=0., sigma=3)
# self.command_noise = OrnsteinUhlenbeckProcess(size=(1,config.MYSELF_NUM, config.COMMAND_DIM), theta=30, mu=0., sigma=3)
# normalize
state_normalization_myelf = [1,100,100,1,100,100,1]
state_normalization_enemy = [1,100,100,100,100,10,100,100,1,1,1,10]
self.state_normalization = state_normalization_myelf
for i in range(config.K):
self.state_normalization += state_normalization_enemy
self.state_normalization = np.asarray(self.state_normalization,dtype=np.float32)
def append_memory(self,states,commands,actions,next_states,rewards,dones):
self.commander_memory.append(states,commands,next_states,rewards,[dones]*self.config.MYSELF_NUM)
for i in range(self.config.MYSELF_NUM):
self.unit_memory.append(states[i],commands[i],actions[i],next_states[i],rewards[i],dones)
def get_command(self,states):
group_states = states.view(int(self.config.UNIT_BATCH_SIZE/8),8,self.config.STATE_DIM)
group_states.volatile = True
command = self.commander_actor_target(group_states,self.commander_actor_h0.repeat(1,int(self.config.UNIT_BATCH_SIZE/8),1)).contiguous()
command = command.view(self.config.UNIT_BATCH_SIZE,self.config.COMMAND_DIM)
command.volatile = False
command.requires_grad = False
return command
def train_commander(self):
# if not warm up
if self.commander_memory.counter < self.config.WARMUP:
return 0,0,0,0
state_batch, command_batch, next_state_batch, reward_batch, done_batch = self.commander_memory.sample()
sb = Variable(torch.from_numpy(state_batch)).float()
cb = Variable(torch.from_numpy(command_batch)).float()
nsb = Variable(torch.from_numpy(next_state_batch)).float()
rb = Variable(torch.from_numpy(reward_batch)).float()
db = Variable(torch.from_numpy(done_batch)).float()
if self.config.GPU>=0:
sb = sb.cuda(device=self.config.GPU)
cb = cb.cuda(device=self.config.GPU)
nsb = nsb.cuda(device=self.config.GPU)
rb = rb.cuda(device=self.config.GPU)
db = db.cuda(device=self.config.GPU)
# update critic
self.commander_critic_optimizer.zero_grad()
ncb = self.commander_actor_target(nsb,self.commander_actor_h0.repeat(1,self.config.BATCH_SIZE,1))
nqb = self.commander_critic_target(nsb, ncb)
q_target = (rb + self.config.GAMMA * db * nqb)
q_eval = self.commander_critic(sb, cb)
q_target = q_target.detach()
value_loss = F.mse_loss(q_eval, q_target)
value_loss.backward()
self.commander_critic_optimizer.step()
# update actor
self.commander_actor_optimizer.zero_grad()
acb = self.commander_actor(sb,self.commander_actor_h0.repeat(1,self.config.BATCH_SIZE,1))
q = self.commander_critic(sb, acb)
policy_loss = -torch.mean(q)
policy_loss.backward()
self.commander_actor_optimizer.step()
# update parameter between two network
update_parameter(self.commander_critic_target, self.commander_critic, self.config.TAN)
update_parameter(self.commander_actor_target, self.commander_actor, self.config.TAN)
# # update priorty
# tderror = torch.mean(torch.abs(q_target-q_eval),1).cpu().data.numpy()
# for i,id in enumerate(idxs):
# self.memory.update_priorty(id,float(tderror[i]))
return value_loss.data[0],policy_loss.data[0],torch.mean(q_eval).data[0],torch.mean(q_target).data[0]
def train_unit(self):
# if not warm up
if self.unit_memory.counter < 10*self.config.WARMUP:
return 0,0,0,0
state_batch, command_batch, action_batch, next_state_batch, reward_batch, done_batch = self.unit_memory.sample()
sb = Variable(torch.from_numpy(state_batch)).float()
cb = Variable(torch.from_numpy(command_batch)).float()
ab = Variable(torch.from_numpy(action_batch)).float()
nsb = Variable(torch.from_numpy(next_state_batch)).float()
rb = Variable(torch.from_numpy(reward_batch)).float()
db = Variable(torch.from_numpy(done_batch)).float()
rb = rb.view(self.config.UNIT_BATCH_SIZE,1)
db = db.view(self.config.UNIT_BATCH_SIZE,1)
if self.config.GPU>=0:
sb = sb.cuda(device=self.config.GPU)
cb = cb.cuda(device=self.config.GPU)
ab = ab.cuda(device=self.config.GPU)
nsb = nsb.cuda(device=self.config.GPU)
rb = rb.cuda(device=self.config.GPU)
db = db.cuda(device=self.config.GPU)
ncb = self.get_command(nsb)
# update critic
self.unit_critic_optimizer.zero_grad()
nab = self.unit_actor_target(nsb,ncb)
nqb = self.unit_critic_target(nsb, ncb, nab)
q_target = (rb + self.config.GAMMA * db * nqb)
q_eval = self.unit_critic(sb,cb,ab)
q_target = q_target.detach()
value_loss = F.mse_loss(q_eval, q_target)
value_loss.backward()
self.unit_critic_optimizer.step()
# update actor
self.unit_actor_optimizer.zero_grad()
aab = self.unit_actor(sb,cb)
q = self.unit_critic(sb, cb, aab)
policy_loss = -torch.mean(q)
policy_loss.backward()
self.unit_actor_optimizer.step()
# update parameter between two network
update_parameter(self.unit_critic_target, self.unit_critic, self.config.TAN)
update_parameter(self.unit_actor_target, self.unit_actor, self.config.TAN)
# # update priorty
# tderror = torch.mean(torch.abs(q_target-q_eval),1).cpu().data.numpy()
# for i,id in enumerate(idxs):
# self.memory.update_priorty(id,float(tderror[i]))
return value_loss.data[0],policy_loss.data[0],torch.mean(q_eval).data[0],torch.mean(q_target).data[0]
def select_action(self, s, is_train=True, decay_e=True,ignor_warmup=False):
'''
:param is_train: if true, action += noise
:param decay_e: if true, decay epsilon every step
:param ignor_warmup: if true, select op will not wait for warmup
:return: action
'''
state = Variable(torch.from_numpy(s),volatile=True).unsqueeze(0)
if self.config.GPU >= 0:
state = state.cuda(device=self.config.GPU)
self.commander_actor.eval()
self.unit_actor.eval()
if self.commander_memory.counter < self.config.WARMUP and is_train and not ignor_warmup:
command = Variable(torch.from_numpy(np.random.uniform(-1, 1, (1,self.config.MYSELF_NUM, self.config.COMMAND_DIM))),volatile=True).float()
if self.config.GPU >= 0:
command = command.cuda(device=self.config.GPU)
else:
command = self.commander_actor(state, self.commander_actor_h0)
command_noise = Variable(torch.from_numpy(self.command_noise.sample())).float()
if self.config.GPU >= 0:
command_noise = command_noise.cuda(device=self.config.GPU)
command += is_train * max(self.epsilon, 0.2) * command_noise
command = command.clamp(-1,1)
actions = []
for i in range(self.config.MYSELF_NUM):
c = command[:,i]
s = state[:,i]
a = self.unit_actor(s,c)
actions.append(a)
actions = torch.cat(actions,0).cpu().data.numpy()
action_noise = self.action_noise.sample()
actions += is_train * max(self.epsilon, 0.4) * action_noise
actions = np.clip(actions, -1., 1.)
if decay_e:
if self.commander_memory.counter > self.config.WARMUP:
self.epsilon -= self.config.EPSILON_DECAY
self.commander_actor.train()
self.unit_actor.train()
return actions,command.cpu().data.numpy()[0]
def extract_state(self, obs):
'''
extract state info from obs
:param obs:
:return: state
'''
# enemy basic :[die, health, shield, x, y, delta_health, attackCD, targetX, targetY]
# enemy add :[attack this myself_agent,if closest,dx,dy,distance]
enemy_basic_len = 8
enemy_add_len = 4
assert self.config.ENEMY_FEATURE == enemy_basic_len+enemy_add_len
myself_units_state = []
units_enemy = {}
myself_units_underattack = []
for index, unit in enumerate(obs['enemy']):
state = [unit.die,unit.health, unit.shield,unit.x, unit.y,unit.attackCD,unit.targetX,unit.targetY,unit.targetUnitId]
units_enemy[unit.id] = state
myself_units_underattack.append(unit.targetUnitId)
myself_units_underattack = set(myself_units_underattack)
for index, unit in enumerate(obs['myself']):
state = [unit.die,unit.health,unit.shield, unit.health > (0.2 * unit.max_health), unit.x, unit.y,int(unit.id in myself_units_underattack)]
# enemy_state
nearly_enemy_ids,num = self.nearyl_topK(unit,obs['enemy'],k=self.config.K)
if num == 0:
enemy_state = [0]*self.config.K*(enemy_basic_len+enemy_add_len)
else:
enemy_state = []
for idx,enemy_id in enumerate(nearly_enemy_ids):
# if there no enough alive enemy, chose the farthest one
if enemy_id == -1:
enemy_id = nearly_enemy_ids[num-1]
es = units_enemy[enemy_id]
if unit.die:
enemy_state_add = [0,255/self.config.DISTANCE_FACTOR,255/self.config.DISTANCE_FACTOR,255*1.4]
else:
under_attack = int(es[-1]==unit.id)
dx,dy = es[3]-unit.x,es[4]-unit.y
distance = math.sqrt(dx**2+dy**2)
enemy_state_add = [under_attack,dx/self.config.DISTANCE_FACTOR,dy/self.config.DISTANCE_FACTOR,distance]
enemy_state.extend(es[:-1]+enemy_state_add)
# cat to state
state.extend(enemy_state)
myself_units_state.append(state)
myself_units_state = np.asarray(myself_units_state,dtype=np.float32)
myself_units_state = myself_units_state/self.state_normalization
assert myself_units_state.shape == (self.config.MYSELF_NUM,self.config.STATE_DIM)
# eunit = obs['enemy'][0]
# ex,ey=eunit.x,eunit.y
# dist = []
# for unit in obs['myself']:
# x,y = unit.x,unit.y
# dist.append(math.sqrt((ex-x)**2+(ey-y)**2))
# print(dist)
return myself_units_state
def nearyl_topK(self,unit,enemy_units,k=3):
x,y = unit.x,unit.y
nearly_enemys = {}
for enemy in enemy_units:
if not enemy.die:
uid = enemy.id
d = math.sqrt((enemy.x-x)**2+(enemy.y-y)**2)
if len(nearly_enemys)<k:
nearly_enemys[uid] = d
else:
max_id, max_d = max(nearly_enemys.items(), key=operator.itemgetter(1))
if d<max_d:
del nearly_enemys[max_id]
nearly_enemys[uid] = d
sorted_nearly_enemys = sorted(nearly_enemys.items(),key=lambda x:x[1])
nearly_enemys_id = [id for id,d in sorted_nearly_enemys ]
num = len(nearly_enemys)
return nearly_enemys_id+[-1]*(k-num),num
def test(self, eposide, test_num, ):
'''
test model, without noise
:return: mean test eposide_total_reward
'''
total_reward = 0
win = 0
for i in range(test_num):
obs = self.env.reset()
state = self.extract_state(obs)
for test_step in range(self.config.MAX_STEP):
# print(test_step)
action,command = self.select_action(state, is_train=False, decay_e=False)
next_obs, reward, done, info = self.env.step(action)
next_state = self.extract_state(next_obs)
time.sleep(0.02)
if done:
if self.env.win:
win += 1
break
total_reward += sum(reward)
state = next_state
win_rate = win/test_num
total_reward = total_reward/(self.config.MYSELF_NUM*test_num)
self.test_record[eposide] = total_reward
return total_reward,win,win_rate
def save(self, episode):
'''
save model, hyperparameters, test record
:param episode:
:return:
'''
timestr = time.strftime('(%m-%d_%H:%M)', time.localtime(time.time()))
mapname = '({})'.format(self.env.getMapName())
if self.save_folder is None:
self.save_folder = self.name + mapname + timestr + self.config.NOTE
if not os.path.exists(self.save_folder):
os.mkdir(self.save_folder)
self.config.MAP = self.env.getMapName()
with open(os.path.join(self.save_folder, 'config'), 'w') as f:
f.write('config:\n\n')
for k, v in sorted(self.config.todict().items()):
f.write('{:<16} : {}\n'.format(k, v))
with open(os.path.join(self.save_folder, 'config.pkl'), 'wb') as f:
pickle.dump(self.config, f)
torch.save(self.commander_actor.cpu(), os.path.join(self.save_folder, 'commander_actor_%d.mod' % episode))
torch.save(self.commander_critic.cpu(), os.path.join(self.save_folder, 'commander_critic_%d.mod' % episode))
torch.save(self.unit_actor.cpu(), os.path.join(self.save_folder, 'unit_actor_%d.mod' % episode))
torch.save(self.unit_critic.cpu(), os.path.join(self.save_folder, 'unit_critic_%d.mod' % episode))
with open(os.path.join(self.save_folder, 'test_record.pkl'), 'wb') as f:
pickle.dump(self.test_record, f)
with open(os.path.join(self.save_folder, 'train_record.pkl'), 'wb') as f:
pickle.dump(self.train_record, f)
# self.commander_memory.save(self.save_folder)
# self.unit_memory.save(self.save_folder)
if self.config.GPU >= 0:
self.commander_actor.cuda(device=self.config.GPU)
self.commander_critic.cuda(device=self.config.GPU)
self.unit_actor.cuda(device=self.config.GPU)
self.unit_critic.cuda(device=self.config.GPU)
def load(self, saved_folder, episode):
# load network
self.commander_actor = torch.load(os.path.join(saved_folder, 'commander_actor_{}.mod'.format(episode)))
self.commander_actor_target = torch.load(os.path.join(saved_folder, 'commander_actor_{}.mod'.format(episode)))
self.commander_critic = torch.load(os.path.join(saved_folder, 'commander_critic_{}.mod'.format(episode)))
self.commander_critic_target = torch.load(os.path.join(saved_folder, 'commander_critic_{}.mod'.format(episode)))
self.unit_actor = torch.load(os.path.join(saved_folder, 'unit_actor_{}.mod'.format(episode)))
self.unit_actor_target = torch.load(os.path.join(saved_folder, 'unit_actor_{}.mod'.format(episode)))
self.unit_critic = torch.load(os.path.join(saved_folder, 'unit_critic_{}.mod'.format(episode)))
self.unit_critic_target = torch.load(os.path.join(saved_folder, 'unit_critic_{}.mod'.format(episode)))
if self.config.GPU >= 0:
self.commander_actor.cuda(device=self.config.GPU)
self.commander_actor_target.cuda(device=self.config.GPU)
self.commander_critic.cuda(device=self.config.GPU)
self.commander_critic_target.cuda(device=self.config.GPU)
self.unit_actor.cuda(device=self.config.GPU)
self.unit_actor_target.cuda(device=self.config.GPU)
self.unit_critic.cuda(device=self.config.GPU)
self.unit_critic_target.cuda(device=self.config.GPU)
self.commander_critic_optimizer = optim.Adam(self.commander_critic.parameters(), lr=self.config.CRITIC_LR)
self.commander_actor_optimizer = optim.Adam(self.commander_actor.parameters(), lr=self.config.ACTOR_LR)
self.unit_critic_optimizer = optim.Adam(self.unit_critic.parameters(), lr=self.config.CRITIC_LR)
self.unit_actor_optimizer = optim.Adam(self.unit_actor.parameters(), lr=self.config.ACTOR_LR)
# load memory
# self.commander_memory = Commander_Memory.memory_load(saved_folder)
# self.unit_memory = Unit_Memory.memory_load(saved_folder)
# # load record
# with open(os.path.join(self.save_folder, 'test_record.pkl'), 'rb') as f:
# self.test_record = pickle.load(f)
# with open(os.path.join(self.save_folder, 'train_record.pkl'), 'rb') as f:
# self.train_record = pickle.load(f)
# for r in self.test_record:
# if r[0]>episode:
# self.test_record.remove(r)
# for r in self.train_record:
# if r[0]>episode:
# self.train_record.remove(r)
def print_action(self,action):
env_cmd = self.env._make_commands(action)
ecs = []
for ec in env_cmd:
if len(ec) > 2:
if ec[2] == tcc.unitcommandtypes.Attack_Unit:
c = [ec[1], 'A', ec[3]]
else:
c = [ec[1], 'M', ec[4], ec[5]]
print(c)
ecs.append(c)
return ecs
def run_agent(args,
model_params,
weights,
data_queue,
weights_queue,
process,
global_step,
updates,
best_reward,
param_noise_prob,
save_dir,
max_steps=10000000):
train_fn, actor_fn, target_update_fn, params_actor, params_crit, actor_lr, critic_lr = build_model(
**model_params)
actor = Agent(actor_fn, params_actor, params_crit)
actor.set_actor_weights(weights)
env = RunEnv2(model=args.modeldim,
prosthetic=args.prosthetic,
difficulty=args.difficulty,
skip_frame=config.skip_frames)
env.spec.timestep_limit = 3000 # ndrw
# random_process = OrnsteinUhlenbeckProcess(theta=.1, mu=0., sigma=.3, size=env.noutput, sigma_min=0.05, n_steps_annealing=1e6)
sigma_rand = random.uniform(0.05, 0.5)
dt_rand = random.uniform(0.002, 0.02)
param_noise_prob = random.uniform(param_noise_prob * 0.25,
min(param_noise_prob * 1.5, 1.))
random_process = OrnsteinUhlenbeckProcess(theta=.1,
mu=0.,
sigma=sigma_rand,
dt=dt_rand,
size=env.noutput,
sigma_min=0.05,
n_steps_annealing=1e6)
print('OUProcess_sigma = ' + str(sigma_rand) + ' OUProcess_dt = ' +
str(dt_rand) + ' param_noise_prob = ' + str(param_noise_prob))
# prepare buffers for data
states = []
actions = []
rewards = []
terminals = []
total_episodes = 0
start = time()
action_noise = True
while global_step.value < max_steps:
seed = random.randrange(2**32 - 2)
state = env.reset(seed=seed, difficulty=args.difficulty)
random_process.reset_states()
total_reward = 0.
total_reward_original = 0.
terminal = False
steps = 0
while not terminal:
state = np.asarray(state, dtype='float32')
action = actor.act(state)
if action_noise:
action += random_process.sample()
next_state, reward, next_terminal, info = env._step(action)
total_reward += reward
total_reward_original += info['original_reward']
steps += 1
global_step.value += 1
# add data to buffers
states.append(state)
actions.append(action)
rewards.append(reward)
terminals.append(terminal)
state = next_state
terminal = next_terminal
if terminal:
break
total_episodes += 1
# add data to buffers after episode end
states.append(state)
actions.append(np.zeros(env.noutput))
rewards.append(0)
terminals.append(terminal)
states_np = np.asarray(states).astype(np.float32)
data = (
states_np,
np.asarray(actions).astype(np.float32),
np.asarray(rewards).astype(np.float32),
np.asarray(terminals),
)
weight_send = None
if total_reward > best_reward.value:
weight_send = actor.get_actor_weights()
# send data for training
data_queue.put((process, data, weight_send, total_reward))
# receive weights and set params to weights
weights = weights_queue.get()
# report_str = 'Global step: {}, steps/sec: {:.2f}, updates: {}, episode len: {}, pelvis_X: {:.2f}, reward: {:.2f}, original_reward {:.4f}, best reward: {:.2f}, noise: {}'. \
# format(global_step.value, 1. * global_step.value / (time() - start), updates.value, steps, info['pelvis_X'], total_reward, total_reward_original, best_reward.value, 'actions' if action_noise else 'params')
# report_str = 'Global step: {}, steps/sec: {:.2f}, updates: {}, episode len: {}, pelvis_X: {:.2f}, reward: {:.2f}, best reward: {:.2f}, noise: {}'. \
# format(global_step.value, 1. * global_step.value / (time() - start), updates.value, steps, info['pelvis_X'], total_reward, best_reward.value, 'actions' if action_noise else 'params')
report_str = 'Global step: {}, steps/sec: {:.2f}, updates: {}, episode len: {}, pelvis_X: {:.2f}, pelvis_Z: {:.2f}, reward: {:.2f}, best reward: {:.2f}, noise: {}'. \
format(global_step.value, 1. * global_step.value / (time() - start), updates.value, steps, info['pelvis'][0], info['pelvis'][2], total_reward, best_reward.value, 'actions' if action_noise else 'params')
print(report_str)
try:
with open(os.path.join(save_dir, 'train_report.log'), 'a') as f:
f.write(report_str + '\n')
except:
print('#############################################')
print(
'except » with open(os.path.join(save_dir, train_report.log), a) as f:'
)
print('#############################################')
actor.set_actor_weights(weights)
action_noise = np.random.rand() < 1 - param_noise_prob
if not action_noise:
set_params_noise(actor, states_np, random_process.current_sigma)
# clear buffers
del states[:]
del actions[:]
del rewards[:]
del terminals[:]
if total_episodes % 100 == 0:
env = RunEnv2(model=args.modeldim,
prosthetic=args.prosthetic,
difficulty=args.difficulty,
skip_frame=config.skip_frames)
class UADDPG(object):
def __init__(self, nb_states, nb_actions, args):
if args.seed > 0:
self.seed(args.seed)
self.nb_states = nb_states
self.nb_actions = nb_actions
self.epistemic_actor = args.epistemic_actor # true / false
self.epistemic_critic = args.epistemic_critic # true / false
self.aleatoric_actor = args.aleatoric_actor # true / false
self.aleatoric_critic = args.aleatoric_critic # true / false
self.dropout_n_actor = args.dropout_n_actor
self.dropout_n_critic = args.dropout_n_critic
self.dropout_p_actor = args.dropout_p_actor
self.dropout_p_critic = args.dropout_p_critic
self.print_var_count = 0
self.action_std = np.array([])
self.save_dir = args.output
self.episode = 0
# self.save_file = open(self.save_dir + '/std.txt', "a")
# Create Actor and Critic Network
net_cfg_actor = {
'dropout_n': args.dropout_n_actor,
'dropout_p': args.dropout_p_actor,
'hidden1': args.hidden1,
'hidden2': args.hidden2,
'init_w': args.init_w
}
net_cfg_critic = {
'dropout_n': args.dropout_n_actor,
'dropout_p': args.dropout_p_critic,
'hidden1': args.hidden1,
'hidden2': args.hidden2,
'init_w': args.init_w
}
self.actor = UAActor(self.nb_states, self.nb_actions, **net_cfg_actor)
self.actor_target = UAActor(self.nb_states, self.nb_actions,
**net_cfg_actor)
self.actor_optim = Adam(self.actor.parameters(), lr=args.prate)
self.critic = UACritic(self.nb_states, self.nb_actions,
**net_cfg_critic)
self.critic_target = UACritic(self.nb_states, self.nb_actions,
**net_cfg_critic)
self.critic_optim = Adam(self.critic.parameters(), lr=args.rate)
hard_update(self.actor_target, self.actor)
hard_update(self.critic_target, self.critic)
# Create replay buffer
self.memory = SequentialMemory(limit=args.rmsize,
window_length=args.window_length)
self.random_process = OrnsteinUhlenbeckProcess(size=nb_actions,
theta=args.ou_theta,
mu=args.ou_mu,
sigma=args.ou_sigma)
# Hyper-parameters
self.batch_size = args.bsize
self.tau = args.tau
self.discount = args.discount
self.depsilon = 1.0 / args.epsilon
#
self.epsilon = 1.0
self.s_t = None # Most recent state
self.a_t = None # Most recent action
self.is_training = True
#
if USE_CUDA:
self.cuda()
def update_policy(self):
# Sample batch
state_batch, action_batch, reward_batch, next_state_batch, terminal_batch = self.memory.sample_and_split(
self.batch_size)
# Prepare for the target q batch
# TODO : Also apply epistemic and aleatoric uncertainty to both actor and critic target network
next_q_values = self.critic_target([
to_tensor(next_state_batch, volatile=True),
self.actor_target(to_tensor(next_state_batch, volatile=True)),
])
target_q_batch = to_tensor(reward_batch) + self.discount * to_tensor(
terminal_batch.astype(np.float)) * next_q_values
#########################
# Critic update
#########################
self.critic.zero_grad()
# TODO : Add epistemic uncertainty for critic network
q_batch = self.critic(
[to_tensor(state_batch),
to_tensor(action_batch)])
# TODO : Add aleatoric uncertainty term from aleatoric uncertainty output of critic network (Add uncertainty term in criterion)
value_loss = criterion(q_batch, target_q_batch)
value_loss.backward()
self.critic_optim.step()
#########################
# Actor update
#########################
self.actor.zero_grad()
# policy loss
# TODO : Add epistemic certainty term from aleatoric certainty output of policy network
policy_loss = -self.critic(
[to_tensor(state_batch),
self.actor(to_tensor(state_batch))])
policy_loss = policy_loss.mean()
# policy_loss = policy_loss.mean() + actor_certainty
policy_loss.backward()
self.actor_optim.step()
#########################
# Target soft update
#########################
soft_update(self.actor_target, self.actor, self.tau)
soft_update(self.critic_target, self.critic, self.tau)
def eval(self):
self.actor.eval()
self.actor_target.eval()
self.critic.eval()
self.critic_target.eval()
def cuda(self):
self.actor.cuda()
self.actor_target.cuda()
self.critic.cuda()
self.critic_target.cuda()
def observe(self, r_t, s_t1, done):
if self.is_training:
self.memory.append(self.s_t, self.a_t, r_t, done)
self.s_t = s_t1
def random_action(self):
action = np.random.uniform(-1., 1., self.nb_actions)
self.a_t = action
return action
# def select_action(self, s_t, decay_epsilon=True):
# action = to_numpy(self.actor(to_tensor(np.array([s_t])))).squeeze(0)
# action += self.is_training*max(self.epsilon, 0)*self.random_process.sample()
#
# if decay_epsilon:
# self.epsilon -= self.depsilon
#
# self.a_t = action
# return action
def select_action_with_dropout(self, s_t, decay_epsilon=True):
dropout_actions = np.array([])
with torch.no_grad():
for _ in range(self.dropout_n):
action = to_numpy(
self.actor.forward_with_dropout(to_tensor(np.array(
[s_t])))).squeeze(0)
dropout_actions = np.append(dropout_actions, [action])
if self.train_with_dropout:
plt_action = to_numpy(
self.actor.forward_with_dropout(to_tensor(np.array(
[s_t])))).squeeze(0)
plt_action += self.is_training * max(
self.epsilon, 0) * self.random_process.sample()
else:
plt_action = to_numpy(self.actor(to_tensor(np.array(
[s_t])))).squeeze(0)
plt_action += self.is_training * max(
self.epsilon, 0) * self.random_process.sample()
"""
UNFIXED RESET POINT for Mujoco
"""
if self.print_var_count != 0 and (self.print_var_count + 1) % 999 == 0:
# self.action_std = np.append(self.action_std, [np.std(dropout_actions)])
with open(self.save_dir + "/std.txt", "a") as myfile:
myfile.write(str(np.std(dropout_actions)) + '\n')
with open(self.save_dir + "/mean.txt", "a") as myfile:
myfile.write(str(np.mean(dropout_actions)) + '\n')
if self.print_var_count % (1000 * 5) == 0:
print("dropout actions std", np.std(dropout_actions),
" ", "dir : ", str(self.save_dir))
"""
FIXED RESET POINT for MCC
"""
# if s_t[0] == -0.5 and s_t[1] == 0:
# # print("fixed dropout actions std", np.std(dropout_actions), " ", "dir : ", str(self.save_dir))
# self.action_std = np.append(self.action_std, [np.std(dropout_actions)])
# # np.savetxt(self.save_dir + '/std.txt', self.action_std, fmt='%4.10f', delimiter=' ')
# with open(self.save_dir + "/std.txt", "a") as myfile:
# myfile.write(str(np.std(dropout_actions))+'\n')
# with open(self.save_dir + "/mean.txt", "a") as myfile:
# myfile.write(str(np.mean(dropout_actions))+'\n')
if not (os.path.isdir(self.save_dir + "/episode/" +
str(self.episode))):
os.makedirs(
os.path.join(self.save_dir + "/episode/" + str(self.episode)))
self.action_std = np.append(self.action_std, [np.std(dropout_actions)])
with open(self.save_dir + "/episode/" + str(self.episode) + "/std.txt",
"a") as myfile:
myfile.write(str(np.std(dropout_actions)) + '\n')
with open(
self.save_dir + "/episode/" + str(self.episode) + "/mean.txt",
"a") as myfile:
myfile.write(str(np.mean(dropout_actions)) + '\n')
self.print_var_count = self.print_var_count + 1
if decay_epsilon:
self.epsilon -= self.depsilon
self.a_t = plt_action
return plt_action
def reset(self, obs):
self.s_t = obs
self.random_process.reset_states()
def load_weights(self, output):
if output is None: return
self.actor.load_state_dict(torch.load('{}/actor.pkl'.format(output)))
self.critic.load_state_dict(torch.load('{}/critic.pkl'.format(output)))
def save_model(self, output):
torch.save(self.actor.state_dict(), '{}/actor.pkl'.format(output))
torch.save(self.critic.state_dict(), '{}/critic.pkl'.format(output))
def seed(self, s):
torch.manual_seed(s)
if USE_CUDA:
torch.cuda.manual_seed(s)
class DDPG(object):
def __init__(self, nb_states, nb_actions):
self.nb_states = nb_states
self.nb_actions = nb_actions
# Create Actor and Critic Network
self.actor = Actor(self.nb_states, self.nb_actions)
self.actor_target = Actor(self.nb_states, self.nb_actions)
self.actor_optim = Adam(self.actor.parameters(), lr=ACTOR_LR)
self.critic = Critic(self.nb_states, self.nb_actions)
self.critic_target = Critic(self.nb_states, self.nb_actions)
self.critic_optim = Adam(self.critic.parameters(), lr=CRITIC_LR)
hard_update(self.actor_target,
self.actor) # Make sure target is with the same weight
hard_update(self.critic_target, self.critic)
#Create replay buffer
self.memory = SequentialMemory(limit=MEMORY_SIZE,
window_length=HISTORY_LEN)
self.random_process = OrnsteinUhlenbeckProcess(size=nb_actions,
theta=OU_THETA,
mu=OU_MU,
sigma=OU_SIGMA)
# Hyper-parameters
self.batch_size = BATCH_SIZE
self.tau = TAU
self.discount = GAMMA
self.depsilon = 1.0 / DEPSILON
self.epsilon = 1.0
self.s_t = None # Most recent state
self.a_t = None # Most recent action
self.is_training = True
if USE_CUDA: self.cuda()
def update_policy(self):
# Sample batch
state_batch, action_batch, reward_batch, \
next_state_batch, terminal_batch = self.memory.sample_and_split(self.batch_size)
# Prepare for the target q batch
next_q_values = self.critic_target([
to_tensor(next_state_batch, volatile=True),
self.actor_target(to_tensor(next_state_batch, volatile=True)),
])[:, 0]
next_q_values.volatile = False
target_q_batch = to_tensor(reward_batch) + \
self.discount*to_tensor(terminal_batch.astype(np.float))*next_q_values
# Critic update
self.critic.zero_grad()
q_batch = self.critic(
[to_tensor(state_batch),
to_tensor(action_batch)])
value_loss = criterion(q_batch, target_q_batch)
value_loss.backward()
torch.nn.utils.clip_grad_norm(self.critic.parameters(), 10.0)
for p in self.critic.parameters():
p.data.add_(-CRITIC_LR, p.grad.data)
self.critic_optim.step()
# Actor update
self.actor.zero_grad()
policy_loss = -self.critic(
[to_tensor(state_batch),
self.actor(to_tensor(state_batch))])
policy_loss = policy_loss.mean()
policy_loss.backward()
torch.nn.utils.clip_grad_norm(self.actor.parameters(), 10.0)
for p in self.actor.parameters():
p.data.add_(-ACTOR_LR, p.grad.data)
self.actor_optim.step()
# Target update
soft_update(self.actor_target, self.actor, self.tau)
soft_update(self.critic_target, self.critic, self.tau)
def eval(self):
self.actor.eval()
self.actor_target.eval()
self.critic.eval()
self.critic_target.eval()
def cuda(self):
self.actor.cuda()
self.actor_target.cuda()
self.critic.cuda()
self.critic_target.cuda()
def observe(self, r_t, s_t1, done):
if self.is_training:
self.memory.append(self.s_t, self.a_t, r_t, done)
self.s_t = s_t1
def random_action(self):
action = np.random.uniform(-1., 1., self.nb_actions)
self.a_t = action
return action
def select_action(self, s_t, decay_epsilon=True):
action = to_numpy(self.actor(to_tensor(np.array([s_t]))))[0]
ou = self.random_process.sample()
prGreen('eps:{}, act:{}, random:{}'.format(self.epsilon, action, ou))
action += self.is_training * max(self.epsilon, 0) * ou
action = np.clip(action, -1., 1.)
if decay_epsilon:
self.epsilon -= self.depsilon
self.a_t = action
return action
def reset(self, obs):
self.s_t = obs
self.random_process.reset_states()
def load_weights(self, output):
if output is None: return
self.actor.load_state_dict(torch.load('{}/actor.pkl'.format(output)))
self.critic.load_state_dict(torch.load('{}/critic.pkl'.format(output)))
def save_model(self, output):
torch.save(self.actor.state_dict(), '{}/actor.pkl'.format(output))
torch.save(self.critic.state_dict(), '{}/critic.pkl'.format(output))
def seed(self, s):
torch.manual_seed(s)
if USE_CUDA:
torch.cuda.manual_seed(s)
class DDPG(object):
def __init__(self, nb_states, nb_actions, args):
if args.seed > 0:
self.seed(args.seed)
self.nb_states = nb_states
self.nb_actions = nb_actions
actor_net_cfg = {
'hidden1': 32,
'hidden2': 32,
'hidden3': 32,
'init_w': args.init_w
}
critic_net_cfg = {
'hidden1': 64,
'hidden2': 64,
'hidden3': 64,
'init_w': args.init_w
}
self.actor = Actor(self.nb_states, self.nb_actions, **actor_net_cfg)
self.actor_target = Actor(self.nb_states, self.nb_actions,
**actor_net_cfg)
self.actor_optim = Adam(self.actor.parameters(), lr=args.prate)
self.critic = Critic(self.nb_states, self.nb_actions, **critic_net_cfg)
self.critic_target = Critic(self.nb_states, self.nb_actions,
**critic_net_cfg)
self.critic_optim = Adam(self.critic.parameters(), lr=args.rate)
hard_update(self.actor_target,
self.actor) # Make sure target is with the same weight
hard_update(self.critic_target, self.critic)
#Create replay buffer
self.memory = SequentialMemory(limit=args.rmsize,
window_length=args.window_length)
self.random_process = OrnsteinUhlenbeckProcess(size=nb_actions,
theta=args.ou_theta,
mu=args.ou_mu,
sigma=args.ou_sigma)
# Hyper-parameters
self.batch_size = args.bsize
self.tau = args.tau
self.discount = args.discount
self.depsilon = 1.0 / args.epsilon
self.epsilon = 1.0
self.s_t = None # Most recent state
self.a_t = None # Most recent action
self.is_training = True
self.best_reward = -10
def update_policy(self, shared_model, args):
# Sample batch
state_batch, action_batch, reward_batch, \
next_state_batch, terminal_batch = self.memory.sample_and_split(self.batch_size, shared=args.use_more_states, num_states=args.num_states)
# Prepare for the target q batch
next_q_values = self.critic_target([
to_tensor(next_state_batch, volatile=True),
self.actor_target(to_tensor(next_state_batch, volatile=True)),
])
next_q_values.volatile = False
target_q_batch = to_tensor(reward_batch) + \
self.discount*to_tensor(terminal_batch.astype(np.float))*next_q_values
# Critic update
self.critic_optim.zero_grad()
q_batch = self.critic(
[to_tensor(state_batch),
to_tensor(action_batch)])
value_loss = criterion(q_batch, target_q_batch)
value_loss.backward()
if args.shared:
ensure_shared_grads(self.critic, shared_model.critic)
self.critic_optim.step()
# Actor update
self.actor_optim.zero_grad()
policy_loss = -self.critic(
[to_tensor(state_batch),
self.actor(to_tensor(state_batch))])
policy_loss = policy_loss.mean()
policy_loss.backward()
if args.shared:
ensure_shared_grads(self.actor, shared_model.actor)
self.actor_optim.step()
# Target update
soft_update(self.actor_target, self.actor, self.tau)
soft_update(self.critic_target, self.critic, self.tau)
def eval(self):
self.actor.eval()
self.actor_target.eval()
self.critic.eval()
self.critic_target.eval()
def share_memory(self):
self.critic.share_memory()
self.actor.share_memory()
def add_optim(self, actor_optim, critic_optim):
self.actor_optim = actor_optim
self.critic_optim = critic_optim
def observe(self, r_t, s_t1, done):
if self.is_training:
self.memory.append(self.s_t, self.a_t, r_t, done)
self.s_t = s_t1
def update_models(self, agent):
self.actor = deepcopy(agent.actor)
self.actor_target = deepcopy(agent.actor_target)
self.critic = deepcopy(agent.critic)
self.critic_target = deepcopy(agent.critic_target)
self.actor_optim = deepcopy(agent.actor_optim)
self.critic_optim = deepcopy(agent.critic_optim)
def random_action(self):
action = np.random.uniform(-1., 1., self.nb_actions)
self.a_t = action
return action
def train(self):
self.critic.train()
self.actor.train()
def state_dict(self):
return [
self.actor.state_dict(),
self.actor_target.state_dict(),
self.critic.state_dict(),
self.critic_target.state_dict()
]
def load_state_dict(self, list_of_dicts):
self.actor.load_state_dict(list_of_dicts[0])
self.actor_target.load_state_dict(list_of_dicts[1])
self.critic.load_state_dict(list_of_dicts[2])
self.critic_target.load_state_dict(list_of_dicts[3])
def select_action(self, s_t, decay_epsilon=True):
action = to_numpy(self.actor(to_tensor(np.array([s_t])))).squeeze(0)
action += self.is_training * max(self.epsilon,
0) * self.random_process.sample()
action = np.clip(action, -1., 1.)
if decay_epsilon:
self.epsilon -= self.depsilon
self.a_t = action
return action
def reset(self, obs):
self.s_t = obs
self.random_process.reset_states()
def load_weights(self, output):
if output is None: return
self.actor.load_state_dict(torch.load('{}/actor.pkl'.format(output)))
self.critic.load_state_dict(torch.load('{}/critic.pkl'.format(output)))
def save_model(self, output):
torch.save(self.actor.state_dict(), '{}/actor.pkl'.format(output))
torch.save(self.critic.state_dict(), '{}/critic.pkl'.format(output))
def seed(self, s):
torch.manual_seed(s)
class DDPG:
def __init__(self,
env,
actor_model,
critic_model,
memory=10000,
batch_size=64,
gamma=0.99,
tau=0.001,
actor_lr=1e-4,
critic_lr=1e-3,
critic_decay=1e-2,
ou_theta=0.15,
ou_sigma=0.2,
render=None,
evaluate=None,
save_path=None,
save_every=10,
render_every=10,
train_per_step=True):
self.env = env
self.actor = actor_model
self.actor_target = actor_model.clone()
self.critic = critic_model
self.critic_target = critic_model.clone()
if use_cuda:
for net in [
self.actor, self.actor_target, self.critic,
self.critic_target
]:
net.cuda()
self.memory = ReplayMemory(memory)
self.batch_size = batch_size
self.gamma = gamma
self.tau = tau
self.random_process = OrnsteinUhlenbeckProcess(
env.action_space.shape[0], theta=ou_theta, sigma=ou_sigma)
self.optim_critic = optim.Adam(self.critic.parameters(),
lr=critic_lr,
weight_decay=critic_decay)
self.optim_actor = optim.Adam(self.actor.parameters(), lr=actor_lr)
self.render = render
self.render_every = render_every
self.evaluate = evaluate
self.save_path = save_path
self.save_every = save_every
self.train_per_step = train_per_step
def update(self, target, source):
zipped = zip(target.parameters(), source.parameters())
for target_param, source_param in zipped:
updated_param = target_param.data * (1 - self.tau) + \
source_param.data * self.tau
target_param.data.copy_(updated_param)
def train_models(self):
if len(self.memory) < self.batch_size:
return None, None
mini_batch = self.memory.sample_batch(self.batch_size)
critic_loss = self.train_critic(mini_batch)
actor_loss = self.train_actor(mini_batch)
self.update(self.actor_target, self.actor)
self.update(self.critic_target, self.critic)
return critic_loss.data[0], actor_loss.data[0]
def mse(self, inputs, targets):
return torch.mean((inputs - targets)**2)
def train_critic(self, batch):
# forward pass
pred_actions = self.actor_target(batch.next_states)
target_q = batch.rewards + batch.done * self.critic_target(
[batch.next_states, pred_actions]) * self.gamma
pred_q = self.critic([batch.states, batch.actions])
# backward pass
loss = self.mse(pred_q, target_q)
self.optim_critic.zero_grad()
loss.backward(retain_graph=True)
for param in self.critic.parameters():
param.grad.data.clamp_(-1, 1)
self.optim_critic.step()
return loss
def train_actor(self, batch):
# forward pass
pred_mu = self.actor(batch.states)
pred_q = self.critic([batch.states, pred_mu])
# backward pass
loss = -pred_q.mean()
self.optim_actor.zero_grad()
loss.backward()
# for param in self.actor.parameters():
# param.grad.data.clamp_(-1, 1)
self.optim_actor.step()
return loss
def prep_state(self, s):
return Variable(torch.from_numpy(s).float().unsqueeze(0))
def select_action(self, state, exploration=True):
if use_cuda:
state = state.cuda()
self.actor.eval()
action = self.actor(state)
self.actor.train()
if exploration:
noise = Variable(
torch.from_numpy(self.random_process.sample()).float())
if use_cuda:
noise = noise.cuda()
action = action + noise
return action
def step(self, action):
next_state, reward, done, _ = self.env.step(
action.data.cpu().numpy()[0])
next_state = self.prep_state(next_state)
reward = FloatTensor([reward])
return next_state, reward, done
def warmup(self, num_steps):
overall_step = 0
while overall_step <= num_steps:
done = False
state = self.prep_state(self.env.reset())
self.random_process.reset()
while not done:
overall_step += 1
action = self.select_action(state)
next_state, reward, done = self.step(action)
self.memory.add(state, action, reward, next_state, done)
state = next_state
def train(self, num_steps):
running_reward = None
reward_sums = []
losses = []
overall_step = 0
episode_number = 0
while overall_step <= num_steps:
episode_number += 1
done = False
state = self.prep_state(self.env.reset())
reward_sum = 0
self.random_process.reset()
while not done:
overall_step += 1
action = self.select_action(state)
next_state, reward, done = self.step(action)
self.memory.add(state, action, reward, next_state, done)
state = next_state
reward_sum += reward[0]
if self.train_per_step:
losses.append(self.train_models())
if not self.train_per_step:
losses.append(self.train_models())
render_this_episode = self.render and (episode_number %
self.render_every == 0)
evaluation_reward = self.run(render=render_this_episode)
reward_sums.append((reward_sum, evaluation_reward))
if self.save_path is not None and (episode_number % self.save_every
== 0):
self.save_models(self.save_path)
self.save_results(self.save_path, losses, reward_sums)
running_reward = reward_sum if running_reward is None else running_reward * 0.99 + reward_sum * 0.01
print(
'episode: {} steps: {} running train reward: {:.4f} eval reward: {:.4f}'
.format(episode_number, overall_step, running_reward,
evaluation_reward))
if self.save_path is not None:
self.save_models(self.save_path)
self.save_results(self.save_path, losses, reward_sums)
return reward_sums, losses
def run(self, render=True):
state = self.env.reset()
done = False
reward_sum = 0
while not done:
if render:
self.env.render()
action = self.select_action(self.prep_state(state),
exploration=False)
state, reward, done, _ = self.env.step(
action.data.cpu().numpy()[0])
reward_sum += reward
return reward_sum
def save_models(self, path):
self.actor.save(path)
self.critic.save(path)
def save_results(self, path, losses, rewards):
losses = np.array([l for l in losses if l[0] is not None])
rewards = np.array(rewards)
np.savetxt(os.path.join(path, 'losses.csv'),
losses,
delimiter=',',
header='critic,actor',
comments='')
np.savetxt(os.path.join(path, 'rewards.csv'),
rewards,
delimiter=',',
header='train,evaluation',
comments='')
class DDPG(object):
def __init__(self, nb_states, nb_actions, args):
if args.seed > 0:
self.seed(args.seed)
self.nb_states = nb_states
self.nb_actions = nb_actions
# Create Actor and Critic Network
net_cfg = {
'hidden1': args.hidden1,
'hidden2': args.hidden2,
'init_w': args.init_w
}
self.actor = Actor(self.nb_states, self.nb_actions, **net_cfg)
self.actor_target = Actor(self.nb_states, self.nb_actions, **net_cfg)
self.actor_optim = Adam(self.actor.parameters(), lr=args.prate)
self.critic = Critic(self.nb_states, self.nb_actions, **net_cfg)
self.critic_target = Critic(self.nb_states, self.nb_actions, **net_cfg)
self.critic_optim = Adam(self.critic.parameters(), lr=args.rate)
hard_update(self.actor_target, self.actor) # Make sure target is with the same weight
hard_update(self.critic_target, self.critic)
# Create replay buffer
self.memory = SequentialMemory(limit=args.rmsize, window_length=args.window_length)
self.random_process = OrnsteinUhlenbeckProcess(size=nb_actions, theta=args.ou_theta, mu=args.ou_mu, sigma=args.ou_sigma)
# Hyper-parameters
self.batch_size = args.bsize
self.tau = args.tau
self.discount = args.discount
self.depsilon = 1.0 / args.epsilon
self.epsilon = 1.0
self.s_t = None # Most recent state
self.a_t = None # Most recent action
self.is_training = True
if USE_CUDA:
self.cuda()
def update_policy(self):
# Sample batch
state_batch, action_batch, reward_batch, \
next_state_batch, terminal_batch = self.memory.sample_and_split(self.batch_size)
# Prepare for the target q batch
next_q_values = self.critic_target([
to_tensor(next_state_batch, volatile=True),
self.actor_target(to_tensor(next_state_batch, volatile=True)),
])
next_q_values.volatile = False
target_q_batch = to_tensor(reward_batch) + \
self.discount * to_tensor(terminal_batch.astype(np.float)) * next_q_values
# Critic update
self.critic.zero_grad()
q_batch = self.critic([to_tensor(state_batch), to_tensor(action_batch)])
value_loss = criterion(q_batch, target_q_batch)
value_loss.backward()
self.critic_optim.step()
# Actor update
self.actor.zero_grad()
policy_loss = -self.critic([
to_tensor(state_batch),
self.actor(to_tensor(state_batch))
])
policy_loss = policy_loss.mean()
policy_loss.backward()
self.actor_optim.step()
# Target update
soft_update(self.actor_target, self.actor, self.tau)
soft_update(self.critic_target, self.critic, self.tau)
def eval(self):
self.actor.eval()
self.actor_target.eval()
self.critic.eval()
self.critic_target.eval()
def cuda(self):
self.actor.cuda()
self.actor_target.cuda()
self.critic.cuda()
self.critic_target.cuda()
def observe(self, r_t, s_t1, done):
if self.is_training:
self.memory.append(self.s_t, self.a_t, r_t, done)
self.s_t = s_t1
def random_action(self, distribution='uniform'):
'''
Produce a random action
'''
if distribution == 'uniform':
action = np.random.uniform(-1., 1., self.nb_actions)
# set the action internally to the agent
self.a_t = action
return action
else:
raise ValueError('Distribution {} not defined'.format(distribution))
def select_action(self, s_t, decay_epsilon=True, clip=None):
'''
Pick action according to actor network.
:param s_t: current state s_t
:param decay_epsilon: bool.
:param clip: tuple to clip action values between
clip[0] and clip[1]. Default (-1, 1)
Set to false if not clip.
'''
# Set default for clip if None
if clip is not False and clip is None:
clip = (-1., 1.)
action = to_numpy(
self.actor(to_tensor(np.array([s_t])))
).squeeze(0)
# Add noise to the action.
action += self.is_training * max(self.epsilon, 0) * self.random_process.sample()
if clip is not False:
if len(clip) != 2:
raise ValueError('Clip parameter malformed, received {}, \
expected a size 2 tuple')
action = np.clip(action, clip[0], clip[1])
if decay_epsilon:
self.epsilon -= self.depsilon
self.a_t = action
return action
def reset(self, obs):
self.s_t = obs
self.random_process.reset_states()
def load_weights(self, output):
if output is None:
return
self.actor.load_state_dict(
torch.load('{}/actor.pkl'.format(output))
)
self.critic.load_state_dict(
torch.load('{}/critic.pkl'.format(output))
)
def save_model(self, output):
torch.save(
self.actor.state_dict(),
'{}/actor.pkl'.format(output)
)
torch.save(
self.critic.state_dict(),
'{}/critic.pkl'.format(output)
)
def seed(self, s):
torch.manual_seed(s)
if USE_CUDA:
torch.cuda.manual_seed(s)
def run_agent(model_params, weights, state_transform, data_queue, weights_queue,
process, global_step, updates, best_reward, param_noise_prob, save_dir,
max_steps=10000000):
train_fn, actor_fn, target_update_fn, params_actor, params_crit, actor_lr, critic_lr = \
build_model(**model_params)
actor = Agent(actor_fn, params_actor, params_crit)
actor.set_actor_weights(weights)
env = RunEnv2(state_transform, max_obstacles=config.num_obstacles, skip_frame=config.skip_frames)
random_process = OrnsteinUhlenbeckProcess(theta=.1, mu=0., sigma=.2, size=env.noutput,
sigma_min=0.05, n_steps_annealing=1e6)
# prepare buffers for data
states = []
actions = []
rewards = []
terminals = []
total_episodes = 0
start = time()
action_noise = True
while global_step.value < max_steps:
seed = random.randrange(2**32-2)
state = env.reset(seed=seed, difficulty=2)
random_process.reset_states()
total_reward = 0.
total_reward_original = 0.
terminal = False
steps = 0
while not terminal:
state = np.asarray(state, dtype='float32')
action = actor.act(state)
if action_noise:
action += random_process.sample()
next_state, reward, next_terminal, info = env.step(action)
total_reward += reward
total_reward_original += info['original_reward']
steps += 1
global_step.value += 1
# add data to buffers
states.append(state)
actions.append(action)
rewards.append(reward)
terminals.append(terminal)
state = next_state
terminal = next_terminal
if terminal:
break
total_episodes += 1
# add data to buffers after episode end
states.append(state)
actions.append(np.zeros(env.noutput))
rewards.append(0)
terminals.append(terminal)
states_np = np.asarray(states).astype(np.float32)
data = (states_np,
np.asarray(actions).astype(np.float32),
np.asarray(rewards).astype(np.float32),
np.asarray(terminals),
)
weight_send = None
if total_reward > best_reward.value:
weight_send = actor.get_actor_weights()
# send data for training
data_queue.put((process, data, weight_send, total_reward))
# receive weights and set params to weights
weights = weights_queue.get()
report_str = 'Global step: {}, steps/sec: {:.2f}, updates: {}, episode len {}, ' \
'reward: {:.2f}, original_reward {:.4f}; best reward: {:.2f} noise {}'. \
format(global_step.value, 1. * global_step.value / (time() - start), updates.value, steps,
total_reward, total_reward_original, best_reward.value, 'actions' if action_noise else 'params')
print(report_str)
with open(os.path.join(save_dir, 'train_report.log'), 'a') as f:
f.write(report_str + '\n')
actor.set_actor_weights(weights)
action_noise = np.random.rand() < 1 - param_noise_prob
if not action_noise:
set_params_noise(actor, states_np, random_process.current_sigma)
# clear buffers
del states[:]
del actions[:]
del rewards[:]
del terminals[:]
if total_episodes % 100 == 0:
env = RunEnv2(state_transform, max_obstacles=config.num_obstacles, skip_frame=config.skip_frames)
class Agent(object):
def __init__(self, nb_states, nb_actions, args):
if args.seed > 0:
self.seed(args.seed)
self.nb_states = nb_states
self.nb_actions = nb_actions
# Create Actor and Critic Network
self.actor = Actor(self.nb_states, self.nb_actions, args.init_w)
self.actor_target = Actor(self.nb_states, self.nb_actions, args.init_w)
self.critic = Critic(self.nb_states, self.nb_actions, args.init_w)
self.critic_target = Critic(self.nb_states, self.nb_actions,
args.init_w)
hard_update(self.actor_target,
self.actor) # Make sure target is with the same weight
hard_update(self.critic_target, self.critic)
#Create replay buffer
self.random_process = OrnsteinUhlenbeckProcess(size=nb_actions,
theta=args.ou_theta,
mu=args.ou_mu,
sigma=args.ou_sigma)
# Hyper-parameters
self.batch_size = args.bsize
self.trajectory_length = args.trajectory_length
self.tau = args.tau
self.discount = args.discount
self.depsilon = 1.0 / args.epsilon
#
self.epsilon = 1.0
self.is_training = True
#
if USE_CUDA: self.cuda()
def eval(self):
self.actor.eval()
self.actor_target.eval()
self.critic.eval()
self.critic_target.eval()
def random_action(self):
action = np.random.uniform(-1., 1., self.nb_actions)
return action
def select_action(self, state, noise_enable=True, decay_epsilon=True):
action, _ = self.actor(to_tensor(np.array([state])))
action = to_numpy(action).squeeze(0)
if noise_enable == True:
action += self.is_training * max(self.epsilon,
0) * self.random_process.sample()
action = np.clip(action, -1., 1.)
if decay_epsilon:
self.epsilon -= self.depsilon
return action
def reset_lstm_hidden_state(self, done=True):
self.actor.reset_lstm_hidden_state(done)
def reset(self):
self.random_process.reset_states()
def cuda(self):
self.actor.cuda()
self.actor_target.cuda()
self.critic.cuda()
self.critic_target.cuda()
def load_weights(self, output):
if output is None: return False
self.actor.load_state_dict(torch.load('{}/actor.pkl'.format(output)))
self.critic.load_state_dict(torch.load('{}/critic.pkl'.format(output)))
return True
def save_model(self, output):
if not os.path.exists(output):
os.mkdir(output)
torch.save(self.actor.state_dict(), '{}/actor.pkl'.format(output))
torch.save(self.critic.state_dict(), '{}/critic.pkl'.format(output))
class DDPG_trainer(object):
def __init__(self, nb_state, nb_action):
self.nb_state = nb_state
self.nb_action = nb_action
self.actor = Actor(self.nb_state, self.nb_action)
self.actor_target = Actor(self.nb_state, self.nb_action)
self.actor_optim = Adam(self.actor.parameters(), lr=LEARNING_RATE)
self.critic = Critic(self.nb_state, self.nb_action)
self.critic_target = Critic(self.nb_state, self.nb_action)
self.critic_optim = Adam(self.critic.parameters(), lr=LEARNING_RATE)
hard_update(self.actor_target,
self.actor) # Make sure target is with the same weight
hard_update(self.critic_target, self.critic)
#Create replay buffer
self.memory = SequentialMemory(limit=MEMORY_SIZE, window_length=1)
self.random_process = OrnsteinUhlenbeckProcess(size=nb_action,
theta=OU_THETA,
mu=OU_MU,
sigma=OU_SIGMA)
self.is_training = True
self.epsilon = 1.0
self.a_t = None
self.s_t = None
if USE_CUDA: self.cuda()
def cuda(self):
self.actor.cuda()
self.actor_target.cuda()
self.critic.cuda()
self.critic_target.cuda()
def select_action(self, s_t, decay_epsilon=True):
action = to_numpy(self.actor(to_tensor(np.array([s_t])))).squeeze(0)
action += self.is_training * max(self.epsilon,
0) * self.random_process.sample()
action = np.clip(action, -1., 1.)
if decay_epsilon:
self.epsilon -= DELTA_EPSILON
self.a_t = action
return action
def reset(self, observation):
self.start_state = observation
self.random_process.reset_states()
def observe(self, r_t, s_t1, done):
if self.is_training:
self.memory.append(self.s_t, self.a_t, r_t, done)
self.s_t = s_t1
def update_all(self):
# Help Warm Up
if self.memory.nb_entries < BATCH_SIZE * 2:
return
# Sample batch
state_batch, action_batch, reward_batch, \
next_state_batch, terminal_batch = self.memory.sample_and_split(BATCH_SIZE)
# Prepare for the target q batch
with torch.no_grad():
next_q_values = self.critic_target([
to_tensor(next_state_batch),
self.actor_target(to_tensor(next_state_batch)),
])
target_q_batch = to_tensor(reward_batch) + \
DISCOUNT * to_tensor(terminal_batch.astype(np.float)) * next_q_values
# Critic update
self.critic.zero_grad()
for state in state_batch:
if state.shape[0] <= 2:
# print("Error sampled memory!")
return
q_batch = self.critic(
[to_tensor(state_batch),
to_tensor(action_batch)])
value_loss = CRITERION(q_batch, target_q_batch)
value_loss.backward()
self.critic_optim.step()
# Actor update
self.actor.zero_grad()
policy_loss = -self.critic(
[to_tensor(state_batch),
self.actor(to_tensor(state_batch))])
policy_loss = policy_loss.mean()
policy_loss.backward()
self.actor_optim.step()
# Target update
soft_update(self.actor_target, self.actor, TAU)
soft_update(self.critic_target, self.critic, TAU)
class DDPG(object):
def __init__(self):
# random seed for torch
__seed = config.get(MODEL_SEED)
self.policy_loss = []
self.critic_loss = []
if __seed > 0:
self.seed(__seed)
self.nb_states = config.get(MODEL_STATE_COUNT)
self.nb_actions = config.get(MODEL_ACTION_COUNT)
# Create Actor and Critic Network
actor_net_cfg = {
'hidden1': config.get(MODEL_ACTOR_HIDDEN1),
'hidden2': config.get(MODEL_ACTOR_HIDDEN2),
'init_w': config.get(MODEL_INIT_WEIGHT)
}
critic_net_cfg = {
'hidden1': config.get(MODEL_CRITIC_HIDDEN1),
'hidden2': config.get(MODEL_CRITIC_HIDDEN2),
'init_w': config.get(MODEL_INIT_WEIGHT)
}
self.actor = Actor(self.nb_states, self.nb_actions, **actor_net_cfg)
self.actor_target = Actor(self.nb_states, self.nb_actions,
**actor_net_cfg)
self.actor_optim = Adam(
self.actor.parameters(),
lr=config.get(MODEL_ACTOR_LR),
weight_decay=config.get(MODEL_ACTOR_WEIGHT_DECAY))
self.critic = Critic(self.nb_states, self.nb_actions, **critic_net_cfg)
self.critic_target = Critic(self.nb_states, self.nb_actions,
**critic_net_cfg)
self.critic_optim = Adam(
self.critic.parameters(),
lr=config.get(MODEL_CRITIC_LR),
weight_decay=config.get(MODEL_CRITIC_WEIGHT_DECAY))
hard_update(self.actor_target, self.actor)
hard_update(self.critic_target, self.critic)
#Create replay buffer
self.memory = Memory()
self.random_process = OrnsteinUhlenbeckProcess(
size=self.nb_actions,
theta=config.get(RANDOM_THETA),
mu=config.get(RANDOM_MU),
sigma=config.get(RANDOM_SIGMA))
# Hyper-parameters
self.batch_size = config.get(MODEL_BATCH_SIZE)
self.tau = config.get(MODEL_TARGET_TAU)
self.discount = config.get(MODEL_DISCOUNT)
self.depsilon = 1.0 / config.get(MODEL_EPSILON)
self.model_path = config.get(MODEL_SAVE_PATH)
#
self.epsilon = 1.0
# init device
self.device_init()
def update_policy(self, memory):
# Sample batch
state_batch, action_batch, reward_batch, \
next_state_batch, terminal_batch = memory.sample_and_split(self.batch_size)
# Prepare for the target q batch
with torch.no_grad():
next_q_values = self.critic_target([
to_tensor(next_state_batch),
self.actor_target(to_tensor(next_state_batch))
])
target_q_batch = to_tensor(reward_batch) + \
self.discount*to_tensor(terminal_batch.astype(np.float))*next_q_values
# Critic update
self.critic.zero_grad()
q_batch = self.critic(
[to_tensor(state_batch),
to_tensor(action_batch)])
value_loss = F.mse_loss(q_batch, target_q_batch)
value_loss.backward()
self.critic_optim.step()
self.critic_loss.append(value_loss.data[0])
# Actor update
self.actor.zero_grad()
policy_loss = -self.critic(
[to_tensor(state_batch),
self.actor(to_tensor(state_batch))])
policy_loss = policy_loss.mean()
policy_loss.backward()
self.actor_optim.step()
self.policy_loss.append(policy_loss.data[0])
# Target update
soft_update(self.actor_target, self.actor, self.tau)
soft_update(self.critic_target, self.critic, self.tau)
def get_loss(self):
return self.policy_loss, self.critic_loss
def eval(self):
self.actor.eval()
self.actor_target.eval()
self.critic.eval()
self.critic_target.eval()
def device_init(self):
self.actor.to(device)
self.actor_target.to(device)
self.critic.to(device)
self.critic_target.to(device)
def random_action(self):
action = np.random.uniform(-1., 1., self.nb_actions)
return action
def select_action(self, s_t):
action = to_numpy(self.actor(to_tensor(np.array([s_t])))).squeeze(0)
action += max(self.epsilon, 0) * self.random_process.sample()
action = np.clip(action, -1., 1.)
return action
def clean(self, decay_epsilon):
if decay_epsilon:
self.epsilon -= self.depsilon
def reset(self):
self.random_process.reset_states()
def load_weights(self):
if not os.path.exists(self.model_path):
return
actor_path = os.path.exists(os.path.join(self.model_path, 'actor.pkl'))
if os.path.exists(actor_path):
self.actor.load_state_dict(torch.load(actor_path))
critic_path = os.path.exists(
os.path.join(self.model_path, 'critic.pkl'))
if os.path.exists(critic_path):
self.critic.load_state_dict(torch.load(critic_path))
def save_model(self):
if not os.path.exists(self.model_path):
os.makedirs(self.model_path)
actor_path = os.path.exists(os.path.join(self.model_path, 'actor.pkl'))
torch.save(self.actor.state_dict(), actor_path)
critic_path = os.path.exists(
os.path.join(self.model_path, 'critic.pkl'))
torch.save(self.critic.state_dict(), critic_path)
def get_model(self):
return self.actor.state_dict(), self.critic.state_dict()
def load_state_dict(self, actor_state, critic_state):
self.actor.load_state_dict(actor_state)
self.critic.load_state_dict(critic_state)
def seed(self, s):
torch.manual_seed(s)
if USE_CUDA:
torch.cuda.manual_seed(s)