class Preyer: def __init__(self, s_dim, a_dim, **kwargs): self.s_dim = s_dim self.a_dim = a_dim self.config = kwargs['config'] self.device = 'cuda' if self.config.use_cuda else 'cpu' self.actor = Actor(s_dim, a_dim) self.actor_target = Actor(s_dim, a_dim) self.critic = Critic(s_dim, a_dim, 1) self.critic_target = Critic(s_dim, a_dim, 1) self.actor_optimizer = torch.optim.Adam(self.actor.parameters(), lr=self.config.a_lr) self.critic_optimizer = torch.optim.Adam(self.critic.parameters(), lr=self.config.c_lr) self.c_loss = 0 self.a_loss = 0 if self.config.use_cuda: self.actor.cuda() self.actor_target.cuda() self.critic.cuda() self.critic_target.cuda() hard_update(self.actor, self.actor_target) hard_update(self.critic, self.critic_target) self.random_process = OrnsteinUhlenbeckProcess( size=self.a_dim, theta=self.config.ou_theta, mu=self.config.ou_mu, sigma=self.config.ou_sigma) self.replay_buffer = list() self.epsilon = 1. self.depsilon = self.epsilon / self.config.epsilon_decay def memory(self, s, a, r, s_, done): self.replay_buffer.append((s, a, r, s_, done)) if len(self.replay_buffer) >= self.config.memory_length: self.replay_buffer.pop(0) def get_batches(self): experiences = random.sample(self.replay_buffer, self.config.batch_size) state_batches = np.array([_[0] for _ in experiences]) action_batches = np.array([_[1] for _ in experiences]) reward_batches = np.array([_[2] for _ in experiences]) next_state_batches = np.array([_[3] for _ in experiences]) done_batches = np.array([_[4] for _ in experiences]) return state_batches, action_batches, reward_batches, next_state_batches, done_batches def choose_action(self, s, noisy=True): if self.config.use_cuda: s = Variable(torch.cuda.FloatTensor(s)) else: s = Variable(torch.FloatTensor(s)) a = self.actor.forward(s).cpu().detach().numpy() if noisy: a += max(self.epsilon, 0.001) * self.random_process.sample() self.epsilon -= self.depsilon a = np.clip(a, -1., 1.) return np.array([a]) def random_action(self): action = np.random.uniform(low=-1., high=1., size=(1, self.a_dim)) return action def reset(self): self.random_process.reset_states() def train(self): state_batches, action_batches, reward_batches, next_state_batches, done_batches = self.get_batches( ) state_batches = Variable(torch.Tensor(state_batches).to(self.device)) action_batches = Variable( torch.Tensor(action_batches).reshape(-1, 1).to(self.device)) reward_batches = Variable( torch.Tensor(reward_batches).reshape(-1, 1).to(self.device)) next_state_batches = Variable( torch.Tensor(next_state_batches).to(self.device)) done_batches = Variable( torch.Tensor( (done_batches == False) * 1).reshape(-1, 1).to(self.device)) target_next_actions = self.actor_target.forward( next_state_batches).detach() target_next_q = self.critic_target.forward( next_state_batches, target_next_actions).detach() main_q = self.critic(state_batches, action_batches) # Critic Loss self.critic.zero_grad() baselines = reward_batches + done_batches * self.config.gamma * target_next_q loss_critic = torch.nn.MSELoss()(main_q, baselines) loss_critic.backward() self.critic_optimizer.step() # Actor Loss self.actor.zero_grad() clear_action_batches = self.actor.forward(state_batches) loss_actor = ( -self.critic.forward(state_batches, clear_action_batches)).mean() loss_actor.backward() self.actor_optimizer.step() # This is for logging self.c_loss = loss_critic.item() self.a_loss = loss_actor.item() soft_update(self.actor, self.actor_target, self.config.tau) soft_update(self.critic, self.critic_target, self.config.tau) def getLoss(self): return self.c_loss, self.a_loss
class DDPG(object): def __init__(self, nb_states, nb_actions, args): if args.seed > 0: self.seed(args.seed) self.nb_states = nb_states self.nb_actions = nb_actions # Create Actor and Critic Network net_cfg = { "hidden1": args.hidden1, "hidden2": args.hidden2, "init_w": args.init_w, } self.actor = Actor(self.nb_states, self.nb_actions, **net_cfg) self.actor_target = Actor(self.nb_states, self.nb_actions, **net_cfg) self.actor_optim = Adam(self.actor.parameters(), lr=args.prate) self.critic = Critic(self.nb_states, self.nb_actions, **net_cfg) self.critic_target = Critic(self.nb_states, self.nb_actions, **net_cfg) self.critic_optim = Adam(self.critic.parameters(), lr=args.rate) hard_update( self.actor_target, self.actor ) # Make sure target is with the same weight hard_update(self.critic_target, self.critic) # Create replay buffer self.memory = SequentialMemory( limit=args.rmsize, window_length=args.window_length ) self.random_process = OrnsteinUhlenbeckProcess( size=nb_actions, theta=args.ou_theta, mu=args.ou_mu, sigma=args.ou_sigma ) # Hyper-parameters self.batch_size = args.bsize self.tau = args.tau self.discount = args.discount self.depsilon = 1.0 / args.epsilon # self.epsilon = 1.0 self.s_t = None # Most recent state self.a_t = None # Most recent action self.is_training = True # if USE_CUDA: self.cuda() def update_policy(self): # Sample batch ( state_batch, action_batch, reward_batch, next_state_batch, terminal_batch, ) = self.memory.sample_and_split(self.batch_size) # Prepare for the target q batch next_q_values = self.critic_target( [ to_tensor(next_state_batch, volatile=True), self.actor_target(to_tensor(next_state_batch, volatile=True)), ] ) # next_q_values.volatile = False target_q_batch = ( to_tensor(reward_batch) + self.discount * to_tensor(terminal_batch.astype(np.float)) * next_q_values ) # Critic update self.critic.zero_grad() q_batch = self.critic([to_tensor(state_batch), to_tensor(action_batch)]) value_loss = criterion(q_batch, target_q_batch) value_loss.backward() self.critic_optim.step() # Actor update self.actor.zero_grad() policy_loss = -self.critic( [to_tensor(state_batch), self.actor(to_tensor(state_batch))] ) policy_loss = policy_loss.mean() policy_loss.backward() self.actor_optim.step() # Target update soft_update(self.actor_target, self.actor, self.tau) soft_update(self.critic_target, self.critic, self.tau) def eval(self): self.actor.eval() self.actor_target.eval() self.critic.eval() self.critic_target.eval() def cuda(self): self.actor.cuda() self.actor_target.cuda() self.critic.cuda() self.critic_target.cuda() def observe(self, r_t, s_t1, done): if self.is_training: self.memory.append(self.s_t, self.a_t, r_t, done) self.s_t = s_t1 def random_action(self): action = np.random.uniform(-1.0, 1.0, self.nb_actions) self.a_t = action return action def select_action(self, s_t, decay_epsilon=True): action = to_numpy(self.actor(to_tensor(np.array([s_t])))).squeeze(0) action += self.is_training * max(self.epsilon, 0) * self.random_process.sample() action = np.clip(action, -1.0, 1.0) if decay_epsilon: self.epsilon -= self.depsilon self.a_t = action return action def reset(self, obs): self.s_t = obs self.random_process.reset_states() def load_weights(self, output): if output is None: return self.actor.load_state_dict(torch.load("{}/actor.pkl".format(output))) self.critic.load_state_dict(torch.load("{}/critic.pkl".format(output))) def save_model(self, output): torch.save(self.actor.state_dict(), "{}/actor.pkl".format(output)) torch.save(self.critic.state_dict(), "{}/critic.pkl".format(output)) def seed(self, s): torch.manual_seed(s) if USE_CUDA: torch.cuda.manual_seed(s)
class Agent(): def __init__(self, nb_states, nb_actions): self.critic = Critic(nb_states, nb_actions) # Q self.critic_target = Critic(nb_states, nb_actions) self.actor = Actor(nb_states, nb_actions) # policy mu self.actor_target = Actor(nb_states, nb_actions) hard_update(self.critic_target, self.critic) hard_update(self.actor_target, self.actor) self.critic_optimizer = torch.optim.Adam(self.critic.parameters(), lr=0.001) self.actor_optimizer = torch.optim.Adam(self.actor.parameters(), lr=0.0001) self.criterion = nn.MSELoss() self.random_process = OrnsteinUhlenbeckProcess(size=nb_actions, theta=0.15, mu=0, sigma=0.2) self.gamma = 0.99 self.batch_size = 64 if USE_CUDA: self.actor.cuda() self.actor_target.cuda() self.critic.cuda() self.critic_target.cuda() def act(self, obs, epsilon=0.1): # epsilon -> tunning paramter if (random.random() < epsilon): # choose random action action = np.random.uniform(-1., 1., nb_actions) return action else: # the action is the output of actor network + Exploration Noise action = self.actor(obs).cpu().data.numpy() action += self.random_process.sample() action = np.clip(action, -1., 1.) # to stay in interval [-1,1] return action def backward(self, transitions): transitions = memory.sample(self.batch_size) batch = Transition(*zip(*transitions)) state_batch = Variable(torch.cat(batch.state)).type( FLOAT) # size 64 x 3 action_batch = Variable(torch.cat(batch.action)).type(FLOAT) # size 64 next_state_batch = Variable(torch.cat(batch.next_state)).type( FLOAT) # size 64 x 3 reward_batch = Variable(torch.cat(batch.reward)).type(FLOAT) # size 64 done_batch = Variable(torch.cat(batch.done)).type(FLOAT) #### Q - CRITIC UPDATE #### # Q(s_t,a_t) action_batch.unsqueeze_(1) # size 64x1 state_action_value = self.critic(state_batch, action_batch) # 64x1 # a_{t+1} = mu_target(s_{t+1}) next_action = self.actor_target( next_state_batch).detach() # 64 x nb_actions # Q'(s_{t+1},a_{t+1}) next_state_action_value = self.critic_target(next_state_batch, next_action).detach() next_state_action_value.squeeze_() # 64 # mask to consider next_state_values to 0 if state is terminal mask = Variable( np.logical_not(done_batch.data).type( torch.FloatTensor)).type(FLOAT) # mask = 1,1,1 .. # Compare Q(s_t,a_t) with r_t + gamma * Q'(s_{t+1},a_{t+1}) expected_state_action_value = reward_batch + ( self.gamma * next_state_action_value * mask) # Compute Huber loss # loss = F.smooth_l1_loss(state_action_values, expected_state_action_values) loss = self.criterion(state_action_value, expected_state_action_value) # Optimize the nn by updating weights with adam descent self.critic_optimizer.zero_grad() loss.backward() self.critic_optimizer.step() #### mu - ACTOR UPDATE #### # a_t = mu(s_t) action = self.actor(state_batch) # J = esperance[Q(s_t,mu(s_t))] -> a maximiser # -J = policy_loss -> a minimiser policy_loss = -self.critic(state_batch, action) policy_loss = policy_loss.mean() self.actor_optimizer.zero_grad() policy_loss.backward() self.actor_optimizer.step() #### update target network with polyak averaging soft_update(self.critic_target, self.critic, tau=0.001) soft_update(self.actor_target, self.actor, tau=0.001) return
class RDPG_v2: def __init__(self, conf, device): self.conf = conf self.state_dim = conf['state_dim'] self.action_dim = conf['action_dim'] self.device = device # create actor and critic network self.actor = Actor_RDPG(self.state_dim, self.action_dim).to(self.device) self.actor_target = Actor_RDPG(self.state_dim, self.action_dim).to(self.device) self.critic = Critic_RDPG(self.state_dim, self.action_dim).to(self.device) self.critic_target = Critic_RDPG(self.state_dim, self.action_dim).to(self.device) hard_update(self.actor_target, self.actor) # Make sure target is with the same weight hard_update(self.critic_target, self.critic) self.critic_optim = optim.Adam(self.critic.parameters(), lr=q_lr) self.actor_optim = optim.Adam(self.actor.parameters(), lr=policy_lr) #Create replay buffer self.random_process = OrnsteinUhlenbeckProcess(size=self.action_dim, theta=0.15, mu=0.0, sigma=0.2) # args.ou_theta:0.15 (noise theta), args.ou_sigma:0.2 (noise sigma), args.out_mu:0.0 (noise mu) self.epsilon = 1.0 self.depsilon = 1.0 / 50000 self.is_training = True self.tau = 0.001 # moving average for target network def random_action(self): action = np.random.uniform( 0., 1., self.action_dim) # [-1,1] select as a number of action_dim return action def select_action(self, state, noise_enable=True, decay_epsilon=True): action, _ = self.actor( to_tensor(state).reshape(-1).unsqueeze(0) ) # input shape = [batch(=1) X state_dim], action : type (tuple), shape [batch X action_dim] action = action.cpu().detach().numpy().squeeze( 0) # action shape [action_dim,] if noise_enable == True: action += self.is_training * max(self.epsilon, 0) * self.random_process.sample() action = np.clip(action, 0., 1.) # input 중 -1~1 을 벗어나는 값에 대해 -1 or 1 로 대체 if decay_epsilon: self.epsilon -= self.depsilon return action def update_policy(self, memory, gamma=0.99): print("updating...") # Sample batch experiences = memory.sample( self.conf['batch_size'] ) # type: list | shape: (max_epi_length(2000)-1 X batch(32) X 5(??)) if len(experiences) == 0: # not enough samples return dtype = torch.cuda.FloatTensor policy_loss_total = 0 value_loss_total = 0 for t in range(len(experiences) - 1): # iterate over episodes # print("t:", t) target_cx = Variable(torch.zeros(self.conf['batch_size'], 50)).type(dtype) target_hx = Variable(torch.zeros(self.conf['batch_size'], 50)).type(dtype) cx = Variable(torch.zeros(self.conf['batch_size'], 50)).type(dtype) hx = Variable(torch.zeros(self.conf['batch_size'], 50)).type(dtype) # we first get the data out of the sampled experience # shape of state0, action, reward: [batch X state_dim], [batch X 1], [batch X 1] state0 = np.stack([ trajectory.state0 for trajectory in experiences[t] ]) # batch 개수만큼 각 epi 중 t 시점에서 상태만 추출 # action = np.expand_dims(np.stack((trajectory.action for trajectory in experiences[t])), axis=1) action = np.stack( [trajectory.action for trajectory in experiences[t]]) reward = np.expand_dims(np.stack( [trajectory.reward for trajectory in experiences[t]]), axis=1) # reward = np.stack((trajectory.reward for trajectory in experiences[t])) state1 = np.stack( [trajectory.state0 for trajectory in experiences[t + 1]]) target_action, (target_hx, target_cx) = self.actor_target( to_tensor(state1).reshape(self.conf['batch_size'], -1), (target_hx, target_cx)) next_q_value = self.critic_target([ to_tensor(state1).reshape(self.conf['batch_size'], -1), target_action ]) target_q = to_tensor(reward) + gamma * next_q_value # Critic update current_q = self.critic([ to_tensor(state0).reshape(self.conf['batch_size'], -1), to_tensor(action) ]) value_loss = F.smooth_l1_loss(current_q, target_q) value_loss /= len(experiences) # divide by trajectory length value_loss_total += value_loss # update per trajectory self.critic.zero_grad() value_loss.backward() # Actor update action, (hx, cx) = self.actor( to_tensor(state0).reshape(self.conf['batch_size'], -1), (hx, cx)) policy_loss = -self.critic([ to_tensor(state0).reshape(self.conf['batch_size'], -1), action ]) policy_loss /= len(experiences) # divide by trajectory length policy_loss_total += policy_loss.mean() policy_loss = policy_loss.mean() self.actor.zero_grad() policy_loss.backward() self.critic_optim.step() self.actor_optim.step() # Target update soft_update(self.actor_target, self.actor, self.tau) soft_update(self.critic_target, self.critic, self.tau) print("update finish!") def reset_lstm_hidden_state(self, done=True): self.actor.reset_lstm_hidden_state(done) def save_model(self, path): torch.save(self.critic.state_dict(), path + '_q') torch.save(self.critic_target.state_dict(), path + '_target_q') torch.save(self.actor.state_dict(), path + '_policy') def load_model(self, path): self.critic.load_state_dict(torch.load(path + '_q')) self.critic_target.load_state_dict(torch.load(path + '_target_q')) self.actor.load_state_dict(torch.load(path + '_policy')) self.critic.eval() self.critic_target.eval() self.actor.eval()
class DDPG(object): def __init__(self, state_size, action_size, memory_size, batch_size=128, tan=0.001, actor_lr=0.001, critic_lr=0.001, epsilon=1.): self.state_size = state_size self.action_size = action_size self.batch_size = batch_size self.tan = tan self.warmup = WARM_UP self.epsilon = epsilon self.epsilon_decay = hyperparameters['D_EPSILON'] self.actor = Actor(state_size, action_size) self.actor_target = Actor(state_size, action_size) self.actor_optimizer = optim.Adam(self.actor.parameters(), lr=actor_lr) self.critic = Critic(state_size, action_size) self.critic_target = Critic(state_size, action_size) self.critic_optimizer = optim.Adam(self.critic.parameters(), lr=critic_lr) self.memory = Memory(memory_size) self.criterion = nn.MSELoss() self.random_process = OrnsteinUhlenbeckProcess(size=action_size, theta=0.15, mu=0., sigma=0.2) copy_parameter(self.actor, self.actor_target) copy_parameter(self.critic, self.critic_target) def train(self): # if not warm up if self.memory.counter < self.warmup: return # get batch state_batch, action_batch, next_state_batch, reward_batch, done_batch = self.memory.sample( self.batch_size) action_batch = action_batch.reshape((-1, self.action_size)) reward_batch = reward_batch.reshape((-1, 1)) done_batch = done_batch.reshape((-1, 1)) # update critic nsb = Variable(torch.from_numpy(next_state_batch).float(), volatile=True) # next_state_batch nab = self.actor_target(nsb) # next_action_batch next_q = self.critic_target(nsb, nab) next_q.volatile = False rb = Variable(torch.from_numpy(reward_batch).float()) # reward_batch db = Variable(torch.from_numpy(done_batch).float( )) # if next state is None, next_q should be 0, which means q = r q_target = rb + hyperparameters['GAMMA'] * db * next_q sb_grad = Variable(torch.from_numpy(state_batch).float() ) # state_batch with grad, mean output need grad ab = Variable(torch.from_numpy(action_batch).float()) # action_batch q_eval = self.critic(sb_grad, ab) value_loss = self.criterion(q_eval, q_target) self.critic.zero_grad() value_loss.backward() # nn.utils.clip_grad_norm(self.critic.parameters(),0.8) self.critic_optimizer.step() # update actor sb_grad = Variable( torch.from_numpy(state_batch).float()) # state_batch aab = self.actor(sb_grad) # actor_action_batch q = self.critic(sb_grad, aab) policy_loss = torch.mean(-q) self.actor.zero_grad() policy_loss.backward() # nn.utils.clip_grad_norm(self.actor.parameters(),0.8) self.actor_optimizer.step() # update parameter between two network update_parameter(self.critic_target, self.critic, self.tan) update_parameter(self.actor_target, self.actor, self.tan) def select_action(self, s, is_train=True, decay_e=True): if self.memory.counter < self.warmup: action = env.action_space.sample()[0] # action = random.uniform(-2.,2.) return action state = Variable(torch.FloatTensor([s]).float()) action = self.actor(state).squeeze(1).data.numpy() action += is_train * max(self.epsilon, 0) * self.random_process.sample() action = float(np.clip(action, -1., 1.)[0]) if decay_e: if self.memory.counter > self.warmup: self.epsilon -= self.epsilon_decay return action
class DDPG(object): def __init__(self, env, mem_size=7 * int(1e3), lr_critic=1e-3, lr_actor=1e-4, epsilon=1., max_epi=1500, epsilon_decay=1. / (1e5), gamma=.99, target_update_frequency=200, batch_size=64, random_process=True, max_step=None): self.CUDA = torch.cuda.is_available() self.orig_env = env #for recording if max_step is not None: self.orig_env._max_episode_steps = max_step self.env = self.orig_env self.N_S = self.env.observation_space.shape[0] self.N_A = self.env.action_space.shape[0] self.MAX_EPI = max_epi self.LOW = self.env.action_space.low self.HIGH = self.env.action_space.high self.actor = Actor(self.N_S, self.N_A) self.critic = Critic(self.N_S, self.N_A) self.target_actor = Actor(self.N_S, self.N_A) self.target_critic = Critic(self.N_S, self.N_A) self.target_actor.eval() self.target_critic.eval() self.target_actor.load_state_dict(self.actor.state_dict()) self.target_critic.load_state_dict(self.critic.state_dict()) if self.CUDA: self.actor.cuda() self.critic.cuda() self.target_actor.cuda() self.target_critic.cuda() self.exp = Experience(mem_size) self.optim_critic = optim.Adam(self.critic.parameters(), lr=lr_critic) self.optim_actor = optim.Adam(self.actor.parameters(), lr=-lr_actor) self.random_process = OrnsteinUhlenbeckProcess(\ size=self.N_A, theta=.15, mu=0, sigma=.2) self.EPSILON = epsilon self.EPSILON_DECAY = epsilon_decay self.GAMMA = gamma self.TARGET_UPDATE_FREQUENCY = target_update_frequency self.BATCH_SIZE = batch_size title = {common.S_EPI: [], common.S_TOTAL_R: []} self.data = pd.DataFrame(title) self.RAND_PROC = random_process def train(self, dir=None, interval=1000): if dir is not None: self.env = wrappers.Monitor(self.orig_env, '{}/train_record'.format(dir), force=True) os.mkdir(os.path.join(dir, 'models')) update_counter = 0 epsilon = self.EPSILON for epi in trange(self.MAX_EPI, desc='train epi', leave=True): self.random_process.reset_states() o = self.env.reset() counter = 0 acc_r = 0 while True: counter += 1 #if dir is not None: # self.env.render() a = self.choose_action(o) if self.RAND_PROC: a += max(epsilon, 0) * self.random_process.sample() a = np.clip(a, -1., 1.) epsilon -= self.EPSILON_DECAY o_, r, done, info = self.env.step(self.map_to_action(a)) self.exp.push(o, a, r, o_, done) if epi > 0: self.update_actor_critic() update_counter += 1 if update_counter % self.TARGET_UPDATE_FREQUENCY == 0: self.update_target() acc_r += r o = o_ if done: break if dir is not None: if (epi + 1) % interval == 0: self.save(os.path.join(dir, 'models'), str(epi + 1), save_data=False) s = pd.Series([epi, acc_r], index=[common.S_EPI, common.S_TOTAL_R]) self.data = self.data.append(s, ignore_index=True) def choose_action(self, state): self.actor.eval() s = Variable(torch.Tensor(state)).unsqueeze(0) if self.CUDA: s = s.cuda() a = self.actor(s).data.cpu().numpy()[0].astype('float64') self.actor.train() return a def map_to_action(self, a): return (self.LOW + self.HIGH) / 2 + a * (self.HIGH - self.LOW) / 2 def update_target(self): self.target_actor.load_state_dict(self.actor.state_dict()) self.target_critic.load_state_dict(self.critic.state_dict()) def update_actor_critic(self): # sample minibatch minibatch = common.Transition(*zip(*self.exp.sample(self.BATCH_SIZE))) bat_o = Variable(torch.Tensor(minibatch.state)) bat_a = Variable(torch.Tensor(minibatch.action)) bat_r = Variable(torch.Tensor(minibatch.reward)).unsqueeze(1) bat_o_ = Variable(torch.Tensor(minibatch.next_state)) bat_not_done_mask = list( map(lambda done: 0 if done else 1, minibatch.done)) bat_not_done_mask = Variable( torch.ByteTensor(bat_not_done_mask)).unsqueeze(1) if self.CUDA: bat_o = bat_o.cuda() bat_a = bat_a.cuda() bat_r = bat_r.cuda() bat_o_ = bat_o_.cuda() bat_not_done_mask = bat_not_done_mask.cuda() # update critic bat_a_o_ = self.target_actor(bat_o_) Gt = bat_r Gt[bat_not_done_mask] += self.GAMMA * self.target_critic( bat_o_, bat_a_o_)[bat_not_done_mask] Gt.detach_() eval_o = self.critic(bat_o, bat_a) criterion = nn.MSELoss() if self.CUDA: criterion.cuda() loss = criterion(eval_o, Gt) self.optim_critic.zero_grad() loss.backward() self.optim_critic.step() # update actor self.critic.eval() bat_a_o = self.actor(bat_o) obj = torch.mean(self.critic(bat_o, bat_a_o)) self.optim_actor.zero_grad() obj.backward() self.optim_actor.step() self.critic.train() def test(self, dir=None, n=1): if dir is not None: self.env = wrappers.Monitor(self.orig_env, '{}/test_record'.format(dir), force=True, video_callable=lambda episode_id: True) title = {common.S_EPI: [], common.S_TOTAL_R: []} df = pd.DataFrame(title) for epi in trange(n, desc='test epi', leave=True): o = self.env.reset() acc_r = 0 while True: #if dir is not None: # self.env.render() a = self.choose_action(o) o_, r, done, info = self.env.step(self.map_to_action(a)) acc_r += r o = o_ if done: break s = pd.Series([epi, acc_r], index=[common.S_EPI, common.S_TOTAL_R]) df = df.append(s, ignore_index=True) if dir is not None: df.to_csv('{}/test_data.csv'.format(dir)) else: print df def save(self, dir, suffix='', save_data=True): torch.save(self.actor.state_dict(), '{}/actor{}.pt'.format(dir, suffix)) torch.save(self.critic.state_dict(), '{}/critic{}.pt'.format(dir, suffix)) if save_data: self.data.to_csv('{}/train_data{}.csv'.format(dir, suffix)) def load_actor(self, dir): self.actor.load_state_dict(torch.load(dir)) def load_critic(self, dir): self.critic.load_state_dict(torch.load(dir)) def get_data(self): return self.data
class DDPG(object): def __init__(self, args, nb_states, nb_actions): USE_CUDA = torch.cuda.is_available() if args.seed > 0: self.seed(args.seed) self.nb_states = nb_states self.nb_actions= nb_actions self.gpu_ids = [i for i in range(args.gpu_nums)] if USE_CUDA and args.gpu_nums > 0 else [-1] self.gpu_used = True if self.gpu_ids[0] >= 0 else False net_cfg = { 'hidden1':args.hidden1, 'hidden2':args.hidden2, 'init_w':args.init_w } self.actor = Actor(self.nb_states, self.nb_actions, **net_cfg).double() self.actor_target = Actor(self.nb_states, self.nb_actions, **net_cfg).double() self.actor_optim = Adam(self.actor.parameters(), lr=args.p_lr, weight_decay=args.weight_decay) self.critic = Critic(self.nb_states, self.nb_actions, **net_cfg).double() self.critic_target = Critic(self.nb_states, self.nb_actions, **net_cfg).double() self.critic_optim = Adam(self.critic.parameters(), lr=args.c_lr, weight_decay=args.weight_decay) hard_update(self.actor_target, self.actor) # Make sure target is with the same weight hard_update(self.critic_target, self.critic) #Create replay buffer self.memory = SequentialMemory(limit=args.rmsize, window_length=args.window_length) self.random_process = OrnsteinUhlenbeckProcess(size=self.nb_actions, theta=args.ou_theta, mu=args.ou_mu, sigma=args.ou_sigma) # Hyper-parameters self.batch_size = args.bsize self.tau_update = args.tau_update self.gamma = args.gamma # Linear decay rate of exploration policy self.depsilon = 1.0 / args.epsilon # initial exploration rate self.epsilon = 1.0 self.s_t = None # Most recent state self.a_t = None # Most recent action self.is_training = True self.continious_action_space = False def update_policy(self): pass def cuda_convert(self): if len(self.gpu_ids) == 1: if self.gpu_ids[0] >= 0: with torch.cuda.device(self.gpu_ids[0]): print('model cuda converted') self.cuda() if len(self.gpu_ids) > 1: self.data_parallel() self.cuda() self.to_device() print('model cuda converted and paralleled') def eval(self): self.actor.eval() self.actor_target.eval() self.critic.eval() self.critic_target.eval() def cuda(self): self.actor.cuda() self.actor_target.cuda() self.critic.cuda() self.critic_target.cuda() def data_parallel(self): self.actor = nn.DataParallel(self.actor, device_ids=self.gpu_ids) self.actor_target = nn.DataParallel(self.actor_target, device_ids=self.gpu_ids) self.critic = nn.DataParallel(self.critic, device_ids=self.gpu_ids) self.critic_target = nn.DataParallel(self.critic_target, device_ids=self.gpu_ids) def to_device(self): self.actor.to(torch.device('cuda:{}'.format(self.gpu_ids[0]))) self.actor_target.to(torch.device('cuda:{}'.format(self.gpu_ids[0]))) self.critic.to(torch.device('cuda:{}'.format(self.gpu_ids[0]))) self.critic_target.to(torch.device('cuda:{}'.format(self.gpu_ids[0]))) def observe(self, r_t, s_t1, done): if self.is_training: self.memory.append(self.s_t, self.a_t, r_t, done) self.s_t = s_t1 def random_action(self): action = np.random.uniform(-1.,1.,self.nb_actions) # self.a_t = action return action def select_action(self, s_t, decay_epsilon=True): # proto action action = to_numpy( self.actor(to_tensor(np.array([s_t]), gpu_used=self.gpu_used, gpu_0=self.gpu_ids[0])), gpu_used=self.gpu_used ).squeeze(0) action += self.is_training * max(self.epsilon, 0) * self.random_process.sample() action = np.clip(action, -1., 1.) if decay_epsilon: self.epsilon -= self.depsilon # self.a_t = action return action def reset(self, s_t): self.s_t = s_t self.random_process.reset_states() def load_weights(self, dir): if dir is None: return if self.gpu_used: # load all tensors to GPU (gpu_id) ml = lambda storage, loc: storage.cuda(self.gpu_ids) else: # load all tensors to CPU ml = lambda storage, loc: storage self.actor.load_state_dict( torch.load('output/{}/actor.pkl'.format(dir), map_location=ml) ) self.critic.load_state_dict( torch.load('output/{}/critic.pkl'.format(dir), map_location=ml) ) print('model weights loaded') def save_model(self,output): if len(self.gpu_ids) == 1 and self.gpu_ids[0] > 0: with torch.cuda.device(self.gpu_ids[0]): torch.save( self.actor.state_dict(), '{}/actor.pt'.format(output) ) torch.save( self.critic.state_dict(), '{}/critic.pt'.format(output) ) elif len(self.gpu_ids) > 1: torch.save(self.actor.module.state_dict(), '{}/actor.pt'.format(output) ) torch.save(self.actor.module.state_dict(), '{}/critic.pt'.format(output) ) else: torch.save( self.actor.state_dict(), '{}/actor.pt'.format(output) ) torch.save( self.critic.state_dict(), '{}/critic.pt'.format(output) ) def seed(self,seed): torch.manual_seed(seed) if len(self.gpu_ids) > 0: torch.cuda.manual_seed_all(seed)
class DDPG(object): def __init__(self, env, config): self.name = 'HierarchicalNet' self.save_folder = None self.test_record = {} self.train_record = {} self.config = config self.env = env self.epsilon = config.EPSILON self.commander_memory = Commander_Memory(config.MEMORY_SIZE,config.BATCH_SIZE) self.unit_memory = Unit_Memory(2*config.MEMORY_SIZE,config.UNIT_BATCH_SIZE) self.commander_actor = Commander_Actor(config.STATE_DIM,config.COMMAND_DIM,config.RNN_INSIZE) self.commander_actor_target = Commander_Actor(config.STATE_DIM,config.COMMAND_DIM,config.RNN_INSIZE) self.commander_critic = Commander_Critic(config.STATE_DIM,config.COMMAND_DIM,config.BATCH_SIZE,config.RNN_INSIZE) self.commander_critic_target = Commander_Critic(config.STATE_DIM,config.COMMAND_DIM,config.BATCH_SIZE,config.RNN_INSIZE) self.unit_actor = Unit_Actor(config.STATE_DIM,config.COMMAND_DIM,config.ACTION_DIM) self.unit_actor_target = Unit_Actor(config.STATE_DIM,config.COMMAND_DIM,config.ACTION_DIM) self.unit_critic = Unit_Critic(config.STATE_DIM,config.COMMAND_DIM,config.ACTION_DIM,config.HIDDEN_SIZE) self.unit_critic_target = Unit_Critic(config.STATE_DIM,config.COMMAND_DIM,config.ACTION_DIM,config.HIDDEN_SIZE) self.commander_actor_h0 = Variable(torch.zeros(2, 1, config.RNN_OUTSIZE),requires_grad=False) if config.GPU >= 0: self.commander_actor.cuda(device=config.GPU) self.commander_actor_target.cuda(device=config.GPU) self.commander_critic.cuda(device=config.GPU) self.commander_critic_target.cuda(device=config.GPU) self.unit_actor.cuda(device=config.GPU) self.unit_actor_target.cuda(device=config.GPU) self.unit_critic.cuda(device=config.GPU) self.unit_critic_target.cuda(device=config.GPU) self.commander_critic.h0 = self.commander_critic.h0.cuda(device=config.GPU) self.commander_critic_target.h0 = self.commander_critic_target.h0.cuda(device=config.GPU) self.commander_actor_h0 = self.commander_actor_h0.cuda(device=config.GPU) copy_parameter(self.commander_actor, self.commander_actor_target) copy_parameter(self.commander_critic, self.commander_critic_target) copy_parameter(self.unit_actor, self.unit_actor_target) copy_parameter(self.unit_critic, self.unit_critic_target) self.commander_actor_optimizer = optim.Adam(self.commander_actor.parameters(),lr=config.ACTOR_LR) self.unit_actor_optimizer = optim.Adam(self.unit_actor.parameters(),lr=config.ACTOR_LR) self.commander_critic_optimizer = optim.Adam(self.commander_critic.parameters(), lr=config.CRITIC_LR) self.unit_critic_optimizer = optim.Adam(self.unit_critic.parameters(), lr=config.CRITIC_LR) self.criterion = nn.MSELoss() self.action_noise = OrnsteinUhlenbeckProcess(size=(config.MYSELF_NUM, config.ACTION_DIM), theta=10, mu=0., sigma=2) self.command_noise = OrnsteinUhlenbeckProcess(size=(1,config.MYSELF_NUM, config.COMMAND_DIM), theta=10, mu=0., sigma=2) # self.action_noise = OrnsteinUhlenbeckProcess(size=(config.MYSELF_NUM, config.ACTION_DIM), theta=30, mu=0., sigma=3) # self.command_noise = OrnsteinUhlenbeckProcess(size=(1,config.MYSELF_NUM, config.COMMAND_DIM), theta=30, mu=0., sigma=3) # normalize state_normalization_myelf = [1,100,100,1,100,100,1] state_normalization_enemy = [1,100,100,100,100,10,100,100,1,1,1,10] self.state_normalization = state_normalization_myelf for i in range(config.K): self.state_normalization += state_normalization_enemy self.state_normalization = np.asarray(self.state_normalization,dtype=np.float32) def append_memory(self,states,commands,actions,next_states,rewards,dones): self.commander_memory.append(states,commands,next_states,rewards,[dones]*self.config.MYSELF_NUM) for i in range(self.config.MYSELF_NUM): self.unit_memory.append(states[i],commands[i],actions[i],next_states[i],rewards[i],dones) def get_command(self,states): group_states = states.view(int(self.config.UNIT_BATCH_SIZE/8),8,self.config.STATE_DIM) group_states.volatile = True command = self.commander_actor_target(group_states,self.commander_actor_h0.repeat(1,int(self.config.UNIT_BATCH_SIZE/8),1)).contiguous() command = command.view(self.config.UNIT_BATCH_SIZE,self.config.COMMAND_DIM) command.volatile = False command.requires_grad = False return command def train_commander(self): # if not warm up if self.commander_memory.counter < self.config.WARMUP: return 0,0,0,0 state_batch, command_batch, next_state_batch, reward_batch, done_batch = self.commander_memory.sample() sb = Variable(torch.from_numpy(state_batch)).float() cb = Variable(torch.from_numpy(command_batch)).float() nsb = Variable(torch.from_numpy(next_state_batch)).float() rb = Variable(torch.from_numpy(reward_batch)).float() db = Variable(torch.from_numpy(done_batch)).float() if self.config.GPU>=0: sb = sb.cuda(device=self.config.GPU) cb = cb.cuda(device=self.config.GPU) nsb = nsb.cuda(device=self.config.GPU) rb = rb.cuda(device=self.config.GPU) db = db.cuda(device=self.config.GPU) # update critic self.commander_critic_optimizer.zero_grad() ncb = self.commander_actor_target(nsb,self.commander_actor_h0.repeat(1,self.config.BATCH_SIZE,1)) nqb = self.commander_critic_target(nsb, ncb) q_target = (rb + self.config.GAMMA * db * nqb) q_eval = self.commander_critic(sb, cb) q_target = q_target.detach() value_loss = F.mse_loss(q_eval, q_target) value_loss.backward() self.commander_critic_optimizer.step() # update actor self.commander_actor_optimizer.zero_grad() acb = self.commander_actor(sb,self.commander_actor_h0.repeat(1,self.config.BATCH_SIZE,1)) q = self.commander_critic(sb, acb) policy_loss = -torch.mean(q) policy_loss.backward() self.commander_actor_optimizer.step() # update parameter between two network update_parameter(self.commander_critic_target, self.commander_critic, self.config.TAN) update_parameter(self.commander_actor_target, self.commander_actor, self.config.TAN) # # update priorty # tderror = torch.mean(torch.abs(q_target-q_eval),1).cpu().data.numpy() # for i,id in enumerate(idxs): # self.memory.update_priorty(id,float(tderror[i])) return value_loss.data[0],policy_loss.data[0],torch.mean(q_eval).data[0],torch.mean(q_target).data[0] def train_unit(self): # if not warm up if self.unit_memory.counter < 10*self.config.WARMUP: return 0,0,0,0 state_batch, command_batch, action_batch, next_state_batch, reward_batch, done_batch = self.unit_memory.sample() sb = Variable(torch.from_numpy(state_batch)).float() cb = Variable(torch.from_numpy(command_batch)).float() ab = Variable(torch.from_numpy(action_batch)).float() nsb = Variable(torch.from_numpy(next_state_batch)).float() rb = Variable(torch.from_numpy(reward_batch)).float() db = Variable(torch.from_numpy(done_batch)).float() rb = rb.view(self.config.UNIT_BATCH_SIZE,1) db = db.view(self.config.UNIT_BATCH_SIZE,1) if self.config.GPU>=0: sb = sb.cuda(device=self.config.GPU) cb = cb.cuda(device=self.config.GPU) ab = ab.cuda(device=self.config.GPU) nsb = nsb.cuda(device=self.config.GPU) rb = rb.cuda(device=self.config.GPU) db = db.cuda(device=self.config.GPU) ncb = self.get_command(nsb) # update critic self.unit_critic_optimizer.zero_grad() nab = self.unit_actor_target(nsb,ncb) nqb = self.unit_critic_target(nsb, ncb, nab) q_target = (rb + self.config.GAMMA * db * nqb) q_eval = self.unit_critic(sb,cb,ab) q_target = q_target.detach() value_loss = F.mse_loss(q_eval, q_target) value_loss.backward() self.unit_critic_optimizer.step() # update actor self.unit_actor_optimizer.zero_grad() aab = self.unit_actor(sb,cb) q = self.unit_critic(sb, cb, aab) policy_loss = -torch.mean(q) policy_loss.backward() self.unit_actor_optimizer.step() # update parameter between two network update_parameter(self.unit_critic_target, self.unit_critic, self.config.TAN) update_parameter(self.unit_actor_target, self.unit_actor, self.config.TAN) # # update priorty # tderror = torch.mean(torch.abs(q_target-q_eval),1).cpu().data.numpy() # for i,id in enumerate(idxs): # self.memory.update_priorty(id,float(tderror[i])) return value_loss.data[0],policy_loss.data[0],torch.mean(q_eval).data[0],torch.mean(q_target).data[0] def select_action(self, s, is_train=True, decay_e=True,ignor_warmup=False): ''' :param is_train: if true, action += noise :param decay_e: if true, decay epsilon every step :param ignor_warmup: if true, select op will not wait for warmup :return: action ''' state = Variable(torch.from_numpy(s),volatile=True).unsqueeze(0) if self.config.GPU >= 0: state = state.cuda(device=self.config.GPU) self.commander_actor.eval() self.unit_actor.eval() if self.commander_memory.counter < self.config.WARMUP and is_train and not ignor_warmup: command = Variable(torch.from_numpy(np.random.uniform(-1, 1, (1,self.config.MYSELF_NUM, self.config.COMMAND_DIM))),volatile=True).float() if self.config.GPU >= 0: command = command.cuda(device=self.config.GPU) else: command = self.commander_actor(state, self.commander_actor_h0) command_noise = Variable(torch.from_numpy(self.command_noise.sample())).float() if self.config.GPU >= 0: command_noise = command_noise.cuda(device=self.config.GPU) command += is_train * max(self.epsilon, 0.2) * command_noise command = command.clamp(-1,1) actions = [] for i in range(self.config.MYSELF_NUM): c = command[:,i] s = state[:,i] a = self.unit_actor(s,c) actions.append(a) actions = torch.cat(actions,0).cpu().data.numpy() action_noise = self.action_noise.sample() actions += is_train * max(self.epsilon, 0.4) * action_noise actions = np.clip(actions, -1., 1.) if decay_e: if self.commander_memory.counter > self.config.WARMUP: self.epsilon -= self.config.EPSILON_DECAY self.commander_actor.train() self.unit_actor.train() return actions,command.cpu().data.numpy()[0] def extract_state(self, obs): ''' extract state info from obs :param obs: :return: state ''' # enemy basic :[die, health, shield, x, y, delta_health, attackCD, targetX, targetY] # enemy add :[attack this myself_agent,if closest,dx,dy,distance] enemy_basic_len = 8 enemy_add_len = 4 assert self.config.ENEMY_FEATURE == enemy_basic_len+enemy_add_len myself_units_state = [] units_enemy = {} myself_units_underattack = [] for index, unit in enumerate(obs['enemy']): state = [unit.die,unit.health, unit.shield,unit.x, unit.y,unit.attackCD,unit.targetX,unit.targetY,unit.targetUnitId] units_enemy[unit.id] = state myself_units_underattack.append(unit.targetUnitId) myself_units_underattack = set(myself_units_underattack) for index, unit in enumerate(obs['myself']): state = [unit.die,unit.health,unit.shield, unit.health > (0.2 * unit.max_health), unit.x, unit.y,int(unit.id in myself_units_underattack)] # enemy_state nearly_enemy_ids,num = self.nearyl_topK(unit,obs['enemy'],k=self.config.K) if num == 0: enemy_state = [0]*self.config.K*(enemy_basic_len+enemy_add_len) else: enemy_state = [] for idx,enemy_id in enumerate(nearly_enemy_ids): # if there no enough alive enemy, chose the farthest one if enemy_id == -1: enemy_id = nearly_enemy_ids[num-1] es = units_enemy[enemy_id] if unit.die: enemy_state_add = [0,255/self.config.DISTANCE_FACTOR,255/self.config.DISTANCE_FACTOR,255*1.4] else: under_attack = int(es[-1]==unit.id) dx,dy = es[3]-unit.x,es[4]-unit.y distance = math.sqrt(dx**2+dy**2) enemy_state_add = [under_attack,dx/self.config.DISTANCE_FACTOR,dy/self.config.DISTANCE_FACTOR,distance] enemy_state.extend(es[:-1]+enemy_state_add) # cat to state state.extend(enemy_state) myself_units_state.append(state) myself_units_state = np.asarray(myself_units_state,dtype=np.float32) myself_units_state = myself_units_state/self.state_normalization assert myself_units_state.shape == (self.config.MYSELF_NUM,self.config.STATE_DIM) # eunit = obs['enemy'][0] # ex,ey=eunit.x,eunit.y # dist = [] # for unit in obs['myself']: # x,y = unit.x,unit.y # dist.append(math.sqrt((ex-x)**2+(ey-y)**2)) # print(dist) return myself_units_state def nearyl_topK(self,unit,enemy_units,k=3): x,y = unit.x,unit.y nearly_enemys = {} for enemy in enemy_units: if not enemy.die: uid = enemy.id d = math.sqrt((enemy.x-x)**2+(enemy.y-y)**2) if len(nearly_enemys)<k: nearly_enemys[uid] = d else: max_id, max_d = max(nearly_enemys.items(), key=operator.itemgetter(1)) if d<max_d: del nearly_enemys[max_id] nearly_enemys[uid] = d sorted_nearly_enemys = sorted(nearly_enemys.items(),key=lambda x:x[1]) nearly_enemys_id = [id for id,d in sorted_nearly_enemys ] num = len(nearly_enemys) return nearly_enemys_id+[-1]*(k-num),num def test(self, eposide, test_num, ): ''' test model, without noise :return: mean test eposide_total_reward ''' total_reward = 0 win = 0 for i in range(test_num): obs = self.env.reset() state = self.extract_state(obs) for test_step in range(self.config.MAX_STEP): # print(test_step) action,command = self.select_action(state, is_train=False, decay_e=False) next_obs, reward, done, info = self.env.step(action) next_state = self.extract_state(next_obs) time.sleep(0.02) if done: if self.env.win: win += 1 break total_reward += sum(reward) state = next_state win_rate = win/test_num total_reward = total_reward/(self.config.MYSELF_NUM*test_num) self.test_record[eposide] = total_reward return total_reward,win,win_rate def save(self, episode): ''' save model, hyperparameters, test record :param episode: :return: ''' timestr = time.strftime('(%m-%d_%H:%M)', time.localtime(time.time())) mapname = '({})'.format(self.env.getMapName()) if self.save_folder is None: self.save_folder = self.name + mapname + timestr + self.config.NOTE if not os.path.exists(self.save_folder): os.mkdir(self.save_folder) self.config.MAP = self.env.getMapName() with open(os.path.join(self.save_folder, 'config'), 'w') as f: f.write('config:\n\n') for k, v in sorted(self.config.todict().items()): f.write('{:<16} : {}\n'.format(k, v)) with open(os.path.join(self.save_folder, 'config.pkl'), 'wb') as f: pickle.dump(self.config, f) torch.save(self.commander_actor.cpu(), os.path.join(self.save_folder, 'commander_actor_%d.mod' % episode)) torch.save(self.commander_critic.cpu(), os.path.join(self.save_folder, 'commander_critic_%d.mod' % episode)) torch.save(self.unit_actor.cpu(), os.path.join(self.save_folder, 'unit_actor_%d.mod' % episode)) torch.save(self.unit_critic.cpu(), os.path.join(self.save_folder, 'unit_critic_%d.mod' % episode)) with open(os.path.join(self.save_folder, 'test_record.pkl'), 'wb') as f: pickle.dump(self.test_record, f) with open(os.path.join(self.save_folder, 'train_record.pkl'), 'wb') as f: pickle.dump(self.train_record, f) # self.commander_memory.save(self.save_folder) # self.unit_memory.save(self.save_folder) if self.config.GPU >= 0: self.commander_actor.cuda(device=self.config.GPU) self.commander_critic.cuda(device=self.config.GPU) self.unit_actor.cuda(device=self.config.GPU) self.unit_critic.cuda(device=self.config.GPU) def load(self, saved_folder, episode): # load network self.commander_actor = torch.load(os.path.join(saved_folder, 'commander_actor_{}.mod'.format(episode))) self.commander_actor_target = torch.load(os.path.join(saved_folder, 'commander_actor_{}.mod'.format(episode))) self.commander_critic = torch.load(os.path.join(saved_folder, 'commander_critic_{}.mod'.format(episode))) self.commander_critic_target = torch.load(os.path.join(saved_folder, 'commander_critic_{}.mod'.format(episode))) self.unit_actor = torch.load(os.path.join(saved_folder, 'unit_actor_{}.mod'.format(episode))) self.unit_actor_target = torch.load(os.path.join(saved_folder, 'unit_actor_{}.mod'.format(episode))) self.unit_critic = torch.load(os.path.join(saved_folder, 'unit_critic_{}.mod'.format(episode))) self.unit_critic_target = torch.load(os.path.join(saved_folder, 'unit_critic_{}.mod'.format(episode))) if self.config.GPU >= 0: self.commander_actor.cuda(device=self.config.GPU) self.commander_actor_target.cuda(device=self.config.GPU) self.commander_critic.cuda(device=self.config.GPU) self.commander_critic_target.cuda(device=self.config.GPU) self.unit_actor.cuda(device=self.config.GPU) self.unit_actor_target.cuda(device=self.config.GPU) self.unit_critic.cuda(device=self.config.GPU) self.unit_critic_target.cuda(device=self.config.GPU) self.commander_critic_optimizer = optim.Adam(self.commander_critic.parameters(), lr=self.config.CRITIC_LR) self.commander_actor_optimizer = optim.Adam(self.commander_actor.parameters(), lr=self.config.ACTOR_LR) self.unit_critic_optimizer = optim.Adam(self.unit_critic.parameters(), lr=self.config.CRITIC_LR) self.unit_actor_optimizer = optim.Adam(self.unit_actor.parameters(), lr=self.config.ACTOR_LR) # load memory # self.commander_memory = Commander_Memory.memory_load(saved_folder) # self.unit_memory = Unit_Memory.memory_load(saved_folder) # # load record # with open(os.path.join(self.save_folder, 'test_record.pkl'), 'rb') as f: # self.test_record = pickle.load(f) # with open(os.path.join(self.save_folder, 'train_record.pkl'), 'rb') as f: # self.train_record = pickle.load(f) # for r in self.test_record: # if r[0]>episode: # self.test_record.remove(r) # for r in self.train_record: # if r[0]>episode: # self.train_record.remove(r) def print_action(self,action): env_cmd = self.env._make_commands(action) ecs = [] for ec in env_cmd: if len(ec) > 2: if ec[2] == tcc.unitcommandtypes.Attack_Unit: c = [ec[1], 'A', ec[3]] else: c = [ec[1], 'M', ec[4], ec[5]] print(c) ecs.append(c) return ecs
def run_agent(args, model_params, weights, data_queue, weights_queue, process, global_step, updates, best_reward, param_noise_prob, save_dir, max_steps=10000000): train_fn, actor_fn, target_update_fn, params_actor, params_crit, actor_lr, critic_lr = build_model( **model_params) actor = Agent(actor_fn, params_actor, params_crit) actor.set_actor_weights(weights) env = RunEnv2(model=args.modeldim, prosthetic=args.prosthetic, difficulty=args.difficulty, skip_frame=config.skip_frames) env.spec.timestep_limit = 3000 # ndrw # random_process = OrnsteinUhlenbeckProcess(theta=.1, mu=0., sigma=.3, size=env.noutput, sigma_min=0.05, n_steps_annealing=1e6) sigma_rand = random.uniform(0.05, 0.5) dt_rand = random.uniform(0.002, 0.02) param_noise_prob = random.uniform(param_noise_prob * 0.25, min(param_noise_prob * 1.5, 1.)) random_process = OrnsteinUhlenbeckProcess(theta=.1, mu=0., sigma=sigma_rand, dt=dt_rand, size=env.noutput, sigma_min=0.05, n_steps_annealing=1e6) print('OUProcess_sigma = ' + str(sigma_rand) + ' OUProcess_dt = ' + str(dt_rand) + ' param_noise_prob = ' + str(param_noise_prob)) # prepare buffers for data states = [] actions = [] rewards = [] terminals = [] total_episodes = 0 start = time() action_noise = True while global_step.value < max_steps: seed = random.randrange(2**32 - 2) state = env.reset(seed=seed, difficulty=args.difficulty) random_process.reset_states() total_reward = 0. total_reward_original = 0. terminal = False steps = 0 while not terminal: state = np.asarray(state, dtype='float32') action = actor.act(state) if action_noise: action += random_process.sample() next_state, reward, next_terminal, info = env._step(action) total_reward += reward total_reward_original += info['original_reward'] steps += 1 global_step.value += 1 # add data to buffers states.append(state) actions.append(action) rewards.append(reward) terminals.append(terminal) state = next_state terminal = next_terminal if terminal: break total_episodes += 1 # add data to buffers after episode end states.append(state) actions.append(np.zeros(env.noutput)) rewards.append(0) terminals.append(terminal) states_np = np.asarray(states).astype(np.float32) data = ( states_np, np.asarray(actions).astype(np.float32), np.asarray(rewards).astype(np.float32), np.asarray(terminals), ) weight_send = None if total_reward > best_reward.value: weight_send = actor.get_actor_weights() # send data for training data_queue.put((process, data, weight_send, total_reward)) # receive weights and set params to weights weights = weights_queue.get() # report_str = 'Global step: {}, steps/sec: {:.2f}, updates: {}, episode len: {}, pelvis_X: {:.2f}, reward: {:.2f}, original_reward {:.4f}, best reward: {:.2f}, noise: {}'. \ # format(global_step.value, 1. * global_step.value / (time() - start), updates.value, steps, info['pelvis_X'], total_reward, total_reward_original, best_reward.value, 'actions' if action_noise else 'params') # report_str = 'Global step: {}, steps/sec: {:.2f}, updates: {}, episode len: {}, pelvis_X: {:.2f}, reward: {:.2f}, best reward: {:.2f}, noise: {}'. \ # format(global_step.value, 1. * global_step.value / (time() - start), updates.value, steps, info['pelvis_X'], total_reward, best_reward.value, 'actions' if action_noise else 'params') report_str = 'Global step: {}, steps/sec: {:.2f}, updates: {}, episode len: {}, pelvis_X: {:.2f}, pelvis_Z: {:.2f}, reward: {:.2f}, best reward: {:.2f}, noise: {}'. \ format(global_step.value, 1. * global_step.value / (time() - start), updates.value, steps, info['pelvis'][0], info['pelvis'][2], total_reward, best_reward.value, 'actions' if action_noise else 'params') print(report_str) try: with open(os.path.join(save_dir, 'train_report.log'), 'a') as f: f.write(report_str + '\n') except: print('#############################################') print( 'except » with open(os.path.join(save_dir, train_report.log), a) as f:' ) print('#############################################') actor.set_actor_weights(weights) action_noise = np.random.rand() < 1 - param_noise_prob if not action_noise: set_params_noise(actor, states_np, random_process.current_sigma) # clear buffers del states[:] del actions[:] del rewards[:] del terminals[:] if total_episodes % 100 == 0: env = RunEnv2(model=args.modeldim, prosthetic=args.prosthetic, difficulty=args.difficulty, skip_frame=config.skip_frames)
class UADDPG(object): def __init__(self, nb_states, nb_actions, args): if args.seed > 0: self.seed(args.seed) self.nb_states = nb_states self.nb_actions = nb_actions self.epistemic_actor = args.epistemic_actor # true / false self.epistemic_critic = args.epistemic_critic # true / false self.aleatoric_actor = args.aleatoric_actor # true / false self.aleatoric_critic = args.aleatoric_critic # true / false self.dropout_n_actor = args.dropout_n_actor self.dropout_n_critic = args.dropout_n_critic self.dropout_p_actor = args.dropout_p_actor self.dropout_p_critic = args.dropout_p_critic self.print_var_count = 0 self.action_std = np.array([]) self.save_dir = args.output self.episode = 0 # self.save_file = open(self.save_dir + '/std.txt', "a") # Create Actor and Critic Network net_cfg_actor = { 'dropout_n': args.dropout_n_actor, 'dropout_p': args.dropout_p_actor, 'hidden1': args.hidden1, 'hidden2': args.hidden2, 'init_w': args.init_w } net_cfg_critic = { 'dropout_n': args.dropout_n_actor, 'dropout_p': args.dropout_p_critic, 'hidden1': args.hidden1, 'hidden2': args.hidden2, 'init_w': args.init_w } self.actor = UAActor(self.nb_states, self.nb_actions, **net_cfg_actor) self.actor_target = UAActor(self.nb_states, self.nb_actions, **net_cfg_actor) self.actor_optim = Adam(self.actor.parameters(), lr=args.prate) self.critic = UACritic(self.nb_states, self.nb_actions, **net_cfg_critic) self.critic_target = UACritic(self.nb_states, self.nb_actions, **net_cfg_critic) self.critic_optim = Adam(self.critic.parameters(), lr=args.rate) hard_update(self.actor_target, self.actor) hard_update(self.critic_target, self.critic) # Create replay buffer self.memory = SequentialMemory(limit=args.rmsize, window_length=args.window_length) self.random_process = OrnsteinUhlenbeckProcess(size=nb_actions, theta=args.ou_theta, mu=args.ou_mu, sigma=args.ou_sigma) # Hyper-parameters self.batch_size = args.bsize self.tau = args.tau self.discount = args.discount self.depsilon = 1.0 / args.epsilon # self.epsilon = 1.0 self.s_t = None # Most recent state self.a_t = None # Most recent action self.is_training = True # if USE_CUDA: self.cuda() def update_policy(self): # Sample batch state_batch, action_batch, reward_batch, next_state_batch, terminal_batch = self.memory.sample_and_split( self.batch_size) # Prepare for the target q batch # TODO : Also apply epistemic and aleatoric uncertainty to both actor and critic target network next_q_values = self.critic_target([ to_tensor(next_state_batch, volatile=True), self.actor_target(to_tensor(next_state_batch, volatile=True)), ]) target_q_batch = to_tensor(reward_batch) + self.discount * to_tensor( terminal_batch.astype(np.float)) * next_q_values ######################### # Critic update ######################### self.critic.zero_grad() # TODO : Add epistemic uncertainty for critic network q_batch = self.critic( [to_tensor(state_batch), to_tensor(action_batch)]) # TODO : Add aleatoric uncertainty term from aleatoric uncertainty output of critic network (Add uncertainty term in criterion) value_loss = criterion(q_batch, target_q_batch) value_loss.backward() self.critic_optim.step() ######################### # Actor update ######################### self.actor.zero_grad() # policy loss # TODO : Add epistemic certainty term from aleatoric certainty output of policy network policy_loss = -self.critic( [to_tensor(state_batch), self.actor(to_tensor(state_batch))]) policy_loss = policy_loss.mean() # policy_loss = policy_loss.mean() + actor_certainty policy_loss.backward() self.actor_optim.step() ######################### # Target soft update ######################### soft_update(self.actor_target, self.actor, self.tau) soft_update(self.critic_target, self.critic, self.tau) def eval(self): self.actor.eval() self.actor_target.eval() self.critic.eval() self.critic_target.eval() def cuda(self): self.actor.cuda() self.actor_target.cuda() self.critic.cuda() self.critic_target.cuda() def observe(self, r_t, s_t1, done): if self.is_training: self.memory.append(self.s_t, self.a_t, r_t, done) self.s_t = s_t1 def random_action(self): action = np.random.uniform(-1., 1., self.nb_actions) self.a_t = action return action # def select_action(self, s_t, decay_epsilon=True): # action = to_numpy(self.actor(to_tensor(np.array([s_t])))).squeeze(0) # action += self.is_training*max(self.epsilon, 0)*self.random_process.sample() # # if decay_epsilon: # self.epsilon -= self.depsilon # # self.a_t = action # return action def select_action_with_dropout(self, s_t, decay_epsilon=True): dropout_actions = np.array([]) with torch.no_grad(): for _ in range(self.dropout_n): action = to_numpy( self.actor.forward_with_dropout(to_tensor(np.array( [s_t])))).squeeze(0) dropout_actions = np.append(dropout_actions, [action]) if self.train_with_dropout: plt_action = to_numpy( self.actor.forward_with_dropout(to_tensor(np.array( [s_t])))).squeeze(0) plt_action += self.is_training * max( self.epsilon, 0) * self.random_process.sample() else: plt_action = to_numpy(self.actor(to_tensor(np.array( [s_t])))).squeeze(0) plt_action += self.is_training * max( self.epsilon, 0) * self.random_process.sample() """ UNFIXED RESET POINT for Mujoco """ if self.print_var_count != 0 and (self.print_var_count + 1) % 999 == 0: # self.action_std = np.append(self.action_std, [np.std(dropout_actions)]) with open(self.save_dir + "/std.txt", "a") as myfile: myfile.write(str(np.std(dropout_actions)) + '\n') with open(self.save_dir + "/mean.txt", "a") as myfile: myfile.write(str(np.mean(dropout_actions)) + '\n') if self.print_var_count % (1000 * 5) == 0: print("dropout actions std", np.std(dropout_actions), " ", "dir : ", str(self.save_dir)) """ FIXED RESET POINT for MCC """ # if s_t[0] == -0.5 and s_t[1] == 0: # # print("fixed dropout actions std", np.std(dropout_actions), " ", "dir : ", str(self.save_dir)) # self.action_std = np.append(self.action_std, [np.std(dropout_actions)]) # # np.savetxt(self.save_dir + '/std.txt', self.action_std, fmt='%4.10f', delimiter=' ') # with open(self.save_dir + "/std.txt", "a") as myfile: # myfile.write(str(np.std(dropout_actions))+'\n') # with open(self.save_dir + "/mean.txt", "a") as myfile: # myfile.write(str(np.mean(dropout_actions))+'\n') if not (os.path.isdir(self.save_dir + "/episode/" + str(self.episode))): os.makedirs( os.path.join(self.save_dir + "/episode/" + str(self.episode))) self.action_std = np.append(self.action_std, [np.std(dropout_actions)]) with open(self.save_dir + "/episode/" + str(self.episode) + "/std.txt", "a") as myfile: myfile.write(str(np.std(dropout_actions)) + '\n') with open( self.save_dir + "/episode/" + str(self.episode) + "/mean.txt", "a") as myfile: myfile.write(str(np.mean(dropout_actions)) + '\n') self.print_var_count = self.print_var_count + 1 if decay_epsilon: self.epsilon -= self.depsilon self.a_t = plt_action return plt_action def reset(self, obs): self.s_t = obs self.random_process.reset_states() def load_weights(self, output): if output is None: return self.actor.load_state_dict(torch.load('{}/actor.pkl'.format(output))) self.critic.load_state_dict(torch.load('{}/critic.pkl'.format(output))) def save_model(self, output): torch.save(self.actor.state_dict(), '{}/actor.pkl'.format(output)) torch.save(self.critic.state_dict(), '{}/critic.pkl'.format(output)) def seed(self, s): torch.manual_seed(s) if USE_CUDA: torch.cuda.manual_seed(s)
class DDPG(object): def __init__(self, nb_states, nb_actions): self.nb_states = nb_states self.nb_actions = nb_actions # Create Actor and Critic Network self.actor = Actor(self.nb_states, self.nb_actions) self.actor_target = Actor(self.nb_states, self.nb_actions) self.actor_optim = Adam(self.actor.parameters(), lr=ACTOR_LR) self.critic = Critic(self.nb_states, self.nb_actions) self.critic_target = Critic(self.nb_states, self.nb_actions) self.critic_optim = Adam(self.critic.parameters(), lr=CRITIC_LR) hard_update(self.actor_target, self.actor) # Make sure target is with the same weight hard_update(self.critic_target, self.critic) #Create replay buffer self.memory = SequentialMemory(limit=MEMORY_SIZE, window_length=HISTORY_LEN) self.random_process = OrnsteinUhlenbeckProcess(size=nb_actions, theta=OU_THETA, mu=OU_MU, sigma=OU_SIGMA) # Hyper-parameters self.batch_size = BATCH_SIZE self.tau = TAU self.discount = GAMMA self.depsilon = 1.0 / DEPSILON self.epsilon = 1.0 self.s_t = None # Most recent state self.a_t = None # Most recent action self.is_training = True if USE_CUDA: self.cuda() def update_policy(self): # Sample batch state_batch, action_batch, reward_batch, \ next_state_batch, terminal_batch = self.memory.sample_and_split(self.batch_size) # Prepare for the target q batch next_q_values = self.critic_target([ to_tensor(next_state_batch, volatile=True), self.actor_target(to_tensor(next_state_batch, volatile=True)), ])[:, 0] next_q_values.volatile = False target_q_batch = to_tensor(reward_batch) + \ self.discount*to_tensor(terminal_batch.astype(np.float))*next_q_values # Critic update self.critic.zero_grad() q_batch = self.critic( [to_tensor(state_batch), to_tensor(action_batch)]) value_loss = criterion(q_batch, target_q_batch) value_loss.backward() torch.nn.utils.clip_grad_norm(self.critic.parameters(), 10.0) for p in self.critic.parameters(): p.data.add_(-CRITIC_LR, p.grad.data) self.critic_optim.step() # Actor update self.actor.zero_grad() policy_loss = -self.critic( [to_tensor(state_batch), self.actor(to_tensor(state_batch))]) policy_loss = policy_loss.mean() policy_loss.backward() torch.nn.utils.clip_grad_norm(self.actor.parameters(), 10.0) for p in self.actor.parameters(): p.data.add_(-ACTOR_LR, p.grad.data) self.actor_optim.step() # Target update soft_update(self.actor_target, self.actor, self.tau) soft_update(self.critic_target, self.critic, self.tau) def eval(self): self.actor.eval() self.actor_target.eval() self.critic.eval() self.critic_target.eval() def cuda(self): self.actor.cuda() self.actor_target.cuda() self.critic.cuda() self.critic_target.cuda() def observe(self, r_t, s_t1, done): if self.is_training: self.memory.append(self.s_t, self.a_t, r_t, done) self.s_t = s_t1 def random_action(self): action = np.random.uniform(-1., 1., self.nb_actions) self.a_t = action return action def select_action(self, s_t, decay_epsilon=True): action = to_numpy(self.actor(to_tensor(np.array([s_t]))))[0] ou = self.random_process.sample() prGreen('eps:{}, act:{}, random:{}'.format(self.epsilon, action, ou)) action += self.is_training * max(self.epsilon, 0) * ou action = np.clip(action, -1., 1.) if decay_epsilon: self.epsilon -= self.depsilon self.a_t = action return action def reset(self, obs): self.s_t = obs self.random_process.reset_states() def load_weights(self, output): if output is None: return self.actor.load_state_dict(torch.load('{}/actor.pkl'.format(output))) self.critic.load_state_dict(torch.load('{}/critic.pkl'.format(output))) def save_model(self, output): torch.save(self.actor.state_dict(), '{}/actor.pkl'.format(output)) torch.save(self.critic.state_dict(), '{}/critic.pkl'.format(output)) def seed(self, s): torch.manual_seed(s) if USE_CUDA: torch.cuda.manual_seed(s)
class DDPG(object): def __init__(self, nb_states, nb_actions, args): if args.seed > 0: self.seed(args.seed) self.nb_states = nb_states self.nb_actions = nb_actions actor_net_cfg = { 'hidden1': 32, 'hidden2': 32, 'hidden3': 32, 'init_w': args.init_w } critic_net_cfg = { 'hidden1': 64, 'hidden2': 64, 'hidden3': 64, 'init_w': args.init_w } self.actor = Actor(self.nb_states, self.nb_actions, **actor_net_cfg) self.actor_target = Actor(self.nb_states, self.nb_actions, **actor_net_cfg) self.actor_optim = Adam(self.actor.parameters(), lr=args.prate) self.critic = Critic(self.nb_states, self.nb_actions, **critic_net_cfg) self.critic_target = Critic(self.nb_states, self.nb_actions, **critic_net_cfg) self.critic_optim = Adam(self.critic.parameters(), lr=args.rate) hard_update(self.actor_target, self.actor) # Make sure target is with the same weight hard_update(self.critic_target, self.critic) #Create replay buffer self.memory = SequentialMemory(limit=args.rmsize, window_length=args.window_length) self.random_process = OrnsteinUhlenbeckProcess(size=nb_actions, theta=args.ou_theta, mu=args.ou_mu, sigma=args.ou_sigma) # Hyper-parameters self.batch_size = args.bsize self.tau = args.tau self.discount = args.discount self.depsilon = 1.0 / args.epsilon self.epsilon = 1.0 self.s_t = None # Most recent state self.a_t = None # Most recent action self.is_training = True self.best_reward = -10 def update_policy(self, shared_model, args): # Sample batch state_batch, action_batch, reward_batch, \ next_state_batch, terminal_batch = self.memory.sample_and_split(self.batch_size, shared=args.use_more_states, num_states=args.num_states) # Prepare for the target q batch next_q_values = self.critic_target([ to_tensor(next_state_batch, volatile=True), self.actor_target(to_tensor(next_state_batch, volatile=True)), ]) next_q_values.volatile = False target_q_batch = to_tensor(reward_batch) + \ self.discount*to_tensor(terminal_batch.astype(np.float))*next_q_values # Critic update self.critic_optim.zero_grad() q_batch = self.critic( [to_tensor(state_batch), to_tensor(action_batch)]) value_loss = criterion(q_batch, target_q_batch) value_loss.backward() if args.shared: ensure_shared_grads(self.critic, shared_model.critic) self.critic_optim.step() # Actor update self.actor_optim.zero_grad() policy_loss = -self.critic( [to_tensor(state_batch), self.actor(to_tensor(state_batch))]) policy_loss = policy_loss.mean() policy_loss.backward() if args.shared: ensure_shared_grads(self.actor, shared_model.actor) self.actor_optim.step() # Target update soft_update(self.actor_target, self.actor, self.tau) soft_update(self.critic_target, self.critic, self.tau) def eval(self): self.actor.eval() self.actor_target.eval() self.critic.eval() self.critic_target.eval() def share_memory(self): self.critic.share_memory() self.actor.share_memory() def add_optim(self, actor_optim, critic_optim): self.actor_optim = actor_optim self.critic_optim = critic_optim def observe(self, r_t, s_t1, done): if self.is_training: self.memory.append(self.s_t, self.a_t, r_t, done) self.s_t = s_t1 def update_models(self, agent): self.actor = deepcopy(agent.actor) self.actor_target = deepcopy(agent.actor_target) self.critic = deepcopy(agent.critic) self.critic_target = deepcopy(agent.critic_target) self.actor_optim = deepcopy(agent.actor_optim) self.critic_optim = deepcopy(agent.critic_optim) def random_action(self): action = np.random.uniform(-1., 1., self.nb_actions) self.a_t = action return action def train(self): self.critic.train() self.actor.train() def state_dict(self): return [ self.actor.state_dict(), self.actor_target.state_dict(), self.critic.state_dict(), self.critic_target.state_dict() ] def load_state_dict(self, list_of_dicts): self.actor.load_state_dict(list_of_dicts[0]) self.actor_target.load_state_dict(list_of_dicts[1]) self.critic.load_state_dict(list_of_dicts[2]) self.critic_target.load_state_dict(list_of_dicts[3]) def select_action(self, s_t, decay_epsilon=True): action = to_numpy(self.actor(to_tensor(np.array([s_t])))).squeeze(0) action += self.is_training * max(self.epsilon, 0) * self.random_process.sample() action = np.clip(action, -1., 1.) if decay_epsilon: self.epsilon -= self.depsilon self.a_t = action return action def reset(self, obs): self.s_t = obs self.random_process.reset_states() def load_weights(self, output): if output is None: return self.actor.load_state_dict(torch.load('{}/actor.pkl'.format(output))) self.critic.load_state_dict(torch.load('{}/critic.pkl'.format(output))) def save_model(self, output): torch.save(self.actor.state_dict(), '{}/actor.pkl'.format(output)) torch.save(self.critic.state_dict(), '{}/critic.pkl'.format(output)) def seed(self, s): torch.manual_seed(s)
class DDPG: def __init__(self, env, actor_model, critic_model, memory=10000, batch_size=64, gamma=0.99, tau=0.001, actor_lr=1e-4, critic_lr=1e-3, critic_decay=1e-2, ou_theta=0.15, ou_sigma=0.2, render=None, evaluate=None, save_path=None, save_every=10, render_every=10, train_per_step=True): self.env = env self.actor = actor_model self.actor_target = actor_model.clone() self.critic = critic_model self.critic_target = critic_model.clone() if use_cuda: for net in [ self.actor, self.actor_target, self.critic, self.critic_target ]: net.cuda() self.memory = ReplayMemory(memory) self.batch_size = batch_size self.gamma = gamma self.tau = tau self.random_process = OrnsteinUhlenbeckProcess( env.action_space.shape[0], theta=ou_theta, sigma=ou_sigma) self.optim_critic = optim.Adam(self.critic.parameters(), lr=critic_lr, weight_decay=critic_decay) self.optim_actor = optim.Adam(self.actor.parameters(), lr=actor_lr) self.render = render self.render_every = render_every self.evaluate = evaluate self.save_path = save_path self.save_every = save_every self.train_per_step = train_per_step def update(self, target, source): zipped = zip(target.parameters(), source.parameters()) for target_param, source_param in zipped: updated_param = target_param.data * (1 - self.tau) + \ source_param.data * self.tau target_param.data.copy_(updated_param) def train_models(self): if len(self.memory) < self.batch_size: return None, None mini_batch = self.memory.sample_batch(self.batch_size) critic_loss = self.train_critic(mini_batch) actor_loss = self.train_actor(mini_batch) self.update(self.actor_target, self.actor) self.update(self.critic_target, self.critic) return critic_loss.data[0], actor_loss.data[0] def mse(self, inputs, targets): return torch.mean((inputs - targets)**2) def train_critic(self, batch): # forward pass pred_actions = self.actor_target(batch.next_states) target_q = batch.rewards + batch.done * self.critic_target( [batch.next_states, pred_actions]) * self.gamma pred_q = self.critic([batch.states, batch.actions]) # backward pass loss = self.mse(pred_q, target_q) self.optim_critic.zero_grad() loss.backward(retain_graph=True) for param in self.critic.parameters(): param.grad.data.clamp_(-1, 1) self.optim_critic.step() return loss def train_actor(self, batch): # forward pass pred_mu = self.actor(batch.states) pred_q = self.critic([batch.states, pred_mu]) # backward pass loss = -pred_q.mean() self.optim_actor.zero_grad() loss.backward() # for param in self.actor.parameters(): # param.grad.data.clamp_(-1, 1) self.optim_actor.step() return loss def prep_state(self, s): return Variable(torch.from_numpy(s).float().unsqueeze(0)) def select_action(self, state, exploration=True): if use_cuda: state = state.cuda() self.actor.eval() action = self.actor(state) self.actor.train() if exploration: noise = Variable( torch.from_numpy(self.random_process.sample()).float()) if use_cuda: noise = noise.cuda() action = action + noise return action def step(self, action): next_state, reward, done, _ = self.env.step( action.data.cpu().numpy()[0]) next_state = self.prep_state(next_state) reward = FloatTensor([reward]) return next_state, reward, done def warmup(self, num_steps): overall_step = 0 while overall_step <= num_steps: done = False state = self.prep_state(self.env.reset()) self.random_process.reset() while not done: overall_step += 1 action = self.select_action(state) next_state, reward, done = self.step(action) self.memory.add(state, action, reward, next_state, done) state = next_state def train(self, num_steps): running_reward = None reward_sums = [] losses = [] overall_step = 0 episode_number = 0 while overall_step <= num_steps: episode_number += 1 done = False state = self.prep_state(self.env.reset()) reward_sum = 0 self.random_process.reset() while not done: overall_step += 1 action = self.select_action(state) next_state, reward, done = self.step(action) self.memory.add(state, action, reward, next_state, done) state = next_state reward_sum += reward[0] if self.train_per_step: losses.append(self.train_models()) if not self.train_per_step: losses.append(self.train_models()) render_this_episode = self.render and (episode_number % self.render_every == 0) evaluation_reward = self.run(render=render_this_episode) reward_sums.append((reward_sum, evaluation_reward)) if self.save_path is not None and (episode_number % self.save_every == 0): self.save_models(self.save_path) self.save_results(self.save_path, losses, reward_sums) running_reward = reward_sum if running_reward is None else running_reward * 0.99 + reward_sum * 0.01 print( 'episode: {} steps: {} running train reward: {:.4f} eval reward: {:.4f}' .format(episode_number, overall_step, running_reward, evaluation_reward)) if self.save_path is not None: self.save_models(self.save_path) self.save_results(self.save_path, losses, reward_sums) return reward_sums, losses def run(self, render=True): state = self.env.reset() done = False reward_sum = 0 while not done: if render: self.env.render() action = self.select_action(self.prep_state(state), exploration=False) state, reward, done, _ = self.env.step( action.data.cpu().numpy()[0]) reward_sum += reward return reward_sum def save_models(self, path): self.actor.save(path) self.critic.save(path) def save_results(self, path, losses, rewards): losses = np.array([l for l in losses if l[0] is not None]) rewards = np.array(rewards) np.savetxt(os.path.join(path, 'losses.csv'), losses, delimiter=',', header='critic,actor', comments='') np.savetxt(os.path.join(path, 'rewards.csv'), rewards, delimiter=',', header='train,evaluation', comments='')
class DDPG(object): def __init__(self, nb_states, nb_actions, args): if args.seed > 0: self.seed(args.seed) self.nb_states = nb_states self.nb_actions = nb_actions # Create Actor and Critic Network net_cfg = { 'hidden1': args.hidden1, 'hidden2': args.hidden2, 'init_w': args.init_w } self.actor = Actor(self.nb_states, self.nb_actions, **net_cfg) self.actor_target = Actor(self.nb_states, self.nb_actions, **net_cfg) self.actor_optim = Adam(self.actor.parameters(), lr=args.prate) self.critic = Critic(self.nb_states, self.nb_actions, **net_cfg) self.critic_target = Critic(self.nb_states, self.nb_actions, **net_cfg) self.critic_optim = Adam(self.critic.parameters(), lr=args.rate) hard_update(self.actor_target, self.actor) # Make sure target is with the same weight hard_update(self.critic_target, self.critic) # Create replay buffer self.memory = SequentialMemory(limit=args.rmsize, window_length=args.window_length) self.random_process = OrnsteinUhlenbeckProcess(size=nb_actions, theta=args.ou_theta, mu=args.ou_mu, sigma=args.ou_sigma) # Hyper-parameters self.batch_size = args.bsize self.tau = args.tau self.discount = args.discount self.depsilon = 1.0 / args.epsilon self.epsilon = 1.0 self.s_t = None # Most recent state self.a_t = None # Most recent action self.is_training = True if USE_CUDA: self.cuda() def update_policy(self): # Sample batch state_batch, action_batch, reward_batch, \ next_state_batch, terminal_batch = self.memory.sample_and_split(self.batch_size) # Prepare for the target q batch next_q_values = self.critic_target([ to_tensor(next_state_batch, volatile=True), self.actor_target(to_tensor(next_state_batch, volatile=True)), ]) next_q_values.volatile = False target_q_batch = to_tensor(reward_batch) + \ self.discount * to_tensor(terminal_batch.astype(np.float)) * next_q_values # Critic update self.critic.zero_grad() q_batch = self.critic([to_tensor(state_batch), to_tensor(action_batch)]) value_loss = criterion(q_batch, target_q_batch) value_loss.backward() self.critic_optim.step() # Actor update self.actor.zero_grad() policy_loss = -self.critic([ to_tensor(state_batch), self.actor(to_tensor(state_batch)) ]) policy_loss = policy_loss.mean() policy_loss.backward() self.actor_optim.step() # Target update soft_update(self.actor_target, self.actor, self.tau) soft_update(self.critic_target, self.critic, self.tau) def eval(self): self.actor.eval() self.actor_target.eval() self.critic.eval() self.critic_target.eval() def cuda(self): self.actor.cuda() self.actor_target.cuda() self.critic.cuda() self.critic_target.cuda() def observe(self, r_t, s_t1, done): if self.is_training: self.memory.append(self.s_t, self.a_t, r_t, done) self.s_t = s_t1 def random_action(self, distribution='uniform'): ''' Produce a random action ''' if distribution == 'uniform': action = np.random.uniform(-1., 1., self.nb_actions) # set the action internally to the agent self.a_t = action return action else: raise ValueError('Distribution {} not defined'.format(distribution)) def select_action(self, s_t, decay_epsilon=True, clip=None): ''' Pick action according to actor network. :param s_t: current state s_t :param decay_epsilon: bool. :param clip: tuple to clip action values between clip[0] and clip[1]. Default (-1, 1) Set to false if not clip. ''' # Set default for clip if None if clip is not False and clip is None: clip = (-1., 1.) action = to_numpy( self.actor(to_tensor(np.array([s_t]))) ).squeeze(0) # Add noise to the action. action += self.is_training * max(self.epsilon, 0) * self.random_process.sample() if clip is not False: if len(clip) != 2: raise ValueError('Clip parameter malformed, received {}, \ expected a size 2 tuple') action = np.clip(action, clip[0], clip[1]) if decay_epsilon: self.epsilon -= self.depsilon self.a_t = action return action def reset(self, obs): self.s_t = obs self.random_process.reset_states() def load_weights(self, output): if output is None: return self.actor.load_state_dict( torch.load('{}/actor.pkl'.format(output)) ) self.critic.load_state_dict( torch.load('{}/critic.pkl'.format(output)) ) def save_model(self, output): torch.save( self.actor.state_dict(), '{}/actor.pkl'.format(output) ) torch.save( self.critic.state_dict(), '{}/critic.pkl'.format(output) ) def seed(self, s): torch.manual_seed(s) if USE_CUDA: torch.cuda.manual_seed(s)
def run_agent(model_params, weights, state_transform, data_queue, weights_queue, process, global_step, updates, best_reward, param_noise_prob, save_dir, max_steps=10000000): train_fn, actor_fn, target_update_fn, params_actor, params_crit, actor_lr, critic_lr = \ build_model(**model_params) actor = Agent(actor_fn, params_actor, params_crit) actor.set_actor_weights(weights) env = RunEnv2(state_transform, max_obstacles=config.num_obstacles, skip_frame=config.skip_frames) random_process = OrnsteinUhlenbeckProcess(theta=.1, mu=0., sigma=.2, size=env.noutput, sigma_min=0.05, n_steps_annealing=1e6) # prepare buffers for data states = [] actions = [] rewards = [] terminals = [] total_episodes = 0 start = time() action_noise = True while global_step.value < max_steps: seed = random.randrange(2**32-2) state = env.reset(seed=seed, difficulty=2) random_process.reset_states() total_reward = 0. total_reward_original = 0. terminal = False steps = 0 while not terminal: state = np.asarray(state, dtype='float32') action = actor.act(state) if action_noise: action += random_process.sample() next_state, reward, next_terminal, info = env.step(action) total_reward += reward total_reward_original += info['original_reward'] steps += 1 global_step.value += 1 # add data to buffers states.append(state) actions.append(action) rewards.append(reward) terminals.append(terminal) state = next_state terminal = next_terminal if terminal: break total_episodes += 1 # add data to buffers after episode end states.append(state) actions.append(np.zeros(env.noutput)) rewards.append(0) terminals.append(terminal) states_np = np.asarray(states).astype(np.float32) data = (states_np, np.asarray(actions).astype(np.float32), np.asarray(rewards).astype(np.float32), np.asarray(terminals), ) weight_send = None if total_reward > best_reward.value: weight_send = actor.get_actor_weights() # send data for training data_queue.put((process, data, weight_send, total_reward)) # receive weights and set params to weights weights = weights_queue.get() report_str = 'Global step: {}, steps/sec: {:.2f}, updates: {}, episode len {}, ' \ 'reward: {:.2f}, original_reward {:.4f}; best reward: {:.2f} noise {}'. \ format(global_step.value, 1. * global_step.value / (time() - start), updates.value, steps, total_reward, total_reward_original, best_reward.value, 'actions' if action_noise else 'params') print(report_str) with open(os.path.join(save_dir, 'train_report.log'), 'a') as f: f.write(report_str + '\n') actor.set_actor_weights(weights) action_noise = np.random.rand() < 1 - param_noise_prob if not action_noise: set_params_noise(actor, states_np, random_process.current_sigma) # clear buffers del states[:] del actions[:] del rewards[:] del terminals[:] if total_episodes % 100 == 0: env = RunEnv2(state_transform, max_obstacles=config.num_obstacles, skip_frame=config.skip_frames)
class Agent(object): def __init__(self, nb_states, nb_actions, args): if args.seed > 0: self.seed(args.seed) self.nb_states = nb_states self.nb_actions = nb_actions # Create Actor and Critic Network self.actor = Actor(self.nb_states, self.nb_actions, args.init_w) self.actor_target = Actor(self.nb_states, self.nb_actions, args.init_w) self.critic = Critic(self.nb_states, self.nb_actions, args.init_w) self.critic_target = Critic(self.nb_states, self.nb_actions, args.init_w) hard_update(self.actor_target, self.actor) # Make sure target is with the same weight hard_update(self.critic_target, self.critic) #Create replay buffer self.random_process = OrnsteinUhlenbeckProcess(size=nb_actions, theta=args.ou_theta, mu=args.ou_mu, sigma=args.ou_sigma) # Hyper-parameters self.batch_size = args.bsize self.trajectory_length = args.trajectory_length self.tau = args.tau self.discount = args.discount self.depsilon = 1.0 / args.epsilon # self.epsilon = 1.0 self.is_training = True # if USE_CUDA: self.cuda() def eval(self): self.actor.eval() self.actor_target.eval() self.critic.eval() self.critic_target.eval() def random_action(self): action = np.random.uniform(-1., 1., self.nb_actions) return action def select_action(self, state, noise_enable=True, decay_epsilon=True): action, _ = self.actor(to_tensor(np.array([state]))) action = to_numpy(action).squeeze(0) if noise_enable == True: action += self.is_training * max(self.epsilon, 0) * self.random_process.sample() action = np.clip(action, -1., 1.) if decay_epsilon: self.epsilon -= self.depsilon return action def reset_lstm_hidden_state(self, done=True): self.actor.reset_lstm_hidden_state(done) def reset(self): self.random_process.reset_states() def cuda(self): self.actor.cuda() self.actor_target.cuda() self.critic.cuda() self.critic_target.cuda() def load_weights(self, output): if output is None: return False self.actor.load_state_dict(torch.load('{}/actor.pkl'.format(output))) self.critic.load_state_dict(torch.load('{}/critic.pkl'.format(output))) return True def save_model(self, output): if not os.path.exists(output): os.mkdir(output) torch.save(self.actor.state_dict(), '{}/actor.pkl'.format(output)) torch.save(self.critic.state_dict(), '{}/critic.pkl'.format(output))
class DDPG_trainer(object): def __init__(self, nb_state, nb_action): self.nb_state = nb_state self.nb_action = nb_action self.actor = Actor(self.nb_state, self.nb_action) self.actor_target = Actor(self.nb_state, self.nb_action) self.actor_optim = Adam(self.actor.parameters(), lr=LEARNING_RATE) self.critic = Critic(self.nb_state, self.nb_action) self.critic_target = Critic(self.nb_state, self.nb_action) self.critic_optim = Adam(self.critic.parameters(), lr=LEARNING_RATE) hard_update(self.actor_target, self.actor) # Make sure target is with the same weight hard_update(self.critic_target, self.critic) #Create replay buffer self.memory = SequentialMemory(limit=MEMORY_SIZE, window_length=1) self.random_process = OrnsteinUhlenbeckProcess(size=nb_action, theta=OU_THETA, mu=OU_MU, sigma=OU_SIGMA) self.is_training = True self.epsilon = 1.0 self.a_t = None self.s_t = None if USE_CUDA: self.cuda() def cuda(self): self.actor.cuda() self.actor_target.cuda() self.critic.cuda() self.critic_target.cuda() def select_action(self, s_t, decay_epsilon=True): action = to_numpy(self.actor(to_tensor(np.array([s_t])))).squeeze(0) action += self.is_training * max(self.epsilon, 0) * self.random_process.sample() action = np.clip(action, -1., 1.) if decay_epsilon: self.epsilon -= DELTA_EPSILON self.a_t = action return action def reset(self, observation): self.start_state = observation self.random_process.reset_states() def observe(self, r_t, s_t1, done): if self.is_training: self.memory.append(self.s_t, self.a_t, r_t, done) self.s_t = s_t1 def update_all(self): # Help Warm Up if self.memory.nb_entries < BATCH_SIZE * 2: return # Sample batch state_batch, action_batch, reward_batch, \ next_state_batch, terminal_batch = self.memory.sample_and_split(BATCH_SIZE) # Prepare for the target q batch with torch.no_grad(): next_q_values = self.critic_target([ to_tensor(next_state_batch), self.actor_target(to_tensor(next_state_batch)), ]) target_q_batch = to_tensor(reward_batch) + \ DISCOUNT * to_tensor(terminal_batch.astype(np.float)) * next_q_values # Critic update self.critic.zero_grad() for state in state_batch: if state.shape[0] <= 2: # print("Error sampled memory!") return q_batch = self.critic( [to_tensor(state_batch), to_tensor(action_batch)]) value_loss = CRITERION(q_batch, target_q_batch) value_loss.backward() self.critic_optim.step() # Actor update self.actor.zero_grad() policy_loss = -self.critic( [to_tensor(state_batch), self.actor(to_tensor(state_batch))]) policy_loss = policy_loss.mean() policy_loss.backward() self.actor_optim.step() # Target update soft_update(self.actor_target, self.actor, TAU) soft_update(self.critic_target, self.critic, TAU)
class DDPG(object): def __init__(self): # random seed for torch __seed = config.get(MODEL_SEED) self.policy_loss = [] self.critic_loss = [] if __seed > 0: self.seed(__seed) self.nb_states = config.get(MODEL_STATE_COUNT) self.nb_actions = config.get(MODEL_ACTION_COUNT) # Create Actor and Critic Network actor_net_cfg = { 'hidden1': config.get(MODEL_ACTOR_HIDDEN1), 'hidden2': config.get(MODEL_ACTOR_HIDDEN2), 'init_w': config.get(MODEL_INIT_WEIGHT) } critic_net_cfg = { 'hidden1': config.get(MODEL_CRITIC_HIDDEN1), 'hidden2': config.get(MODEL_CRITIC_HIDDEN2), 'init_w': config.get(MODEL_INIT_WEIGHT) } self.actor = Actor(self.nb_states, self.nb_actions, **actor_net_cfg) self.actor_target = Actor(self.nb_states, self.nb_actions, **actor_net_cfg) self.actor_optim = Adam( self.actor.parameters(), lr=config.get(MODEL_ACTOR_LR), weight_decay=config.get(MODEL_ACTOR_WEIGHT_DECAY)) self.critic = Critic(self.nb_states, self.nb_actions, **critic_net_cfg) self.critic_target = Critic(self.nb_states, self.nb_actions, **critic_net_cfg) self.critic_optim = Adam( self.critic.parameters(), lr=config.get(MODEL_CRITIC_LR), weight_decay=config.get(MODEL_CRITIC_WEIGHT_DECAY)) hard_update(self.actor_target, self.actor) hard_update(self.critic_target, self.critic) #Create replay buffer self.memory = Memory() self.random_process = OrnsteinUhlenbeckProcess( size=self.nb_actions, theta=config.get(RANDOM_THETA), mu=config.get(RANDOM_MU), sigma=config.get(RANDOM_SIGMA)) # Hyper-parameters self.batch_size = config.get(MODEL_BATCH_SIZE) self.tau = config.get(MODEL_TARGET_TAU) self.discount = config.get(MODEL_DISCOUNT) self.depsilon = 1.0 / config.get(MODEL_EPSILON) self.model_path = config.get(MODEL_SAVE_PATH) # self.epsilon = 1.0 # init device self.device_init() def update_policy(self, memory): # Sample batch state_batch, action_batch, reward_batch, \ next_state_batch, terminal_batch = memory.sample_and_split(self.batch_size) # Prepare for the target q batch with torch.no_grad(): next_q_values = self.critic_target([ to_tensor(next_state_batch), self.actor_target(to_tensor(next_state_batch)) ]) target_q_batch = to_tensor(reward_batch) + \ self.discount*to_tensor(terminal_batch.astype(np.float))*next_q_values # Critic update self.critic.zero_grad() q_batch = self.critic( [to_tensor(state_batch), to_tensor(action_batch)]) value_loss = F.mse_loss(q_batch, target_q_batch) value_loss.backward() self.critic_optim.step() self.critic_loss.append(value_loss.data[0]) # Actor update self.actor.zero_grad() policy_loss = -self.critic( [to_tensor(state_batch), self.actor(to_tensor(state_batch))]) policy_loss = policy_loss.mean() policy_loss.backward() self.actor_optim.step() self.policy_loss.append(policy_loss.data[0]) # Target update soft_update(self.actor_target, self.actor, self.tau) soft_update(self.critic_target, self.critic, self.tau) def get_loss(self): return self.policy_loss, self.critic_loss def eval(self): self.actor.eval() self.actor_target.eval() self.critic.eval() self.critic_target.eval() def device_init(self): self.actor.to(device) self.actor_target.to(device) self.critic.to(device) self.critic_target.to(device) def random_action(self): action = np.random.uniform(-1., 1., self.nb_actions) return action def select_action(self, s_t): action = to_numpy(self.actor(to_tensor(np.array([s_t])))).squeeze(0) action += max(self.epsilon, 0) * self.random_process.sample() action = np.clip(action, -1., 1.) return action def clean(self, decay_epsilon): if decay_epsilon: self.epsilon -= self.depsilon def reset(self): self.random_process.reset_states() def load_weights(self): if not os.path.exists(self.model_path): return actor_path = os.path.exists(os.path.join(self.model_path, 'actor.pkl')) if os.path.exists(actor_path): self.actor.load_state_dict(torch.load(actor_path)) critic_path = os.path.exists( os.path.join(self.model_path, 'critic.pkl')) if os.path.exists(critic_path): self.critic.load_state_dict(torch.load(critic_path)) def save_model(self): if not os.path.exists(self.model_path): os.makedirs(self.model_path) actor_path = os.path.exists(os.path.join(self.model_path, 'actor.pkl')) torch.save(self.actor.state_dict(), actor_path) critic_path = os.path.exists( os.path.join(self.model_path, 'critic.pkl')) torch.save(self.critic.state_dict(), critic_path) def get_model(self): return self.actor.state_dict(), self.critic.state_dict() def load_state_dict(self, actor_state, critic_state): self.actor.load_state_dict(actor_state) self.critic.load_state_dict(critic_state) def seed(self, s): torch.manual_seed(s) if USE_CUDA: torch.cuda.manual_seed(s)