def _init_model(self): """init model from parameters""" self.env, env_continuous, num_states, num_actions = get_env_info( self.env_id) # seeding torch.manual_seed(self.seed) self.env.seed(self.seed) if env_continuous: self.policy_net = Policy(num_states, num_actions).double().to( device) # current policy else: self.policy_net = DiscretePolicy(num_states, num_actions).double().to(device) self.value_net = Value(num_states).double().to(device) self.running_state = ZFilter((num_states, ), clip=5) if self.model_path: print("Loading Saved Model {}_trpo.p".format(self.env_id)) self.policy_net, self.value_net, self.running_state = pickle.load( open('{}/{}_trpo.p'.format(self.model_path, self.env_id), "rb")) self.collector = MemoryCollector(self.env, self.policy_net, render=self.render, running_state=self.running_state, num_process=self.num_process) self.optimizer_v = optim.Adam(self.value_net.parameters(), lr=self.lr_v)
def _init_model(self): """init model from parameters""" self.env, env_continuous, num_states, self.num_actions = get_env_info(self.env_id) assert env_continuous, "SAC is only applicable to continuous environment !!!!" self.action_low, self.action_high = self.env.action_space.low[0], self.env.action_space.high[0] # seeding np.random.seed(self.seed) torch.manual_seed(self.seed) self.env.seed(self.seed) self.policy_net = Policy(num_states, self.num_actions, max_action=self.action_high).double().to(device) self.value_net = Value(num_states).double().to(device) self.value_net_target = Value(num_states).double().to(device) self.q_net_1 = QValue(num_states, self.num_actions).double().to(device) self.q_net_2 = QValue(num_states, self.num_actions).double().to(device) self.running_state = ZFilter((num_states,), clip=5) if self.model_path: print("Loading Saved Model {}_sac.p".format(self.env_id)) self.policy_net, self.value_net, self.q_net_1, self.q_net_2, self.running_state \ = pickle.load(open('{}/{}_sac.p'.format(self.model_path, self.env_id), "rb")) self.value_net_target.load_state_dict(self.value_net.state_dict()) self.optimizer_p = optim.Adam(self.policy_net.parameters(), lr=self.lr_p) self.optimizer_v = optim.Adam(self.value_net.parameters(), lr=self.lr_v) self.optimizer_q_1 = optim.Adam(self.q_net_1.parameters(), lr=self.lr_q) self.optimizer_q_2 = optim.Adam(self.q_net_2.parameters(), lr=self.lr_q)
def _init_model(self): # seeding seed = self.config["train"]["general"]["seed"] torch.manual_seed(seed) np.random.seed(seed) self.env, env_continuous, num_states, num_actions = get_env_info( self.env_id) # check env assert num_states == self.expert_dataset.num_states and num_actions == self.expert_dataset.num_actions, \ "Expected corresponding expert dataset and env" dim_dict = {"dim_state": num_states, "dim_action": num_actions} self.config["value"].update(dim_dict) self.config["policy"].update(dim_dict) self.config["discriminator"].update(dim_dict) self.value = Value(dim_state=self.config["value"]["dim_state"], dim_hidden=self.config["value"]["dim_hidden"], activation=resolve_activate_function( self.config["value"]["activation"])) self.policy = Policy(config=self.config["policy"]) self.discriminator = Discriminator( dim_state=self.config["discriminator"]["dim_state"], dim_action=self.config["discriminator"]["dim_action"], dim_hidden=self.config["discriminator"]["dim_hidden"], activation=resolve_activate_function( self.config["discriminator"]["activation"])) self.discriminator_func = nn.BCELoss() self.running_state = None self.collector = MemoryCollector(self.env, self.policy, render=self.render, running_state=self.running_state, num_process=self.num_process) print("Model Structure") print(self.policy) print(self.value) print(self.discriminator) print() self.optimizer_policy = optim.Adam( self.policy.parameters(), lr=self.config["policy"]["learning_rate"]) self.optimizer_value = optim.Adam( self.value.parameters(), lr=self.config["value"]["learning_rate"]) self.optimizer_discriminator = optim.Adam( self.discriminator.parameters(), lr=self.config["discriminator"]["learning_rate"]) to_device(self.value, self.policy, self.discriminator, self.discriminator_func)
class SAC: def __init__(self, env_id, render=False, num_process=1, memory_size=1000000, lr_p=1e-3, lr_v=1e-3, lr_q=1e-3, gamma=0.99, polyak=0.995, explore_size=10000, step_per_iter=3000, batch_size=100, min_update_step=1000, update_step=50, target_update_delay=1, seed=1, model_path=None): self.env_id = env_id self.gamma = gamma self.polyak = polyak self.memory = FixedMemory(memory_size) self.explore_size = explore_size self.step_per_iter = step_per_iter self.render = render self.num_process = num_process self.lr_p = lr_p self.lr_v = lr_v self.lr_q = lr_q self.batch_size = batch_size self.min_update_step = min_update_step self.update_step = update_step self.target_update_delay = target_update_delay self.model_path = model_path self.seed = seed self._init_model() def _init_model(self): """init model from parameters""" self.env, env_continuous, num_states, self.num_actions = get_env_info( self.env_id) assert env_continuous, "SAC is only applicable to continuous environment !!!!" self.action_low, self.action_high = self.env.action_space.low[ 0], self.env.action_space.high[0] # seeding np.random.seed(self.seed) torch.manual_seed(self.seed) self.env.seed(self.seed) self.policy_net = Policy(num_states, self.num_actions, max_action=self.action_high, use_sac=True).to(device) self.value_net = Value(num_states).to(device) self.value_net_target = Value(num_states).to(device) self.q_net_1 = QValue(num_states, self.num_actions).to(device) self.q_net_2 = QValue(num_states, self.num_actions).to(device) self.running_state = ZFilter((num_states, ), clip=5) if self.model_path: print("Loading Saved Model {}_sac.p".format(self.env_id)) self.policy_net, self.value_net, self.q_net_1, self.q_net_2, self.running_state \ = pickle.load(open('{}/{}_sac.p'.format(self.model_path, self.env_id), "rb")) self.value_net_target.load_state_dict(self.value_net.state_dict()) self.optimizer_p = optim.Adam(self.policy_net.parameters(), lr=self.lr_p) self.optimizer_v = optim.Adam(self.value_net.parameters(), lr=self.lr_v) self.optimizer_q_1 = optim.Adam(self.q_net_1.parameters(), lr=self.lr_q) self.optimizer_q_2 = optim.Adam(self.q_net_2.parameters(), lr=self.lr_q) def choose_action(self, state): """select action""" state = FLOAT(state).unsqueeze(0).to(device) with torch.no_grad(): action, _ = self.policy_net.rsample(state) action = action.cpu().numpy()[0] return action, None def eval(self, i_iter, render=False): """evaluate model""" state = self.env.reset() test_reward = 0 while True: if render: self.env.render() state = self.running_state(state) action, _ = self.choose_action(state) state, reward, done, _ = self.env.step(action) test_reward += reward if done: break print(f"Iter: {i_iter}, test Reward: {test_reward}") self.env.close() def learn(self, writer, i_iter): """interact""" global_steps = (i_iter - 1) * self.step_per_iter + 1 log = dict() num_steps = 0 num_episodes = 0 total_reward = 0 min_episode_reward = float('inf') max_episode_reward = float('-inf') while num_steps < self.step_per_iter: state = self.env.reset() state = self.running_state(state) episode_reward = 0 for t in range(10000): if self.render: self.env.render() if global_steps < self.explore_size: # explore action = self.env.action_space.sample() else: # action action, _ = self.choose_action(state) next_state, reward, done, _ = self.env.step(action) next_state = self.running_state(next_state) mask = 0 if done else 1 # ('state', 'action', 'reward', 'next_state', 'mask', 'log_prob') self.memory.push(state, action, reward, next_state, mask, None) episode_reward += reward global_steps += 1 num_steps += 1 if global_steps >= self.min_update_step and global_steps % self.update_step == 0: for k in range(1, self.update_step + 1): batch = self.memory.sample( self.batch_size) # random sample batch self.update(batch, k) if done or num_steps >= self.step_per_iter: break state = next_state num_episodes += 1 total_reward += episode_reward min_episode_reward = min(episode_reward, min_episode_reward) max_episode_reward = max(episode_reward, max_episode_reward) self.env.close() log['num_steps'] = num_steps log['num_episodes'] = num_episodes log['total_reward'] = total_reward log['avg_reward'] = total_reward / num_episodes log['max_episode_reward'] = max_episode_reward log['min_episode_reward'] = min_episode_reward print( f"Iter: {i_iter}, num steps: {log['num_steps']}, total reward: {log['total_reward']: .4f}, " f"min reward: {log['min_episode_reward']: .4f}, max reward: {log['max_episode_reward']: .4f}, " f"average reward: {log['avg_reward']: .4f}") # record reward information writer.add_scalars( "sac", { "total reward": log['total_reward'], "average reward": log['avg_reward'], "min reward": log['min_episode_reward'], "max reward": log['max_episode_reward'], "num steps": log['num_steps'] }, i_iter) def update(self, batch, k_iter): """learn model""" batch_state = FLOAT(batch.state).to(device) batch_action = FLOAT(batch.action).to(device) batch_reward = FLOAT(batch.reward).to(device) batch_next_state = FLOAT(batch.next_state).to(device) batch_mask = FLOAT(batch.mask).to(device) # update by SAC sac_step(self.policy_net, self.value_net, self.value_net_target, self.q_net_1, self.q_net_2, self.optimizer_p, self.optimizer_v, self.optimizer_q_1, self.optimizer_q_2, batch_state, batch_action, batch_reward, batch_next_state, batch_mask, self.gamma, self.polyak, k_iter % self.target_update_delay == 0) def save(self, save_path): """save model""" check_path(save_path) pickle.dump((self.policy_net, self.value_net, self.q_net_1, self.q_net_2, self.running_state), open('{}/{}_sac.p'.format(save_path, self.env_id), 'wb'))
class TRPO: def __init__(self, env_id, render=False, num_process=1, min_batch_size=2048, lr_v=3e-4, gamma=0.99, tau=0.95, max_kl=1e-2, damping=1e-2, seed=1, model_path=None): self.env_id = env_id self.gamma = gamma self.tau = tau self.max_kl = max_kl self.damping = damping self.render = render self.num_process = num_process self.lr_v = lr_v self.min_batch_size = min_batch_size self.model_path = model_path self.seed = seed self._init_model() def _init_model(self): """init model from parameters""" self.env, env_continuous, num_states, num_actions = get_env_info( self.env_id) # seeding torch.manual_seed(self.seed) self.env.seed(self.seed) if env_continuous: self.policy_net = Policy(num_states, num_actions).double().to( device) # current policy else: self.policy_net = DiscretePolicy(num_states, num_actions).double().to(device) self.value_net = Value(num_states).double().to(device) self.running_state = ZFilter((num_states, ), clip=5) if self.model_path: print("Loading Saved Model {}_trpo.p".format(self.env_id)) self.policy_net, self.value_net, self.running_state = pickle.load( open('{}/{}_trpo.p'.format(self.model_path, self.env_id), "rb")) self.collector = MemoryCollector(self.env, self.policy_net, render=self.render, running_state=self.running_state, num_process=self.num_process) self.optimizer_v = optim.Adam(self.value_net.parameters(), lr=self.lr_v) def choose_action(self, state): """select action""" state = DOUBLE(state).unsqueeze(0).to(device) with torch.no_grad(): action, log_prob = self.policy_net.get_action_log_prob(state) return action, log_prob def eval(self, i_iter, render=False): """evaluate model""" state = self.env.reset() test_reward = 0 while True: if render: self.env.render() state = self.running_state(state) action, _ = self.choose_action(state) action = action.cpu().numpy()[0] state, reward, done, _ = self.env.step(action) test_reward += reward if done: break print(f"Iter: {i_iter}, test Reward: {test_reward}") self.env.close() def learn(self, writer, i_iter): """learn model""" memory, log = self.collector.collect_samples(self.min_batch_size) print( f"Iter: {i_iter}, num steps: {log['num_steps']}, total reward: {log['total_reward']: .4f}, " f"min reward: {log['min_episode_reward']: .4f}, max reward: {log['max_episode_reward']: .4f}, " f"average reward: {log['avg_reward']: .4f}, sample time: {log['sample_time']: .4f}" ) # record reward information writer.add_scalars( "trpo", { "total reward": log['total_reward'], "average reward": log['avg_reward'], "min reward": log['min_episode_reward'], "max reward": log['max_episode_reward'], "num steps": log['num_steps'] }, i_iter) batch = memory.sample() # sample all items in memory batch_state = DOUBLE(batch.state).to(device) batch_action = DOUBLE(batch.action).to(device) batch_reward = DOUBLE(batch.reward).to(device) batch_mask = DOUBLE(batch.mask).to(device) batch_log_prob = DOUBLE(batch.log_prob).to(device) with torch.no_grad(): batch_value = self.value_net(batch_state) batch_advantage, batch_return = estimate_advantages( batch_reward, batch_mask, batch_value, self.gamma, self.tau) # update by TRPO trpo_step(self.policy_net, self.value_net, batch_state, batch_action, batch_return, batch_advantage, batch_log_prob, self.max_kl, self.damping, 1e-3, None) def save(self, save_path): """save model""" check_path(save_path) pickle.dump((self.policy_net, self.value_net, self.running_state), open('{}/{}_trpo.p'.format(save_path, self.env_id), 'wb'))
class PPO: def __init__(self, env_id, render=False, num_process=4, min_batch_size=2048, lr_p=3e-4, lr_v=3e-4, gamma=0.99, tau=0.95, clip_epsilon=0.2, ppo_epochs=10, ppo_mini_batch_size=64, seed=1, model_path=None): self.env_id = env_id self.gamma = gamma self.tau = tau self.ppo_epochs = ppo_epochs self.ppo_mini_batch_size = ppo_mini_batch_size self.clip_epsilon = clip_epsilon self.render = render self.num_process = num_process self.lr_p = lr_p self.lr_v = lr_v self.min_batch_size = min_batch_size self.model_path = model_path self.seed = seed self._init_model() def _init_model(self): """init model from parameters""" self.env, env_continuous, num_states, num_actions = get_env_info( self.env_id) # seeding torch.manual_seed(self.seed) self.env.seed(self.seed) if env_continuous: self.policy_net = Policy(num_states, num_actions).to(device) else: self.policy_net = DiscretePolicy(num_states, num_actions).to(device) self.value_net = Value(num_states).to(device) self.running_state = ZFilter((num_states, ), clip=5) if self.model_path: print("Loading Saved Model {}_ppo.p".format(self.env_id)) self.policy_net, self.value_net, self.running_state = pickle.load( open('{}/{}_ppo.p'.format(self.model_path, self.env_id), "rb")) self.collector = MemoryCollector(self.env, self.policy_net, render=self.render, running_state=self.running_state, num_process=self.num_process) self.optimizer_p = optim.Adam(self.policy_net.parameters(), lr=self.lr_p) self.optimizer_v = optim.Adam(self.value_net.parameters(), lr=self.lr_v) def choose_action(self, state): """select action""" state = FLOAT(state).unsqueeze(0).to(device) with torch.no_grad(): action, log_prob = self.policy_net.get_action_log_prob(state) return action, log_prob def eval(self, i_iter, render=False): state = self.env.reset() test_reward = 0 while True: if render: self.env.render() state = self.running_state(state) action, _ = self.choose_action(state) action = action.cpu().numpy()[0] state, reward, done, _ = self.env.step(action) test_reward += reward if done: break print(f"Iter: {i_iter}, test Reward: {test_reward}") self.env.close() def learn(self, writer, i_iter): """learn model""" memory, log = self.collector.collect_samples(self.min_batch_size) print( f"Iter: {i_iter}, num steps: {log['num_steps']}, total reward: {log['total_reward']: .4f}, " f"min reward: {log['min_episode_reward']: .4f}, max reward: {log['max_episode_reward']: .4f}, " f"average reward: {log['avg_reward']: .4f}, sample time: {log['sample_time']: .4f}" ) # record reward information writer.add_scalars( "ppo", { "total reward": log['total_reward'], "average reward": log['avg_reward'], "min reward": log['min_episode_reward'], "max reward": log['max_episode_reward'], "num steps": log['num_steps'] }, i_iter) batch = memory.sample() # sample all items in memory # ('state', 'action', 'reward', 'next_state', 'mask', 'log_prob') batch_state = FLOAT(batch.state).to(device) batch_action = FLOAT(batch.action).to(device) batch_reward = FLOAT(batch.reward).to(device) batch_mask = FLOAT(batch.mask).to(device) batch_log_prob = FLOAT(batch.log_prob).to(device) with torch.no_grad(): batch_value = self.value_net(batch_state) batch_advantage, batch_return = estimate_advantages( batch_reward, batch_mask, batch_value, self.gamma, self.tau) v_loss, p_loss = torch.empty(1), torch.empty(1) for _ in range(self.ppo_epochs): if self.ppo_mini_batch_size: batch_size = batch_state.shape[0] mini_batch_num = int( math.ceil(batch_size / self.ppo_mini_batch_size)) # update with mini-batch for _ in range(self.ppo_epochs): index = torch.randperm(batch_size) for i in range(mini_batch_num): ind = index[slice( i * self.ppo_mini_batch_size, min(batch_size, (i + 1) * self.ppo_mini_batch_size))] state, action, returns, advantages, old_log_pis = batch_state[ind], batch_action[ind], \ batch_return[ ind], batch_advantage[ind], \ batch_log_prob[ ind] v_loss, p_loss = ppo_step( self.policy_net, self.value_net, self.optimizer_p, self.optimizer_v, 1, state, action, returns, advantages, old_log_pis, self.clip_epsilon, 1e-3) else: v_loss, p_loss = ppo_step(self.policy_net, self.value_net, self.optimizer_p, self.optimizer_v, 1, batch_state, batch_action, batch_return, batch_advantage, batch_log_prob, self.clip_epsilon, 1e-3) return v_loss, p_loss def save(self, save_path): """save model""" check_path(save_path) pickle.dump((self.policy_net, self.value_net, self.running_state), open('{}/{}_ppo.p'.format(save_path, self.env_id), 'wb'))
class GAIL: def __init__(self, render=False, num_process=4, config=None, expert_data_path=None, env_id=None): self.render = render self.env_id = env_id self.num_process = num_process self.expert_data_path = expert_data_path self.config = config self._load_expert_trajectory() self._init_model() def _load_expert_trajectory(self): self.expert_dataset = ExpertDataset( expert_data_path=self.expert_data_path, train_fraction=self.config["expert_data"]["train_fraction"], traj_limitation=self.config["expert_data"]["traj_limitation"], shuffle=self.config["expert_data"]["shuffle"], batch_size=self.config["expert_data"]["batch_size"]) def _init_model(self): # seeding seed = self.config["train"]["general"]["seed"] torch.manual_seed(seed) np.random.seed(seed) self.env, env_continuous, num_states, num_actions = get_env_info( self.env_id) # check env assert num_states == self.expert_dataset.num_states and num_actions == self.expert_dataset.num_actions, \ "Expected corresponding expert dataset and env" dim_dict = {"dim_state": num_states, "dim_action": num_actions} self.config["value"].update(dim_dict) self.config["policy"].update(dim_dict) self.config["discriminator"].update(dim_dict) self.value = Value(dim_state=self.config["value"]["dim_state"], dim_hidden=self.config["value"]["dim_hidden"], activation=resolve_activate_function( self.config["value"]["activation"])) self.policy = Policy(config=self.config["policy"]) self.discriminator = Discriminator( dim_state=self.config["discriminator"]["dim_state"], dim_action=self.config["discriminator"]["dim_action"], dim_hidden=self.config["discriminator"]["dim_hidden"], activation=resolve_activate_function( self.config["discriminator"]["activation"])) self.discriminator_func = nn.BCELoss() self.running_state = None self.collector = MemoryCollector(self.env, self.policy, render=self.render, running_state=self.running_state, num_process=self.num_process) print("Model Structure") print(self.policy) print(self.value) print(self.discriminator) print() self.optimizer_policy = optim.Adam( self.policy.parameters(), lr=self.config["policy"]["learning_rate"]) self.optimizer_value = optim.Adam( self.value.parameters(), lr=self.config["value"]["learning_rate"]) self.optimizer_discriminator = optim.Adam( self.discriminator.parameters(), lr=self.config["discriminator"]["learning_rate"]) to_device(self.value, self.policy, self.discriminator, self.discriminator_func) def choose_action(self, state): """select action""" state = FLOAT(state).unsqueeze(0).to(device) with torch.no_grad(): action, log_prob = self.policy.get_action_log_prob(state) return action, log_prob def learn(self, writer, i_iter): memory, log = self.collector.collect_samples( self.config["train"]["generator"]["sample_batch_size"]) self.policy.train() self.value.train() self.discriminator.train() print( f"Iter: {i_iter}, num steps: {log['num_steps']}, total reward: {log['total_reward']: .4f}, " f"min reward: {log['min_episode_reward']: .4f}, max reward: {log['max_episode_reward']: .4f}, " f"average reward: {log['avg_reward']: .4f}, sample time: {log['sample_time']: .4f}" ) # record reward information writer.add_scalar("gail/average reward", log['avg_reward'], i_iter) writer.add_scalar("gail/num steps", log['num_steps'], i_iter) # collect generated batch # gen_batch = self.collect_samples(self.config["ppo"]["sample_batch_size"]) gen_batch = memory.sample() gen_batch_state = FLOAT(gen_batch.state).to( device) # [batch size, state size] gen_batch_action = FLOAT(gen_batch.action).to( device) # [batch size, action size] gen_batch_old_log_prob = FLOAT(gen_batch.log_prob).to( device) # [batch size, 1] gen_batch_mask = FLOAT(gen_batch.mask).to(device) # [batch, 1] #################################################### # update discriminator #################################################### d_optim_i_iters = self.config["train"]["discriminator"]["optim_step"] if i_iter % d_optim_i_iters == 0: for step, (expert_batch_state, expert_batch_action) in enumerate( self.expert_dataset.train_loader): if step >= d_optim_i_iters: break # calculate probs and logits gen_prob, gen_logits = self.discriminator( gen_batch_state, gen_batch_action) expert_prob, expert_logits = self.discriminator( expert_batch_state.to(device), expert_batch_action.to(device)) # calculate accuracy gen_acc = torch.mean((gen_prob < 0.5).float()) expert_acc = torch.mean((expert_prob > 0.5).float()) # calculate regression loss expert_labels = torch.ones_like(expert_prob) gen_labels = torch.zeros_like(gen_prob) e_loss = self.discriminator_func(expert_prob, target=expert_labels) g_loss = self.discriminator_func(gen_prob, target=gen_labels) d_loss = e_loss + g_loss # calculate entropy loss logits = torch.cat([gen_logits, expert_logits], 0) entropy = ((1. - torch.sigmoid(logits)) * logits - torch.nn.functional.logsigmoid(logits)).mean() entropy_loss = - \ self.config["train"]["discriminator"]["ent_coeff"] * entropy total_loss = d_loss + entropy_loss self.optimizer_discriminator.zero_grad() total_loss.backward() self.optimizer_discriminator.step() writer.add_scalar('discriminator/d_loss', d_loss.item(), i_iter) writer.add_scalar("discriminator/e_loss", e_loss.item(), i_iter) writer.add_scalar("discriminator/g_loss", g_loss.item(), i_iter) writer.add_scalar("discriminator/ent", entropy.item(), i_iter) writer.add_scalar('discriminator/expert_acc', gen_acc.item(), i_iter) writer.add_scalar('discriminator/gen_acc', expert_acc.item(), i_iter) #################################################### # update policy by ppo [mini_batch] #################################################### with torch.no_grad(): gen_batch_value = self.value(gen_batch_state) d_out, _ = self.discriminator(gen_batch_state, gen_batch_action) gen_batch_reward = -torch.log(1 - d_out + 1e-6) gen_batch_advantage, gen_batch_return = estimate_advantages( gen_batch_reward, gen_batch_mask, gen_batch_value, self.config["train"]["generator"]["gamma"], self.config["train"]["generator"]["tau"]) ppo_optim_i_iters = self.config["train"]["generator"]["optim_step"] ppo_mini_batch_size = self.config["train"]["generator"][ "mini_batch_size"] for _ in range(ppo_optim_i_iters): if ppo_mini_batch_size > 0: gen_batch_size = gen_batch_state.shape[0] optim_iter_num = int( math.ceil(gen_batch_size / ppo_mini_batch_size)) perm = torch.randperm(gen_batch_size) for i in range(optim_iter_num): ind = perm[slice( i * ppo_mini_batch_size, min((i + 1) * ppo_mini_batch_size, gen_batch_size))] mini_batch_state, mini_batch_action, mini_batch_advantage, mini_batch_return, \ mini_batch_old_log_prob = gen_batch_state[ind], gen_batch_action[ind], \ gen_batch_advantage[ind], gen_batch_return[ind], gen_batch_old_log_prob[ ind] v_loss, p_loss, ent_loss = ppo_step( policy_net=self.policy, value_net=self.value, optimizer_policy=self.optimizer_policy, optimizer_value=self.optimizer_value, optim_value_iternum=self.config["value"] ["optim_value_iter"], states=mini_batch_state, actions=mini_batch_action, returns=mini_batch_return, old_log_probs=mini_batch_old_log_prob, advantages=mini_batch_advantage, clip_epsilon=self.config["train"]["generator"] ["clip_ratio"], l2_reg=self.config["value"]["l2_reg"]) else: v_loss, p_loss, ent_loss = ppo_step( policy_net=self.policy, value_net=self.value, optimizer_policy=self.optimizer_policy, optimizer_value=self.optimizer_value, optim_value_iternum=self.config["value"] ["optim_value_iter"], states=gen_batch_state, actions=gen_batch_action, returns=gen_batch_return, old_log_probs=gen_batch_old_log_prob, advantages=gen_batch_advantage, clip_epsilon=self.config["train"]["generator"] ["clip_ratio"], l2_reg=self.config["value"]["l2_reg"]) writer.add_scalar('generator/p_loss', p_loss, i_iter) writer.add_scalar('generator/v_loss', v_loss, i_iter) writer.add_scalar('generator/ent_loss', ent_loss, i_iter) print(f" Training episode:{i_iter} ".center(80, "#")) print('d_gen_prob:', gen_prob.mean().item()) print('d_expert_prob:', expert_prob.mean().item()) print('d_loss:', d_loss.item()) print('e_loss:', e_loss.item()) print("d/bernoulli_entropy:", entropy.item()) def eval(self, i_iter, render=False): self.policy.eval() self.value.eval() self.discriminator.eval() state = self.env.reset() test_reward = 0 while True: if render: self.env.render() if self.running_state: state = self.running_state(state) action, _ = self.choose_action(state) action = action.cpu().numpy()[0] state, reward, done, _ = self.env.step(action) test_reward += reward if done: break print(f"Iter: {i_iter}, test Reward: {test_reward}") self.env.close() def save_model(self, save_path): check_path(save_path) # torch.save((self.discriminator, self.policy, self.value), f"{save_path}/{self.exp_name}.pt") torch.save(self.discriminator, f"{save_path}/{self.env_id}_Discriminator.pt") torch.save(self.policy, f"{save_path}/{self.env_id}_Policy.pt") torch.save(self.value, f"{save_path}/{self.env_id}_Value.pt") def load_model(self, model_path): # load entire model # self.discriminator, self.policy, self.value = torch.load(model_path, map_location=device) self.discriminator = torch.load(f"{model_path}_Discriminator.pt", map_location=device) self.policy = torch.load(f"{model_path}_Policy.pt", map_location=device) self.value = torch.load(f"{model_path}_Value.pt", map_location=device)