def train_agent_discrete( class_agent, env_name, cwd, net_dim, max_step, max_memo, max_epoch, # env batch_size, gamma, update_gap, reward_scale, **_kwargs): # 2020-05-20 env = gym.make(env_name) '''init''' state_dim, action_dim, action_max, target_reward = get_env_info( env, is_print=True) assert isinstance(action_max, int) # means Discrete action space agent = class_agent(env, state_dim, action_dim, net_dim) # training agent buffer = BufferArray(max_memo, state_dim, action_dim=1) # experiment replay buffer recorder = Recorder(agent, max_step, action_max, target_reward, env_name, **_kwargs) '''loop''' with torch.no_grad(): # update replay buffer rewards, steps = initial_exploration(env, buffer, max_step, action_max, reward_scale, gamma, action_dim) recorder.show_reward(rewards, steps, 0, 0) try: for epoch in range(max_epoch): '''update replay buffer by interact with environment''' with torch.no_grad(): # for saving the GPU buffer rewards, steps = agent.update_buffer(env, buffer, max_step, action_max, reward_scale, gamma) '''update network parameters by random sampling buffer for stochastic gradient descent''' loss_a, loss_c = agent.update_parameters(buffer, max_step, batch_size, update_gap) '''show/check the reward, save the max reward actor''' with torch.no_grad(): # for saving the GPU buffer '''NOTICE! Recorder saves the agent with max reward automatically. ''' recorder.show_reward(rewards, steps, loss_a, loss_c) is_solved = recorder.check_reward(cwd, loss_a, loss_c) if is_solved: break except KeyboardInterrupt: print("raise KeyboardInterrupt while training.") # except AssertionError: # for BipedWalker BUG 2020-03-03 # print("AssertionError: OpenAI gym r.LengthSquared() > 0.0f ??? Please run again.") # return False train_time = recorder.print_and_save_npy(env_name, cwd) # agent.save_or_load_model(cwd, is_save=True) # save max reward agent in Recorder # buffer.save_or_load_memo(cwd, is_save=True) draw_plot_with_npy(cwd, train_time) return True
def train_agent__off_policy(class_agent, net_dim, batch_size, repeat_times, gamma, reward_scale, cwd, env_name, max_step, max_memo, max_epoch, **_kwargs): # 2020-06-01 env = gym.make(env_name) state_dim, action_dim, max_action, target_reward, is_discrete = get_env_info( env, is_print=False) '''init''' agent = class_agent(state_dim, action_dim, net_dim) # training agent agent.state = env.reset() buffer = BufferArray(max_memo, state_dim, action_dim=1 if is_discrete else action_dim) # experiment replay buffer recorder = Recorder(agent, max_step, max_action, target_reward, env_name, **_kwargs) # unnecessary '''loop''' with torch.no_grad(): # update replay buffer # rewards, steps = agent.update_buffer(env, buffer, max_step, max_action, reward_scale, gamma) rewards, steps = initial_exploration(env, buffer, max_step, max_action, reward_scale, gamma, action_dim) recorder.show_reward(rewards, steps, loss_a=0, loss_c=0) try: for epoch in range(max_epoch): # update replay buffer by interact with environment with torch.no_grad(): # for saving the GPU buffer rewards, steps = agent.update_buffer(env, buffer, max_step, max_action, reward_scale, gamma) # update network parameters by random sampling buffer for gradient descent buffer.init_before_sample() loss_a, loss_c = agent.update_parameters(buffer, max_step, batch_size, repeat_times) # show/check the reward, save the max reward actor with torch.no_grad(): # for saving the GPU buffer # NOTICE! Recorder saves the agent with max reward automatically. recorder.show_reward(rewards, steps, loss_a, loss_c) is_solved = recorder.check_reward(cwd, loss_a, loss_c) if is_solved: break except KeyboardInterrupt: print("| raise KeyboardInterrupt and break training loop") # except AssertionError: # for BipedWalker BUG 2020-03-03 # print("AssertionError: OpenAI gym r.LengthSquared() > 0.0f ??? Please run again.") train_time = recorder.print_and_save_npy(env_name, cwd) if is_solved: agent.save_or_load_model(cwd, is_save=True) # buffer.save_or_load_memo(cwd, is_save=True) draw_plot_with_npy(cwd, train_time)
def train_agent_ppo(class_agent, net_dim, batch_size, repeat_times, gamma, reward_scale, cwd, env_name, max_step, max_memo, max_epoch, **_kwargs): # 2020-0430 env = gym.make(env_name) state_dim, action_dim, max_action, target_reward, is_discrete = get_env_info( env, is_print=False) agent = class_agent(state_dim, action_dim, net_dim) agent.save_or_load_model(cwd, is_save=False) recorder = Recorder(agent, max_step, max_action, target_reward, env_name) # r_norm = RewardNorm(n_max=target_reward, n_min=recorder.reward_avg) # running_state = ZFilter((state_dim,), clip=5.0) try: for epoch in range(max_epoch): with torch.no_grad(): # just the GPU memory rewards, steps, memory = agent.inactive_in_env_ppo( env, max_step, max_memo, max_action, reward_scale, gamma) loss_a, loss_c = agent.update_parameter_ppo( memory, batch_size, repeat_times) with torch.no_grad(): # just the GPU memory recorder.show_reward(rewards, steps, loss_a, loss_c) is_solved = recorder.check_reward(cwd, loss_a, loss_c) if is_solved: break except KeyboardInterrupt: print("raise KeyboardInterrupt while training.") except AssertionError: # for BipedWalker BUG 2020-03-03 print( "AssertionError: OpenAI gym r.LengthSquared() > 0.0f ??? Please run again." ) return False train_time = recorder.print_and_save_npy(env_name, cwd) draw_plot_with_npy(cwd, train_time) return True
def process__workers(gpu_id, root_cwd, q_aggr, q_dist, args, **_kwargs): class_agent = args.class_agent env_name = args.env_name cwd = args.cwd net_dim = args.net_dim max_step = args.max_step # max_memo = args.max_memo max_epoch = args.max_epoch batch_size = args.batch_size * 1.5 gamma = args.gamma update_gap = args.update_gap reward_scale = args.reward_scale cwd = '{}/{}_{}'.format(root_cwd, cwd, gpu_id) os.makedirs(cwd, exist_ok=True) os.environ['CUDA_VISIBLE_DEVICES'] = str(gpu_id) random_seed = 42 + gpu_id np.random.seed(random_seed) torch.manual_seed(random_seed) torch.set_default_dtype(torch.float32) torch.set_num_threads(8) env = gym.make(env_name) is_solved = False class BufferArrayMP(BufferArray): def init_before_sample(self): q_aggr.put((self.memories, is_solved)) # self.now_len = self.max_len if self.is_full else self.next_idx def random_sample(self, _batch_size, device=None): batch_arrays = q_dist.get() '''convert array into torch.tensor''' tensors = [ torch.tensor(ary, device=device) for ary in batch_arrays ] return tensors '''init''' state_dim, action_dim, max_action, target_reward, is_discrete = get_env_info( env, is_print=True) agent = class_agent(env, state_dim, action_dim, net_dim) # training agent buffer = BufferArrayMP(max_step, state_dim, action_dim) # experiment replay buffer recorder = Recorder(agent, max_step, max_action, target_reward, env_name, **_kwargs) '''loop''' # with torch.no_grad(): # update replay buffer # # rewards, steps = agent.update_buffer( # # env, buffer, max_step, max_action, reward_scale, gamma) # rewards, steps = initial_exploration( # env, buffer, max_step, max_action, reward_scale, gamma, action_dim) # recorder.show_reward(rewards, steps, 0, 0) try: for epoch in range(max_epoch): '''update replay buffer by interact with environment''' with torch.no_grad(): # for saving the GPU buffer rewards, steps = agent.update_buffer(env, buffer, max_step, max_action, reward_scale, gamma) '''update network parameters by random sampling buffer for stochastic gradient descent''' loss_a, loss_c = agent.update_parameters(buffer, max_step, batch_size, update_gap) '''show/check the reward, save the max reward actor''' with torch.no_grad(): # for saving the GPU buffer '''NOTICE! Recorder saves the agent with max reward automatically. ''' recorder.show_reward(rewards, steps, loss_a, loss_c) is_solved = recorder.check_reward(cwd, loss_a, loss_c) if is_solved: break except KeyboardInterrupt: print("raise KeyboardInterrupt while training.") # except AssertionError: # for BipedWalker BUG 2020-03-03 # print("AssertionError: OpenAI gym r.LengthSquared() > 0.0f ??? Please run again.") # return False train_time = recorder.print_and_save_npy(env_name, cwd) # agent.save_or_load_model(cwd, is_save=True) # save max reward agent in Recorder # buffer.save_or_load_memo(cwd, is_save=True) draw_plot_with_npy(cwd, train_time) return True
def train_agent_ppo(class_agent, batch_size, repeat_times, gamma, reward_scale, cwd, env_name, max_step, net_dim, max_memo, max_epoch, **_kwargs): # 2020-0430 env = gym.make(env_name) state_dim, action_dim, max_action, target_reward, is_discrete = get_env_info( env, is_print=False) '''default hyper-parameters for on-policy RL algorithm PPO max_memo = 2 ** 11 repeat_times = 2 ** 3 batch_size = 2 ** 8 net_dim = 2 ** 7 gamma = 0.99 env_name = "LunarLanderContinuous-v2" env_name = "BipedalWalker-v3" ''' '''init''' agent = class_agent(state_dim, action_dim, net_dim) buffer = BufferListPPO( ) # on policy algorithm. Generalization Advantage Estimate. ICLR. 2016. state_norm = AutoNormalization( (state_dim, ), clip=6.0) # on policy algorithm can do normalization for state recorder = Recorder(agent, max_step, max_action, target_reward, env_name, state_norm=state_norm, **_kwargs) try: for epoch in range(max_epoch): # on policy algorithm refresh replay buffer for each parameters update buffer.storage = list() # update replay buffer by interact with environment with torch.no_grad(): # for saving the GPU buffer rewards, steps = agent.update_buffer_ppo( env, buffer, max_step, max_memo, max_action, reward_scale, gamma, state_norm) # update network parameters by random sampling buffer for gradient descent loss_a, loss_c = agent.update_parameters_ppo( buffer, batch_size, repeat_times) # show/check the reward, save the max reward actor with torch.no_grad(): # for saving the GPU buffer # NOTICE! Recorder saves the agent with max reward automatically. recorder.show_reward(rewards, steps, loss_a, loss_c) is_solved = recorder.check_reward(cwd, loss_a, loss_c) if is_solved: break except KeyboardInterrupt: print("raise KeyboardInterrupt while training.") train_time = recorder.print_and_save_npy(env_name, cwd) if is_solved: agent.save_or_load_model(cwd, is_save=True) draw_plot_with_npy(cwd, train_time)
def train_agent_on_policy0( class_agent, env_name, cwd, net_dim, max_step, max_memo, max_epoch, # env batch_size, gamma, **_kwargs): # 2020-0430 env = gym.make(env_name) state_dim, action_dim, max_action, target_reward = get_env_info(env) agent = class_agent(state_dim, action_dim, net_dim) agent.save_or_load_model(cwd, is_save=False) # memo_action_dim = action_dim if max_action else 1 # Discrete action space # memo = Memories(max_memo, memo_dim=1 + 1 + state_dim + memo_action_dim + state_dim) # memo.save_or_load_memo(cwd, is_save=False) state_norm = AutoNormalization((state_dim, ), clip=6.0) recorder = Recorder(agent, max_step, max_action, target_reward, env_name, state_norm=state_norm) # r_norm = RewardNorm(n_max=target_reward, n_min=recorder.reward_avg) try: for epoch in range(max_epoch): with torch.no_grad(): # just the GPU memory rewards, steps, memory = agent.update_buffer_ppo( env, max_step, max_memo, max_action, state_norm) l_total, l_value = agent.update_parameters_ppo(memory, batch_size, gamma, ep_ratio=1 - epoch / max_epoch) if np.isnan(l_total) or np.isnan(l_value): print( "ValueError: loss value should not be 'nan'. Please run again." ) return False with torch.no_grad(): # for saving the GPU memory recorder.show_reward(epoch, rewards, steps, l_value, l_total) is_solved = recorder.check_reward(cwd, l_value, l_total) if is_solved: break except KeyboardInterrupt: print("raise KeyboardInterrupt while training.") except AssertionError: # for BipedWalker BUG 2020-03-03 print( "AssertionError: OpenAI gym r.LengthSquared() > 0.0f ??? Please run again." ) return False train_time = recorder.print_and_save_npy(env_name, cwd) # agent.save_or_load_model(cwd, is_save=True) # save max reward agent in Recorder # memo.save_or_load_memo(cwd, is_save=True) draw_plot_with_npy(cwd, train_time) return True