def test(rank, args, ns, best_result): env = RunEnv(False) env.seed(args.seed + rank) nb_states = env.observation_space.shape[0] nb_actions = env.action_space.shape[0] if args.use_more_states: agent = DDPG(nb_states * args.num_states, nb_actions, args) else: agent = DDPG(nb_states, nb_actions, args) if args.load_weights: agent.load_weights("weights") agent.is_training = False agent.eval() done = True policy = lambda x: agent.select_action(x, decay_epsilon=False) last_reward = -10 episode = 0 observation = None observations = None episode_reward = 0. step = 0 best_episode_reward = -10 while True: # reset at the start of episode if observation is None: observation = deepcopy(env.reset()) agent.reset(observation) if args.use_more_states: observations = deque( list(observation for i in range(2**args.num_states)), 2**args.num_states) # observations = deque(list(observation for i in range(args.num_states)), args.num_states) if best_result.value > best_episode_reward and step > args.warmup: best_model = deepcopy(ns.best_model) test_agent = deepcopy(agent) test_agent.load_state_dict(best_model.state_dict()) if test_new_state_dict(test_agent, episode_reward, env, use_more_states=args.use_more_states, num_states=args.num_states): agent = test_agent agent.best_reward = best_model.best_reward prRed("updated test agent from ns {:.3f}".format( best_model.best_reward)) last_reward = best_result.value observation = None episode_steps = 0 episode_reward = 0. # start episode if observation is None: observation = deepcopy(env.reset()) agent.reset(observation) if args.use_more_states: observations = deque( list(observation for i in range(2**args.num_states)), 2**args.num_states) # observations = deque(list(observation for i in range(args.num_states)), args.num_states) done = False while not done: if args.use_more_states: cur_observations = list() for i in range(args.num_states): cur_observations.append(list(observations)[2**i - 1]) action = policy( np.concatenate(list(cur_observations)).ravel().tolist()) else: action = policy(observation) observation, reward, done, info = env.step(action) if args.use_more_states and observation: observations.appendleft(observation) episode_reward += reward episode_steps += 1 step += 1 if episode % 50 == 0 and episode != 0: print("saving models") os.makedirs("weights", exist_ok=True) agent.save_model("weights") episode += 1 observation = None observations = None current_best_result = best_result.value best_episode_reward = max(episode_reward, best_episode_reward) best_result.value = max(episode_reward, current_best_result - 0.05) print( '#Ep{}: episode_reward:{:.3f} episode_steps:{} br: {:.3f} -> {:.3f}' .format(episode, episode_reward, episode_steps, current_best_result, best_result.value))
def train(rank, args, ns, best_result, actor_optim, critic_optim, shared_model, debug=True): torch.manual_seed(args.seed + rank) env = RunEnv(False) env.seed(args.seed + rank) nb_states = env.observation_space.shape[0] nb_actions = env.action_space.shape[0] if args.use_more_states: agent = DDPG(nb_states * args.num_states, nb_actions, args) else: agent = DDPG(nb_states, nb_actions, args) if args.shared: agent.add_optim(actor_optim, critic_optim) agent.train() if args.load_weights: agent.load_weights("weights") agent.is_training = True step = episode = episode_steps = 0 observation = None done = True episode_reward = 0. observations = None last_reward = -10 while True: if args.shared: agent.load_state_dict(shared_model.state_dict()) if observation is None: observation = deepcopy(env.reset()) agent.reset(observation) if args.use_more_states: observations = deque( list(observation for i in range(2**args.num_states)), 2**args.num_states) # observations = deque(list(observation for i in range(args.num_states)), args.num_states) if step <= args.warmup: action = agent.random_action() elif args.use_more_states: cur_observations = list() for i in range(args.num_states): cur_observations.append(list(observations)[2**i - 1]) action = agent.select_action( np.concatenate(list(cur_observations)).ravel().tolist()) else: action = agent.select_action(observation) observation2, reward, done, info = env.step(action) observation = deepcopy(observation2) if observation and args.use_more_states: observations.appendleft(observation) if args.use_more_states: cur_observations = list() for i in range(args.num_states): cur_observations.append(list(observations)[2**i - 1]) agent.observe( reward, np.concatenate(list(cur_observations)).ravel().tolist(), done) else: agent.observe(reward, observation, done) if step > args.warmup: for i in range(5): agent.update_policy(shared_model, args) step += 1 episode_steps += 1 episode_reward += reward if done: if args.use_more_states: cur_observations = list() for i in range(args.num_states): cur_observations.append(list(observations)[2**i - 1]) agent.memory.append( np.concatenate(list(cur_observations)).ravel().tolist(), agent.select_action( np.concatenate( list(cur_observations)).ravel().tolist()), 0., False) else: agent.memory.append(observation, agent.select_action(observation), 0., False) if step > args.warmup and best_result.value < episode_reward and episode_reward > last_reward: best_model = ns.best_model best_model.load_state_dict(agent.state_dict()) if debug: prLightPurple( "best reward: {:.3f} current reward: {:.3f} updated best model from agent {}" .format(best_model.best_reward, episode_reward, rank)) best_model.best_reward = episode_reward agent.best_reward = episode_reward ns.best_model = best_model best_result.value = episode_reward last_reward = best_result.value elif step > args.warmup and episode % 10 == 0 and episode > 0 and args.update_train_agents > 0 \ and best_result.value > episode_reward and best_result.value > agent.best_reward: best_model = ns.best_model test_agent = deepcopy(agent) test_agent.load_state_dict(best_model.state_dict()) if test_new_state_dict(test_agent, episode_reward, env, args.update_train_agents, use_more_states=args.use_more_states, num_states=args.num_states): agent = test_agent agent.best_reward = best_model.best_reward if debug: prGreen("best result {:.3f} updated agent {}".format( best_model.best_reward, rank)) last_reward = best_model.best_reward observation = None observations = None if debug: prCyan('agent_{:02d} ep:{} ep_steps:{} reward:{:.3f} '.format( rank, episode, episode_steps, episode_reward)) episode_steps = 0 episode += 1 episode_reward = 0.