def generate_expert_trajectorys(args): env = launch_env() env = ResizeWrapper(env) env = NormalizeWrapper(env) env = ImgWrapper(env) env = ActionWrapper(env) env = DtRewardWrapper(env) print("Initialized Wrappers") expert = PurePursuitExpert(env=env) for episode in range(0, args.episodes): print("Starting episode", episode) observations = [] actions = [] for steps in range(0, args.steps): # use our 'expert' to predict the next action. action = expert.predict(None) observation, reward, done, info = env.step(action) observations.append(observation) actions.append(action) # env.render() env.reset() torch.save(actions, '{}/data_a_{}.pt'.format(args.data_directory, episode)) torch.save(observations, '{}/data_o_{}.pt'.format(args.data_directory, episode)) env.close()
def _enjoy(): # Launch the env with our helper function env = launch_env() print("Initialized environment") # Wrappers env = ResizeWrapper(env) env = NormalizeWrapper(env) env = ImgWrapper(env) # to make the images from 160x120x3 into 3x160x120 env = ActionWrapper(env) env = DtRewardWrapper(env) print("Initialized Wrappers") state_dim = env.observation_space.shape action_dim = env.action_space.shape[0] max_action = float(env.action_space.high[0]) # Initialize policy policy = DDPG(state_dim, action_dim, max_action, net_type="cnn") policy.load(filename='ddpg', directory='reinforcement/pytorch/models/') obs = env.reset() done = False while True: while not done: action = policy.predict(np.array(obs)) # Perform action obs, reward, done, _ = env.step(action) env.render() done = False obs = env.reset()
def _enjoy(args): from learning.utils.env import launch_env from learning.utils.wrappers import NormalizeWrapper, ImgWrapper, \ DtRewardWrapper, ActionWrapper, ResizeWrapper from learning.utils.teacher import PurePursuitExpert # model = Model(action_dim=2, max_action=1.) model = Generator(action_dim=2) try: # state_dict = torch.load('models/imitate.pt', map_location=device) state_dict = torch.load('models/G{}.pt'.format(args.enjoy_tag), map_location=device) model.load_state_dict(state_dict) except: print("Unexpected error:", sys.exc_info()[0]) print('failed to load model') exit() model.eval().to(device) env = launch_env() env = ResizeWrapper(env) env = NormalizeWrapper(env) env = ImgWrapper(env) env = ActionWrapper(env) env = DtRewardWrapper(env) obs = env.reset() # max_count = 0 while True: obs = torch.from_numpy(obs).float().to(device).unsqueeze(0) action = model(obs) action = action.squeeze().data.cpu().numpy() print("\nAction taken::", action, "\n") obs, reward, done, info = env.step(action) env.render() # if max_count > 50: # max_count = 0 # obs = env.reset() if done: if reward < 0: print('*** FAILED ***') time.sleep(0.7) # max_count += 1 obs = env.reset() env.render()
def _train(args): if not os.path.exists("./results"): os.makedirs("./results") if not os.path.exists(args.model_dir): os.makedirs(args.model_dir) # Launch the env with our helper function env = launch_env() print("Initialized environment") # Wrappers env = ResizeWrapper(env) env = NormalizeWrapper(env) env = ImgWrapper(env) # to make the images from 160x120x3 into 3x160x120 env = ActionWrapper(env) env = DtRewardWrapper(env) print("Initialized Wrappers") # Set seeds seed(args.seed) state_dim = env.observation_space.shape action_dim = env.action_space.shape[0] max_action = float(env.action_space.high[0]) # Initialize policy policy = DDPG(state_dim, action_dim, max_action, net_type="cnn") replay_buffer = ReplayBuffer(args.replay_buffer_max_size) print("Initialized DDPG") # Evaluate untrained policy evaluations = [evaluate_policy(env, policy)] total_timesteps = 0 timesteps_since_eval = 0 episode_num = 0 done = True episode_reward = None env_counter = 0 reward = 0 print("Starting training") while total_timesteps < args.max_timesteps: print("timestep: {} | reward: {}".format(total_timesteps, reward)) if done: if total_timesteps != 0: print( ("Total T: %d Episode Num: %d Episode T: %d Reward: %f") % (total_timesteps, episode_num, episode_timesteps, episode_reward)) policy.train(replay_buffer, episode_timesteps, args.batch_size, args.discount, args.tau) # Evaluate episode if timesteps_since_eval >= args.eval_freq: timesteps_since_eval %= args.eval_freq evaluations.append(evaluate_policy(env, policy)) print("rewards at time {}: {}".format( total_timesteps, evaluations[-1])) if args.save_models: policy.save(file_name, directory=args.model_dir) np.savez("./results/{}.npz".format(file_name), evaluations) # Reset environment env_counter += 1 obs = env.reset() done = False episode_reward = 0 episode_timesteps = 0 episode_num += 1 # Select action randomly or according to policy if total_timesteps < args.start_timesteps: action = env.action_space.sample() else: action = policy.predict(np.array(obs)) if args.expl_noise != 0: action = (action + np.random.normal( 0, args.expl_noise, size=env.action_space.shape[0])).clip( env.action_space.low, env.action_space.high) # Perform action new_obs, reward, done, _ = env.step(action) if episode_timesteps >= args.env_timesteps: done = True done_bool = 0 if episode_timesteps + 1 == args.env_timesteps else float( done) episode_reward += reward # Store data in replay buffer replay_buffer.add(obs, new_obs, action, reward, done_bool) obs = new_obs episode_timesteps += 1 total_timesteps += 1 timesteps_since_eval += 1 print("Training done, about to save..") policy.save(filename='ddpg', directory=args.model_dir) print("Finished saving..should return now!")
def _main(args): ############## Hyperparameters ############## # env_name = "BipedalWalker-v2" env_name = 'Duckietown-loop_empty-v0' render = False lr = 0.0003 # parameters for Adam optimizer betas = (0.9, 0.999) random_seed = None print(args) ############################################# # creating environment env = launch_env() # Wrappers env = ResizeWrapper(env) env = NormalizeWrapper(env) env = ImgWrapper(env) # to make the images from 160x120x3 into 3x160x120 env = ActionWrapper(env) env = DtRewardWrapper(env) print("Initialized Wrappers") # state_dim = env.observation_space.shape[0] state_dim = env.observation_space.shape state_dim = functools.reduce(operator.mul, state_dim, 1) action_dim = env.action_space.shape[0] max_action = float(env.action_space.high[0]) if random_seed: print("Random Seed: {}".format(random_seed)) torch.manual_seed(random_seed) env.seed(random_seed) np.random.seed(random_seed) memory = Memory() ppo = PPO(state_dim, action_dim, args.action_std, lr, betas, args.gamma, args.K_epochs, args.eps_clip, max_action, args.batch_size) print(lr, betas) # logging variables running_reward = 0 avg_length = 0 time_step = 0 episode_reward = 0 # stats = pd.DataFrame(columns = ["Episode", "Length", "Reward"]) stats = [] with open("PPO_stats.csv", 'w') as statsfile: statsfile.write("Epoch, Timesteps, Reward\n") # training loop for i_episode in range(1, args.max_episodes + 1): state = env.reset() for t in range(args.max_timesteps): time_step += 1 # Running policy_old: action = ppo.select_action(state, memory) state, reward, done, _ = env.step(action) # Saving reward and is_terminals: memory.rewards.append(reward) memory.is_terminals.append(done) # update if its time if time_step % args.update_timestep == 0: ppo.update(memory) memory.clear_memory() time_step = 0 episode_reward += reward if render: env.render() if done: break avg_length += t # stats = stats.append({"Episode" : i_episode, "Length" : t, "Reward" : episode_reward}, ignore_index=True) stats.append((i_episode, t, episode_reward)) running_reward += episode_reward episode_reward = 0 if i_episode % args.store_interval == 0: torch.save(ppo.policy.state_dict(), './PPO_continuous_{}.pth'.format(env_name)) # stats.to_csv("PPO_stats.csv", index=False) #This line does not work on Google Colab! with open("PPO_stats.csv", 'a') as statsfile: for eps, ts, rwd in stats: statsfile.write("%d, %d, %f\n" % (eps, ts, rwd)) stats = [] # logging if i_episode % args.log_interval == 0: avg_length = int(avg_length / args.log_interval) running_reward = int((running_reward / args.log_interval)) print('Episode {} \t Avg length: {} \t Avg reward: {}'.format( i_episode, avg_length, running_reward)) running_reward = 0 avg_length = 0
def test(): ############## Hyperparameters ############## # env_name = "BipedalWalker-v2" env_name = 'Duckietown-loop_empty-v0' # env = gym.make(env_name) # creating environment env = launch_env() # Wrappers env = ResizeWrapper(env) env = NormalizeWrapper(env) env = ImgWrapper(env) # to make the images from 160x120x3 into 3x160x120 env = ActionWrapper(env) env = DtRewardWrapper(env) print("Initialized Wrappers") # state_dim = env.observation_space.shape[0] action_dim = env.action_space.shape[0] state_dim = env.observation_space.shape state_dim = functools.reduce(operator.mul, state_dim, 1) max_action = float(env.action_space.high[0]) n_episodes = 3 # num of episodes to run max_timesteps = 500 # max timesteps in one episode render = True # render the environment save_gif = False # png images are saved in gif folder # filename and directory to load model from filename = "PPO_continuous_" + env_name + ".pth" directory = "./preTrained/" action_std = 0.05 # constant std for action distribution (Multivariate Normal) K_epochs = 80 # update policy for K epochs eps_clip = 0.2 # clip parameter for PPO gamma = 0.99 # discount factor lr = 0.0003 # parameters for Adam optimizer betas = (0.9, 0.999) ############################################# memory = Memory() ppo = PPO(state_dim, action_dim, action_std, lr, betas, gamma, K_epochs, eps_clip, max_action, 32) ppo.policy_old.load_state_dict( torch.load(directory + filename, map_location=torch.device('cpu'))) for ep in range(1, n_episodes + 1): ep_reward = 0 state = env.reset() for t in range(max_timesteps): action = ppo.select_action(state, memory) state, reward, done, _ = env.step(action) ep_reward += reward if render: env.render() if save_gif: img = env.render(mode='rgb_array') img = Image.fromarray(img) img.save('./gif/{}.jpg'.format(t)) if done: break print('Episode: {}\tReward: {}'.format(ep, int(ep_reward))) ep_reward = 0 env.close()
def _train(args): print("Running Expert for {} Episodes of {} Steps".format( args.episodes, args.steps)) print("Training Learning for {} Epochs with Batch Size of {}".format( args.epochs, args.batch_size)) env = launch_env() env = ResizeWrapper(env) env = NormalizeWrapper(env) env = ActionWrapper(env) env = DtRewardWrapper(env) print("Initialized Wrappers") observation_shape = (None, ) + env.observation_space.shape action_shape = (None, ) + env.action_space.shape # Create an imperfect demonstrator expert = PurePursuitExpert(env=env) observations = [] actions = [] # let's collect our samples for episode in range(0, args.episodes): print("Starting episode", episode) for steps in range(0, args.steps): # use our 'expert' to predict the next action. action = expert.predict(None) observation, reward, done, info = env.step(action) observations.append(observation) actions.append(action) env.reset() env.close() actions = np.array(actions) observations = np.array(observations) model = TensorflowModel( observation_shape=observation_shape, # from the logs we've got action_shape=action_shape, # same graph_location=args. model_directory, # where do we want to store our trained models seed=args. seed # to seed all random operations in the model (e.g., dropout) ) for i in range(args.epochs): # we defined the batch size, this can be adjusted according to your computing resources loss = None for batch in range(0, len(observations), args.batch_size): print("Training batch", batch) loss = model.train(observations=observations[batch:batch + args.batch_size], actions=actions[batch:batch + args.batch_size]) # every 10 epochs, we store the model we have if i % 10 == 0: model.commit() print("Training complete!")
def _train(args): env = launch_env() env = ResizeWrapper(env) env = NormalizeWrapper(env) env = ImgWrapper(env) env = ActionWrapper(env) env = DtRewardWrapper(env) print("Initialized Wrappers") observation_shape = (None, ) + env.observation_space.shape action_shape = (None, ) + env.action_space.shape # Create an imperfect demonstrator expert = PurePursuitExpert(env=env) observations = [] actions = [] # let's collect our samples for episode in range(0, args.episodes): print("Starting episode", episode) for steps in range(0, args.steps): # use our 'expert' to predict the next action. action = expert.predict(None) observation, reward, done, info = env.step(action) observations.append(observation) actions.append(action) env.reset() env.close() actions = np.array(actions) observations = np.array(observations) # model = Model(action_dim=2, max_action=1.) model = Generator(action_dim=2) # state_dict = torch.load('models/G_imitate_2.pt', map_location=device) # model.load_state_dict(state_dict) model.train().to(device) # weight_decay is L2 regularization, helps avoid overfitting optimizer = optim.SGD(model.parameters(), lr=0.0004, weight_decay=1e-3) avg_loss = 0 for epoch in range(args.epochs): optimizer.zero_grad() batch_indices = np.random.randint(0, observations.shape[0], (args.batch_size)) obs_batch = torch.from_numpy( observations[batch_indices]).float().to(device) act_batch = torch.from_numpy(actions[batch_indices]).float().to(device) model_actions = model(obs_batch) loss = (model_actions - act_batch).norm(2).mean() loss.backward() optimizer.step() loss = loss.data.item() avg_loss = avg_loss * 0.995 + loss * 0.005 print('epoch %d, loss=%.3f' % (epoch, avg_loss)) # Periodically save the trained model if epoch % 200 == 0: torch.save(model.state_dict(), '{}/G_imitate.pt'.format(args.model_directory))