def _train(args): print("Running Expert for {} Episodes of {} Steps".format(args.episodes, args.steps)) print("Training Learning for {} Epochs with Batch Size of {}".format(args.epochs, args.batch_size)) env = launch_env() env = ResizeWrapper(env) env = NormalizeWrapper(env) env = ActionWrapper(env) env = DtRewardWrapper(env) print("Initialized Wrappers") observation_shape = (None, ) + env.observation_space.shape action_shape = (None, ) + env.action_space.shape # Create an imperfect demonstrator expert = PurePursuitExpert(env=env) observations = [] actions = [] # let's collect our samples for episode in range(0, args.episodes): print("Starting episode", episode) for steps in range(0, args.steps): # use our 'expert' to predict the next action. action = expert.predict(None) observation, reward, done, info = env.step(action) observations.append(observation) actions.append(action) env.reset() env.close() actions = np.array(actions) observations = np.array(observations) model = TensorflowModel( observation_shape=observation_shape, # from the logs we've got action_shape=action_shape, # same graph_location=args.model_directory, # where do we want to store our trained models seed=args.seed # to seed all random operations in the model (e.g., dropout) ) for i in range(args.epochs): # we defined the batch size, this can be adjusted according to your computing resources loss = None for batch in range(0, len(observations), args.batch_size): print("Training batch", batch) loss = model.train( observations=observations[batch:batch + args.batch_size], actions=actions[batch:batch + args.batch_size] ) # every 10 epochs, we store the model we have if i % 10 == 0: model.commit() print("Training complete!")
def initenv2(): env = launch_env2() env = ResizeWrapper(env) env = NormalizeWrapper(env) env = ImgWrapper(env) env = ActionWrapper(env) env = DtRewardWrapper(env) return env
def make_env(): # Launch the env with our helper function env = launch_env() print("Initialized environment") # Wrappers env = ResizeWrapper(env) env = NormalizeWrapper(env) env = ImgWrapper(env) # to make the images from 160x120x3 into 3x160x120 env = ActionWrapper(env) env = DtRewardWrapper(env) print("Initialized Wrappers") return env
def initWindowModel(): model = WindowModel(action_dim=2, max_action=1.) try: state_dict = torch.load('./models/windowimitate.pt') model.load_state_dict(state_dict) except: print('failed to load model') exit() model.eval().to(device) env = launch_env1() env = ResizeWrapper(env) env = NormalizeWrapper(env) env = ImgWrapper(env) env = ActionWrapper(env) env = DtRewardWrapper(env) env.reset() env.render() return env, model
def extract(): env = launch_env() print("Initialized environment") # Wrappers env = ResizeWrapper(env) env = NormalizeWrapper(env) env = ImgWrapper(env) # to make the images from 160x120x3 into 3x160x120 env = ActionWrapper(env) env = DtRewardWrapper(env) print("Initialized Wrappers") state_dim = env.observation_space.shape action_dim = env.action_space.shape[0] max_action = float(env.action_space.high[0]) total_timesteps = 0 timesteps_since_eval = 0 episode_num = 0 done = True episode_reward = None env_counter = 0 reward = 0 episode_timesteps = 0 obs = env.reset() print("Starting training") while total_timesteps < 2: action = env.action_space.sample() # Perform action new_obs, reward, done, _ = env.step(action) obs = new_obs print(obs) print(action) total_timesteps += 1
def _enjoy(): if not os.path.exists("./results"): os.makedirs("./results") if not os.path.exists(args.model_dir): os.makedirs(args.model_dir) # Launch the env with our helper function env = launch_env() print("Initialized environment") # Wrappers env = ResizeWrapper(env) env = NormalizeWrapper(env) env = ImgWrapper(env) # to make the images from 160x120x3 into 3x160x120 env = ActionWrapper(env) env = DtRewardWrapper(env) print("Initialized Wrappers") state_dim = env.observation_space.shape action_dim = env.action_space.shape[0] max_action = float(env.action_space.high[0]) # Initialize policy policy = DDPG(state_dim, action_dim, max_action, net_type="cnn") policy.load(filename='ddpg', directory='reinforcement/pytorch/models/') obs = env.reset() while True: while not done: action = policy.predict(np.array(obs)) # Perform action obs, reward, done, _ = env.step(action) env.render() done = False obs = env.reset()
def _train(args): if not os.path.exists("./results"): os.makedirs("./results") if not os.path.exists(args.model_dir): os.makedirs(args.model_dir) # Launch the env with our helper function env = launch_env() print("Initialized environment") # Wrappers env = ResizeWrapper(env) env = NormalizeWrapper(env) env = ImgWrapper(env) # to make the images from 160x120x3 into 3x160x120 env = ActionWrapper(env) env = DtRewardWrapper(env) print("Initialized Wrappers") # Set seeds seed(args.seed) state_dim = env.observation_space.shape action_dim = env.action_space.shape[0] max_action = float(env.action_space.high[0]) # Initialize policy policy = DDPG(state_dim, action_dim, max_action, net_type="cnn") replay_buffer = ReplayBuffer(args.replay_buffer_max_size) print("Initialized DDPG") # Evaluate untrained policy evaluations = [evaluate_policy(env, policy)] total_timesteps = 0 timesteps_since_eval = 0 episode_num = 0 done = True episode_reward = None env_counter = 0 reward = 0 episode_timesteps = 0 print("Starting training") while total_timesteps < args.max_timesteps: print("timestep: {} | reward: {}".format(total_timesteps, reward)) if done: if total_timesteps != 0: print( ("Total T: %d Episode Num: %d Episode T: %d Reward: %f") % (total_timesteps, episode_num, episode_timesteps, episode_reward)) policy.train(replay_buffer, episode_timesteps, args.batch_size, args.discount, args.tau) # Evaluate episode if timesteps_since_eval >= args.eval_freq: timesteps_since_eval %= args.eval_freq evaluations.append(evaluate_policy(env, policy)) print("rewards at time {}: {}".format( total_timesteps, evaluations[-1])) if args.save_models: policy.save(filename='{}_{}'.format( 'ddpg', total_timesteps), directory=args.model_dir) np.savez("./results/rewards.npz", evaluations) # Reset environment env_counter += 1 obs = env.reset() done = False episode_reward = 0 episode_timesteps = 0 episode_num += 1 # Select action randomly or according to policy if total_timesteps < args.start_timesteps: action = env.action_space.sample() else: action = policy.predict(np.array(obs)) if args.expl_noise != 0: action = (action + np.random.normal( 0, args.expl_noise, size=env.action_space.shape[0])).clip( env.action_space.low, env.action_space.high) # Perform action new_obs, reward, done, _ = env.step(action) if episode_timesteps >= args.env_timesteps: done = True done_bool = 0 if episode_timesteps + 1 == args.env_timesteps else float( done) episode_reward += reward # Store data in replay buffer replay_buffer.add(obs, new_obs, action, reward, done_bool) obs = new_obs episode_timesteps += 1 total_timesteps += 1 timesteps_since_eval += 1 print("Training done, about to save..") policy.save(filename='ddpg', directory=args.model_dir) print("Finished saving..should return now!")
def _train(args): env = launch_env() env = ResizeWrapper(env) env = NormalizeWrapper(env) env = ImgWrapper(env) env = DtRewardWrapper(env) env = MetricsWrapper(env) env = ActionWrapper(env) print("Initialized Wrappers") observation_shape = (None, ) + env.observation_space.shape action_shape = (None, ) + env.action_space.shape # Create an imperfect demonstrator expert = PurePursuitExpert(env=env) observations = [] actions = [] # let's collect our samples for episode in range(0, args.episodes): print("Starting episode", episode) for steps in range(0, args.steps): # use our 'expert' to predict the next action. action = expert.predict(None) observation, reward, done, info = env.step(action) observations.append(observation) actions.append(action) env.reset() env.close() actions = np.array(actions) observations = np.array(observations) model = Model(action_dim=2, max_action=1.) model.train().to(device) # weight_decay is L2 regularization, helps avoid overfitting optimizer = optim.SGD(model.parameters(), lr=0.0004, weight_decay=1e-3) avg_loss = 0 for epoch in range(args.epochs): optimizer.zero_grad() batch_indices = np.random.randint(0, observations.shape[0], (args.batch_size)) obs_batch = torch.from_numpy( observations[batch_indices]).float().to(device) act_batch = torch.from_numpy(actions[batch_indices]).float().to(device) model_actions = model(obs_batch) loss = (model_actions - act_batch).norm(2).mean() loss.backward() optimizer.step() loss = loss.data[0] avg_loss = avg_loss * 0.995 + loss * 0.005 print('epoch %d, loss=%.3f' % (epoch, avg_loss)) # Periodically save the trained model if epoch % 200 == 0: torch.save(model.state_dict(), 'imitation/pytorch/models/imitate.pt')
def _enjoy(): model = Model(action_dim=2, max_action=1.) try: state_dict = torch.load('trained_models/imitate.pt', map_location=device) model.load_state_dict(state_dict) except: print('failed to load model') exit() model.eval().to(device) env = launch_env() env = ResizeWrapper(env) env = NormalizeWrapper(env) env = ImgWrapper(env) env = ActionWrapper(env) env = DtRewardWrapper(env) obs = env.reset() while True: obs = torch.from_numpy(obs).float().to(device).unsqueeze(0) action = model(obs) action = action.squeeze().data.cpu().numpy() obs, reward, done, info = env.step(action) env.render() if done: if reward < 0: print('*** FAILED ***') time.sleep(0.7) obs = env.reset() env.render()
def _enjoyWindow(): model = WindowModel(action_dim=2, max_action=1.) try: state_dict = torch.load('./models/windowimitate.pt') model.load_state_dict(state_dict) except: print('failed to load model') exit() model.eval().to(device) env = launch_env1() env = ResizeWrapper(env) env = NormalizeWrapper(env) env = ImgWrapper(env) env = ActionWrapper(env) env = DtRewardWrapper(env) obs = env.reset() obsWindow = np.zeros((12, 160, 120)) while True: obsWindow[:9, :, :] = obsWindow[3:, :, :] obsWindow[9:12, :, :] = obs obs = torch.from_numpy(obsWindow).float().to(device).unsqueeze(0) action = model(obs) action = action.squeeze().data.cpu().numpy() obs, reward, done, info = env.step(action) env.render() if done: if reward < 0: print('*** FAILED ***') time.sleep(0.7) obs = env.reset() env.render()
def _dagger(): model = Model(action_dim=2, max_action=1.) try: state_dict = torch.load('./models/imitate.pt') model.load_state_dict(state_dict) except: print('failed to load model') exit() model.eval().to(device) env = launch_env1() # Register a keyboard handler env = ResizeWrapper(env) env = NormalizeWrapper(env) env = ImgWrapper(env) env = ActionWrapper(env) env = DtRewardWrapper(env) obs = env.reset() env.render() key_handler = key.KeyStateHandler() env.unwrapped.window.push_handlers(key_handler) print(env.map_name) raise Exception("asdfsadf") obsHistory = [] actionHistory = [] while True: obs = torch.from_numpy(obs).float().to(device).unsqueeze(0) action = model(obs) action = action.squeeze().data.cpu().numpy() obs, reward, done, info = env.step(action) print(key_handler) daggerAction = np.array([0.0, 0.0]) if key_handler[key.UP]: print("as===as=df=sad=f=asdf=sad=fs=adf") daggerAction = np.array([1.00, 0.0]) #action = np.array([0.44, 0.0]) if key_handler[key.DOWN]: print("as===as=df=sad=f=asdf=sad=fs=adf") daggerAction = np.array([-1.00, 0]) #action = np.array([-0.44, 0]) if key_handler[key.LEFT]: print("as===as=df=sad=f=asdf=sad=fs=adf") daggerAction = np.array([0.35, +1]) if key_handler[key.RIGHT]: print("as===as=df=sad=f=asdf=sad=fs=adf") daggerAction = np.array([0.35, -1]) if key_handler[key.SPACE]: obsHistoryArray = np.array(obsHistory) actionHistoryArray = np.array(actionHistory) np.save('./dagger/obs_{}.npy'.format(len(count)), obsHistoryArray) np.save('./dagger/actions_{}.npy'.format(len(count)), actionHistoryArray) print(daggerAction) obsHistory.append(obs) actionHistory.append(daggerAction) env.render() if done: if reward < 0: print('*** FAILED ***') time.sleep(0.7) obs = env.reset() env.render()
def _train(args): env = launch_env1() env1 = ResizeWrapper(env) env2 = NormalizeWrapper(env) env3 = ImgWrapper(env) env = ActionWrapper(env) env = DtRewardWrapper(env) print("Initialized Wrappers") def transformObs(obs): obs = env1.observation(obs) obs = env2.observation(obs) obs = env3.observation(obs) return obs actions = None rawObs = None for map in MAP_NAMES: if map == "loop_obstacles": episodes = 3 else: episodes = 2 print(map) for episode in range(episodes): actionFile = "actions_{}.npy".format(episode) action = np.load(TRAINING_DATA_PATH.format(map, actionFile)) print(action.shape) observationFile = "obs_{}.npy".format(episode) observation = np.load(TRAINING_DATA_PATH.format(map, observationFile)) if actions is None: actions = action rawObs = observation else: actions = np.concatenate((actions, action), axis=0) rawObs = np.concatenate((rawObs, observation), axis=0) print(actions.shape) print(actions.shape) print("---") observations = np.zeros((rawObs.shape[0], 3, 160, 120)) for i, obs in enumerate(rawObs): observations[i] = transformObs(obs) ''' # Create an imperfect demonstrator expert = PurePursuitExpert(env=env) observations = [] actions = [] # let's collect our samples for episode in range(0, 2): #for episode in range(0, args.episodes): print("Starting episode", episode) #for steps in range(0, args.steps): for steps in range(0, 4): # use our 'expert' to predict the next action. action = expert.predict(None) observation, reward, done, info = env.step(action) observations.append(observation) actions.append(action) env.reset() actions = np.array(actions) observations = np.array(observations) print(observations.shape) ''' env.close() #raise Exception("Done with testing") model = Model(action_dim=2, max_action=1.) model.train().to(device) # weight_decay is L2 regularization, helps avoid overfitting optimizer = optim.SGD( model.parameters(), lr=0.0004, weight_decay=1e-3 ) loss_list = [] avg_loss = 0 for epoch in range(args.epochs): optimizer.zero_grad() batch_indices = np.random.randint(0, observations.shape[0], (args.batch_size)) obs_batch = torch.from_numpy(observations[batch_indices]).float().to(device) act_batch = torch.from_numpy(actions[batch_indices]).long().to(device) model_actions = model(obs_batch) loss = (model_actions - act_batch).norm(2).mean() loss.backward() optimizer.step() #loss = loss.data[0] loss = loss.item() avg_loss = avg_loss * 0.995 + loss * 0.005 print('epoch %d, loss=%.3f' % (epoch, loss)) loss_list.append(loss) # Periodically save the trained model if epoch % 50 == 0: print("Saving...") torch.save(model.state_dict(), 'imitation/pytorch/models/imitate.pt') save_loss(loss_list, 'imitation/pytorch/loss.npy') print("Saving...") torch.save(model.state_dict(), 'imitation/pytorch/models/imitate.pt')
def _enjoy(args): # Launch the env with our helper function env = launch_env() print("Initialized environment") # Wrappers env = ResizeWrapper(env) env = GrayscaleWrapper(env) env = NormalizeWrapper(env) env = FrameStack(env, 4) env = DtRewardWrapper(env) env = ActionWrapper(env) print("Initialized Wrappers") state_dim = env.observation_space.shape action_dim = env.action_space.shape[0] max_action = float(env.action_space.high[0]) # Initialize policy # policy = TD3(state_dim, action_dim, max_action, net_type="cnn") # policy.load(filename=args.policy, directory='reinforcement/pytorch/models/') policy = policies[args.policy](state_dim, action_dim, max_action) policy.load("reinforcement/pytorch/models/", args.policy) obs = env.reset() done = False while True: while not done: action = policy.predict(np.array(obs)) # Perform action obs, reward, done, _ = env.step(action) env.render() done = False obs = env.reset()
def _enjoy(): # Launch the env with our helper function env = launch_env() print("Initialized environment") # Wrappers env = ResizeWrapper(env) env = NormalizeWrapper(env) env = ImgWrapper( env, canny=False) # to make the images from 160x120x3 into 3x160x120 env = ActionWrapper(env) env = DtRewardWrapper(env) print("Initialized Wrappers") state_dim = env.observation_space.shape action_dim = env.action_space.shape[0] max_action = float(env.action_space.high[0]) # Initialize policy policy = DDPG(state_dim, action_dim, max_action, net_type="cnn") policy.load(filename='ddpg', directory='reinforcement/pytorch/models/') obs = env.reset() done = False i = 0 while True: while not done: action = policy.predict(np.array(obs)) # Perform action obs, reward, done, _ = env.step(action) print(action, reward) env.render() time.sleep(0.1) print("\nRESET\n") done = False obs = env.reset() np.save('env{}.npz'.format(i), obs) i += 1
def _train(args): if not os.path.exists("./results"): os.makedirs("./results") if not os.path.exists(args.model_dir): os.makedirs(args.model_dir) # Launch the env with our helper function env = launch_env() print("Initialized environment") # Wrappers env = ResizeWrapper(env) env = GrayscaleWrapper(env) env = NormalizeWrapper(env) env = FrameStack(env, 4) env = DtRewardWrapper(env) env = ActionWrapper(env) print("Initialized Wrappers") # Set seeds seed(args.seed) state_dim = env.observation_space.shape action_dim = env.action_space.shape[0] max_action = float(env.action_space.high[0]) # Init training data total_timesteps = 0 timesteps_since_eval = 0 episode_num = 0 episode_reward = 0 env_counter = 0 reward = 0 episode_timesteps = 0 avg_episodes = 100 # Keep track of the best reward over time best_reward = -np.inf # Keep track of train_rewards train_rewards = [] # To print mean actions per episode mean_action = [] # To keep track of moving averages moving_avgs = [] # Summary writer for tensorboard writer = SummaryWriter(log_dir="reinforcement/pytorch/runs") # Initialize policy if args.policy not in policies: raise ValueError( "Policy {} is not available, chose one of : {}".format( args.policy, list(policies.keys()))) policy = policies[args.policy](state_dim, action_dim, max_action, args.per, args.gradclip) # Evaluate untrained policy evaluations = [evaluate_policy(env, policy)] moving_avgs.append(evaluations[0]) writer.add_scalar("Timesteps/EvaluationReward", evaluations[0], total_timesteps) ## Initialize ReplayBuffer if args.per: print("Training with Prioritized Experience Reply") replay_buffer = PrioritizedReplayBuffer( args.replay_buffer_max_size, args.batch_size, args.seed, initial_beta=0.5, delta_beta=2 / args.max_timesteps, ) else: replay_buffer = ReplayBuffer(args.replay_buffer_max_size, args.batch_size, args.seed) # Load previous policy if args.load_initial_policy: # Disable random start steps args.start_timesteps = 0 # Load training data checkpoint = load_training_state(args.model_dir, args.policy + "_training") evaluations = checkpoint["evaluations"] total_timesteps = checkpoint["total_timesteps"] train_rewards = checkpoint["train_rewards"] episode_num = checkpoint["episode_num"] best_reward = checkpoint["best_reward"] moving_avgs = checkpoint["moving_avgs"] # Load policy policy.load(args.model_dir, args.policy) print("Starting training") obs = env.reset() while total_timesteps < args.max_timesteps: # Select action if total_timesteps < args.start_timesteps: action = env.action_space.sample() else: action = policy.predict(np.array(obs)) action = add_noise(action, args.expl_noise, env.action_space.low, env.action_space.high) mean_action.append(action) # Perform action new_obs, reward, done, _ = env.step(action) # Update episode reward episode_reward += reward # Store data in replay buffer replay_buffer.add(obs, action, reward, new_obs, float(done)) # Update network if len(replay_buffer) >= args.batch_size: policy.update(replay_buffer, args.discount, args.tau) # Update env obs = new_obs episode_timesteps += 1 total_timesteps += 1 timesteps_since_eval += 1 if episode_timesteps >= args.env_timesteps: done = True if done: print(( "Total T: %d Episode Num: %d \nMean actions: %.2f %.2f Episode T: %d Reward: %.1f Moving Average: %.1f" ) % ( total_timesteps, episode_num, np.mean(np.array(mean_action), axis=0)[0], np.mean(np.array(mean_action), axis=0)[1], episode_timesteps, episode_reward, moving_avgs[-1], )) train_rewards.append(episode_reward) moving_avgs.append(moving_average(train_rewards, avg_episodes)) writer.add_scalar("Timesteps/Rewards", episode_reward, total_timesteps) writer.add_scalar("Timesteps/MovingAverage", moving_avgs[-1], total_timesteps) # writer.add_scalar("Episode/Wheel1Mean", np.mean(np.array(mean_action), axis=0)[0], episode_num) # writer.add_scalar("Episode/Wheel2Mean", np.mean(np.array(mean_action), axis=0)[1], episode_num) # Evaluate episode if timesteps_since_eval >= args.eval_freq: timesteps_since_eval %= args.eval_freq eval_reward = evaluate_policy(env, policy) evaluations.append(eval_reward) writer.add_scalar("Timesteps/EvaluationReward", eval_reward, total_timesteps) print( "\n-+-+-+-+-+-+-+-+-+-+ Evaluation reward at time {}: {} +-+-+-+-+-+-+-+-+-+-" .format(total_timesteps, eval_reward)) np.savetxt( "reinforcement/pytorch/results/eval_rewards_" + args.policy + ".csv", np.array(evaluations), delimiter=",", ) np.savetxt( "reinforcement/pytorch/results/train_rewards_" + args.policy + ".csv", np.array(train_rewards), delimiter=",", ) np.savetxt( "reinforcement/pytorch/results/moving_averages_" + args.policy + ".csv", np.array(moving_avgs), delimiter=",", ) # Save the policy according to the best reward over training if eval_reward > best_reward: best_reward = eval_reward policy.save(args.model_dir, args.policy) save_training_state( args.model_dir, args.policy + "_training", best_reward, total_timesteps, evaluations, train_rewards, episode_num, moving_avgs, ) print( "-+-+-+-+-+-+-+-+-+-+ Model saved +-+-+-+-+-+-+-+-+-+-\n" ) # Reset environment mean_action = [] obs = env.reset() env_counter += 1 episode_reward = 0 episode_timesteps = 0 episode_num += 1 print("Finished..should return now!")