def _try_to_eval(self, epoch): logger.save_extra_data(self.get_extra_data_to_save(epoch)) if self._can_evaluate(): if self.environment_farming: # Create new new eval_sampler each evaluation time in order to avoid relesed environment problem env_for_eval_sampler = self.farmer.force_acq_env() print(env_for_eval_sampler) self.eval_sampler = InPlacePathSampler( env=env_for_eval_sampler, policy=self.eval_policy, max_samples=self.num_steps_per_eval + self.max_path_length, max_path_length=self.max_path_length, ) self.evaluate(epoch) # Adding env back to free_env list self.farmer.add_free_env(env_for_eval_sampler) params = self.get_epoch_snapshot(epoch) logger.save_itr_params(epoch, params) table_keys = logger.get_table_key_set() if self._old_table_keys is not None: assert table_keys == self._old_table_keys, ( "Table keys cannot change from iteration to iteration.") self._old_table_keys = table_keys logger.record_tabular( "Number of train steps total", self._n_train_steps_total, ) logger.record_tabular( "Number of env steps total", self._n_env_steps_total, ) logger.record_tabular( "Number of rollouts total", self._n_rollouts_total, ) times_itrs = gt.get_times().stamps.itrs train_time = times_itrs['train'][-1] sample_time = times_itrs['sample'][-1] eval_time = times_itrs['eval'][-1] if epoch > 0 else 0 epoch_time = train_time + sample_time + eval_time total_time = gt.get_times().total logger.record_tabular('Train Time (s)', train_time) logger.record_tabular('(Previous) Eval Time (s)', eval_time) logger.record_tabular('Sample Time (s)', sample_time) logger.record_tabular('Epoch Time (s)', epoch_time) logger.record_tabular('Total Train Time (s)', total_time) logger.record_tabular("Epoch", epoch) logger.dump_tabular(with_prefix=False, with_timestamp=False) else: logger.log("Skipping eval for now.")
def eval_alg(policy, env, max_path_length, num_eval_rollouts, env_seed, eval_deterministic=False): if eval_deterministic: policy = MakeDeterministic(policy) env.seed(env_seed) eval_sampler = InPlacePathSampler( env=env, policy=policy, max_samples=max_path_length * (num_eval_rollouts + 1), max_path_length=max_path_length, policy_uses_pixels=False, policy_uses_task_params=False, concat_task_params_to_policy_obs=False ) test_paths = eval_sampler.obtain_samples() path_trajs = [np.array([d['xy_pos'] for d in path["env_infos"]]) for path in test_paths] return {'path_trajs': path_trajs}
def eval_alg(policy, env, num_eval_rollouts, eval_deterministic=False, max_path_length=1000): if eval_deterministic: policy = MakeDeterministic(policy) eval_sampler = InPlacePathSampler(env=env, policy=policy, max_samples=max_path_length * (num_eval_rollouts + 1), max_path_length=max_path_length, policy_uses_pixels=False, policy_uses_task_params=False, concat_task_params_to_policy_obs=False) test_paths = eval_sampler.obtain_samples() average_returns = get_average_returns(test_paths) return average_returns
def __init__( self, env, qf1, qf2, policy, replay_buffer1, replay_buffer2, num_epochs=1000, num_steps_per_epoch=1000, policy_learning_rate=1e-4, batch_size=128, num_steps_per_eval=3000, max_path_length=300, discount=0.99, ): super().__init__() self.env = env self.qf1 = qf1 self.qf2 = qf2 self.policy = policy self.replay_buffer1 = replay_buffer1 self.replay_buffer2 = replay_buffer2 self.num_steps_per_epoch = num_steps_per_epoch self.num_epochs = num_epochs self.policy_learning_rate = policy_learning_rate self.batch_size = batch_size self.discount = discount self.eval_sampler = InPlacePathSampler( env=env, policy=self.policy, max_samples=num_steps_per_eval, max_path_length=max_path_length, ) self.policy_optimizer = optim.Adam(self.policy.parameters(), lr=self.policy_learning_rate)
def __init__( self, env, policy, train_tasks, eval_tasks, meta_batch=64, num_iterations=100, num_train_steps_per_itr=1000, num_tasks_sample=100, num_steps_per_task=100, num_evals=10, num_steps_per_eval=1000, batch_size=1024, embedding_batch_size=1024, embedding_mini_batch_size=1024, max_path_length=1000, discount=0.99, replay_buffer_size=1000000, #1000000, reward_scale=1, train_embedding_source='posterior_only', eval_embedding_source='initial_pool', eval_deterministic=True, render=False, save_replay_buffer=False, save_algorithm=False, save_environment=False, obs_emb_dim=0): """ Base class for Meta RL Algorithms :param env: training env :param policy: policy that is conditioned on a latent variable z that rl_algorithm is responsible for feeding in :param train_tasks: list of tasks used for training :param eval_tasks: list of tasks used for eval :param meta_batch: number of tasks used for meta-update :param num_iterations: number of meta-updates taken :param num_train_steps_per_itr: number of meta-updates performed per iteration :param num_tasks_sample: number of train tasks to sample to collect data for :param num_steps_per_task: number of transitions to collect per task :param num_evals: number of independent evaluation runs, with separate task encodings :param num_steps_per_eval: number of transitions to sample for evaluation :param batch_size: size of batches used to compute RL update :param embedding_batch_size: size of batches used to compute embedding :param embedding_mini_batch_size: size of batch used for encoder update :param max_path_length: max episode length :param discount: :param replay_buffer_size: max replay buffer size :param reward_scale: :param render: :param save_replay_buffer: :param save_algorithm: :param save_environment: """ self.env = env self.policy = policy self.exploration_policy = policy # Can potentially use a different policy purely for exploration rather than also solving tasks, currently not being used self.train_tasks = train_tasks self.eval_tasks = eval_tasks self.meta_batch = meta_batch self.num_iterations = num_iterations self.num_train_steps_per_itr = num_train_steps_per_itr self.num_tasks_sample = num_tasks_sample self.num_steps_per_task = num_steps_per_task self.num_evals = num_evals self.num_steps_per_eval = num_steps_per_eval self.batch_size = batch_size self.embedding_batch_size = embedding_batch_size self.embedding_mini_batch_size = embedding_mini_batch_size self.max_path_length = max_path_length self.discount = discount self.replay_buffer_size = min( int(replay_buffer_size / (len(train_tasks))), 1000) self.reward_scale = reward_scale self.train_embedding_source = train_embedding_source self.eval_embedding_source = eval_embedding_source # TODO: add options for computing embeddings on train tasks too self.eval_deterministic = eval_deterministic self.render = render self.save_replay_buffer = save_replay_buffer self.save_algorithm = save_algorithm self.save_environment = save_environment self.eval_sampler = InPlacePathSampler( env=env, policy=policy, max_samples=self.num_steps_per_eval, max_path_length=self.max_path_length, ) # separate replay buffers for # - training RL update # - training encoder update # - testing encoder self.replay_buffer = MultiTaskReplayBuffer(self.replay_buffer_size, env, self.train_tasks, state_dim=obs_emb_dim) self.enc_replay_buffer = MultiTaskReplayBuffer(self.replay_buffer_size, env, self.train_tasks, state_dim=obs_emb_dim) self.eval_enc_replay_buffer = MultiTaskReplayBuffer( self.replay_buffer_size, env, self.eval_tasks, state_dim=obs_emb_dim) self._n_env_steps_total = 0 self._n_train_steps_total = 0 self._n_rollouts_total = 0 self._do_train_time = 0 self._epoch_start_time = None self._algo_start_time = None self._old_table_keys = None self._current_path_builder = PathBuilder() self._exploration_paths = []
} # set up the policy # policy = joblib.load(POLICY_SAVE_PATH)['exploration_policy'] policy = joblib.load(POLICY_SAVE_PATH) # set up the env # if env_specs['train_test_env']: # _, training_env = get_env(env_specs) # else: # training_env, _ = get_env(env_specs) # training_env = DebugFetchReachAndLiftEnv() training_env = WrappedRotatedFetchReachAnywhereEnv() # build an eval sampler that also renders eval_sampler = InPlacePathSampler( env=training_env, policy=policy, max_samples=max_samples, max_path_length=max_path_length, policy_uses_pixels=policy_specs['policy_uses_pixels'], policy_uses_task_params=policy_specs['policy_uses_task_params'], concat_task_params_to_policy_obs=policy_specs['concat_task_params_to_policy_obs'], animated=True ) eval_sampler.obtain_samples() training_env.close() eval_sampler = None
def __init__(self, env, agent, train_tasks, eval_tasks, goal_radius, eval_deterministic=True, render=False, render_eval_paths=False, plotter=None, **kwargs): """ :param env: training env :param agent: agent that is conditioned on a latent variable z that rl_algorithm is responsible for feeding in :param train_tasks: list of tasks used for training :param eval_tasks: list of tasks used for eval :param goal_radius: reward threshold for defining sparse rewards see default experiment config file for descriptions of the rest of the arguments """ self.env = env self.agent = agent self.train_tasks = train_tasks self.eval_tasks = eval_tasks self.goal_radius = goal_radius self.meta_batch = kwargs['meta_batch'] self.batch_size = kwargs['batch_size'] self.num_iterations = kwargs['num_iterations'] self.num_train_steps_per_itr = kwargs['num_train_steps_per_itr'] self.num_initial_steps = kwargs['num_initial_steps'] self.num_tasks_sample = kwargs['num_tasks_sample'] self.num_steps_prior = kwargs['num_steps_prior'] self.num_steps_posterior = kwargs['num_steps_posterior'] self.num_extra_rl_steps_posterior = kwargs[ 'num_extra_rl_steps_posterior'] self.num_evals = kwargs['num_evals'] self.num_steps_per_eval = kwargs['num_steps_per_eval'] self.embedding_batch_size = kwargs['embedding_batch_size'] self.embedding_mini_batch_size = kwargs['embedding_mini_batch_size'] self.max_path_length = kwargs['max_path_length'] self.discount = kwargs['discount'] self.replay_buffer_size = kwargs['replay_buffer_size'] self.reward_scale = kwargs['reward_scale'] self.update_post_train = kwargs['update_post_train'] self.num_exp_traj_eval = kwargs['num_exp_traj_eval'] self.save_replay_buffer = kwargs['save_replay_buffer'] self.save_algorithm = kwargs['save_algorithm'] self.save_environment = kwargs['save_environment'] self.dump_eval_paths = kwargs['dump_eval_paths'] self.data_dir = kwargs['data_dir'] self.train_epoch = kwargs['train_epoch'] self.eval_epoch = kwargs['eval_epoch'] self.sample = kwargs['sample'] self.n_trj = kwargs['n_trj'] self.allow_eval = kwargs['allow_eval'] self.mb_replace = kwargs['mb_replace'] self.eval_deterministic = eval_deterministic self.render = render self.eval_statistics = None self.render_eval_paths = render_eval_paths self.plotter = plotter self.train_buffer = MultiTaskReplayBuffer(self.replay_buffer_size, env, self.train_tasks, self.goal_radius) self.eval_buffer = MultiTaskReplayBuffer(self.replay_buffer_size, env, self.eval_tasks, self.goal_radius) self.replay_buffer = MultiTaskReplayBuffer(self.replay_buffer_size, env, self.train_tasks, self.goal_radius) self.enc_replay_buffer = MultiTaskReplayBuffer(self.replay_buffer_size, env, self.train_tasks, self.goal_radius) # offline sampler which samples from the train/eval buffer self.offline_sampler = OfflineInPlacePathSampler( env=env, policy=agent, max_path_length=self.max_path_length) # online sampler for evaluation (if collect on-policy context, for offline context, use self.offline_sampler) self.sampler = InPlacePathSampler(env=env, policy=agent, max_path_length=self.max_path_length) self._n_env_steps_total = 0 self._n_train_steps_total = 0 self._n_rollouts_total = 0 self._do_train_time = 0 self._epoch_start_time = None self._algo_start_time = None self._old_table_keys = None self._current_path_builder = PathBuilder() self._exploration_paths = [] self.init_buffer()
def __init__( self, env, exploration_policy: ExplorationPolicy, expert_replay_buffer, training_env=None, num_epochs=100, num_steps_per_epoch=10000, num_steps_per_eval=1000, num_steps_between_updates=1000, min_steps_before_training=1000, max_path_length=1000, discount=0.99, replay_buffer_size=10000, render=False, save_replay_buffer=False, save_algorithm=False, save_environment=False, save_best=False, save_best_starting_from_epoch=0, eval_sampler=None, eval_policy=None, replay_buffer=None, policy_uses_pixels=False, wrap_absorbing=False, freq_saving=1, # some environment like halfcheetah_v2 have a timelimit that defines the terminal # this is used as a minor hack to turn off time limits no_terminal=False, policy_uses_task_params=False, concat_task_params_to_policy_obs=False ): """ Base class for RL Algorithms :param env: Environment used to evaluate. :param exploration_policy: Policy used to explore :param training_env: Environment used by the algorithm. By default, a copy of `env` will be made. :param num_epochs: :param num_steps_per_epoch: :param num_steps_per_eval: :param num_updates_per_env_step: Used by online training mode. :param num_updates_per_epoch: Used by batch training mode. :param batch_size: :param max_path_length: :param discount: :param replay_buffer_size: :param render: :param save_replay_buffer: :param save_algorithm: :param save_environment: :param eval_sampler: :param eval_policy: Policy to evaluate with. :param replay_buffer: """ self.training_env = training_env or pickle.loads(pickle.dumps(env)) # self.training_env = training_env or deepcopy(env) self.exploration_policy = exploration_policy self.expert_replay_buffer = expert_replay_buffer self.num_epochs = num_epochs self.num_env_steps_per_epoch = num_steps_per_epoch self.num_steps_per_eval = num_steps_per_eval self.num_steps_between_updates = num_steps_between_updates self.min_steps_before_training = min_steps_before_training self.max_path_length = max_path_length self.discount = discount self.replay_buffer_size = replay_buffer_size self.render = render self.save_replay_buffer = save_replay_buffer self.save_algorithm = save_algorithm self.save_environment = save_environment self.save_best = save_best self.save_best_starting_from_epoch = save_best_starting_from_epoch self.policy_uses_pixels = policy_uses_pixels self.policy_uses_task_params = policy_uses_task_params self.concat_task_params_to_policy_obs = concat_task_params_to_policy_obs if eval_sampler is None: if eval_policy is None: eval_policy = exploration_policy eval_sampler = InPlacePathSampler( env=env, policy=eval_policy, max_samples=self.num_steps_per_eval + self.max_path_length, max_path_length=self.max_path_length, policy_uses_pixels=policy_uses_pixels, policy_uses_task_params=policy_uses_task_params, concat_task_params_to_policy_obs=concat_task_params_to_policy_obs ) self.eval_policy = eval_policy self.eval_sampler = eval_sampler self.action_space = env.action_space self.obs_space = env.observation_space self.env = env if replay_buffer is None: replay_buffer = EnvReplayBuffer( self.replay_buffer_size, self.env, policy_uses_pixels=self.policy_uses_pixels, policy_uses_task_params=self.policy_uses_task_params, concat_task_params_to_policy_obs=self.concat_task_params_to_policy_obs ) self.replay_buffer = replay_buffer self._n_env_steps_total = 0 self._n_train_steps_total = 0 self._n_rollouts_total = 0 self._do_train_time = 0 self._epoch_start_time = None self._algo_start_time = None self._old_table_keys = None self._current_path_builder = PathBuilder() self._exploration_paths = [] self.wrap_absorbing = wrap_absorbing self.freq_saving = freq_saving self.no_terminal = no_terminal
def __init__( self, env, exploration_policy: ExplorationPolicy, training_env=None, num_epochs=100, num_steps_per_epoch=10000, num_steps_per_eval=1000, num_updates_per_env_step=1, batch_size=1024, max_path_length=1000, discount=0.99, replay_buffer_size=1000000, reward_scale=1, render=False, save_replay_buffer=False, save_algorithm=False, save_environment=False, eval_sampler=None, eval_policy=None, replay_buffer=None, demo_path=None, action_skip=1, experiment_name="default", mix_demo=False, ): """ Base class for RL Algorithms :param env: Environment used to evaluate. :param exploration_policy: Policy used to explore :param training_env: Environment used by the algorithm. By default, a copy of `env` will be made. :param num_epochs: :param num_steps_per_epoch: :param num_steps_per_eval: :param num_updates_per_env_step: Used by online training mode. :param num_updates_per_epoch: Used by batch training mode. :param batch_size: :param max_path_length: :param discount: :param replay_buffer_size: :param reward_scale: :param render: :param save_replay_buffer: :param save_algorithm: :param save_environment: :param eval_sampler: :param eval_policy: Policy to evaluate with. :param replay_buffer: """ ### TODO: look at NormalizedBoxEnv, do we need it? ### # self.training_env = training_env or gym.make("HalfCheetah-v2") self.training_env = training_env or MujocoManipEnv( env.env.__class__.__name__) self.exploration_policy = exploration_policy self.num_epochs = num_epochs self.num_env_steps_per_epoch = num_steps_per_epoch self.num_steps_per_eval = num_steps_per_eval self.num_updates_per_train_call = num_updates_per_env_step self.batch_size = batch_size self.max_path_length = max_path_length self.discount = discount self.replay_buffer_size = replay_buffer_size self.reward_scale = reward_scale self.render = render self.save_replay_buffer = save_replay_buffer self.save_algorithm = save_algorithm self.save_environment = save_environment if eval_sampler is None: if eval_policy is None: eval_policy = exploration_policy eval_sampler = InPlacePathSampler( env=env, policy=eval_policy, max_samples=self.num_steps_per_eval + self.max_path_length, max_path_length=self.max_path_length, ) self.eval_policy = eval_policy self.eval_sampler = eval_sampler self.action_space = env.action_space self.obs_space = env.observation_space self.env = env if replay_buffer is None: replay_buffer = EnvReplayBuffer( self.replay_buffer_size, self.env, ) self.replay_buffer = replay_buffer self.demo_sampler = None self.mix_demo = mix_demo if demo_path is not None: self.demo_sampler = DemoSampler( demo_path=demo_path, observation_dim=self.obs_space.shape[0], action_dim=self.action_space.shape[0], preload=True) self.action_skip = action_skip self.action_skip_count = 0 self._n_env_steps_total = 0 self._n_train_steps_total = 0 self._n_rollouts_total = 0 self._do_train_time = 0 self._epoch_start_time = None self._algo_start_time = None self._old_table_keys = None self._current_path_builder = PathBuilder() self._exploration_paths = [] t_now = time.time() time_str = datetime.datetime.fromtimestamp(t_now).strftime( '%Y%m%d%H%M%S') os.makedirs(os.path.join(LOCAL_EXP_PATH, experiment_name, time_str)) self._writer = SummaryWriter( os.path.join(LOCAL_EXP_PATH, experiment_name, time_str))
def __init__( self, env, exploration_policy: ExplorationPolicy, training_env=None, num_epochs=100, num_steps_per_epoch=10000, num_steps_per_eval=1000, num_updates_per_env_step=1, num_updates_per_epoch=None, batch_size=1024, max_path_length=1000, discount=0.99, replay_buffer_size=1000000, reward_scale=1, min_num_steps_before_training=None, render=False, save_replay_buffer=False, save_algorithm=False, save_environment=True, eval_sampler=None, eval_policy=None, replay_buffer=None, collection_mode='online', ): """ Base class for RL Algorithms :param env: Environment used to evaluate. :param exploration_policy: Policy used to explore :param training_env: Environment used by the algorithm. By default, a copy of `env` will be made for training, so that training and evaluation are completely independent. :param num_epochs: :param num_steps_per_epoch: :param num_steps_per_eval: :param num_updates_per_env_step: Used by online training mode. :param num_updates_per_epoch: Used by batch training mode. :param batch_size: :param max_path_length: :param discount: :param replay_buffer_size: :param reward_scale: :param min_num_steps_before_training: :param render: :param save_replay_buffer: :param save_algorithm: :param save_environment: :param eval_sampler: :param eval_policy: Policy to evaluate with. :param replay_buffer: :param collection_mode: String determining how training happens - 'online': Train after every step taken in the environment. - 'batch': Train after every epoch. """ assert collection_mode in ['online', 'batch'] if collection_mode == 'batch': assert num_updates_per_epoch is not None self.training_env = training_env #or pickle.loads(pickle.dumps(env)) self.exploration_policy = exploration_policy self.num_epochs = num_epochs self.num_env_steps_per_epoch = num_steps_per_epoch self.num_steps_per_eval = num_steps_per_eval if collection_mode == 'online': self.num_updates_per_train_call = num_updates_per_env_step else: self.num_updates_per_train_call = num_updates_per_epoch self.batch_size = batch_size self.max_path_length = max_path_length self.discount = discount self.replay_buffer_size = replay_buffer_size self.reward_scale = reward_scale self.render = render self.collection_mode = collection_mode self.save_replay_buffer = save_replay_buffer self.save_algorithm = save_algorithm self.save_environment = save_environment if min_num_steps_before_training is None: min_num_steps_before_training = self.num_env_steps_per_epoch self.min_num_steps_before_training = min_num_steps_before_training if eval_sampler is None: if eval_policy is None: eval_policy = exploration_policy eval_sampler = InPlacePathSampler( env=env, policy=eval_policy, max_samples=self.num_steps_per_eval + self.max_path_length, max_path_length=self.max_path_length, ) self.eval_policy = eval_policy self.eval_sampler = eval_sampler self.eval_statistics = OrderedDict() self.need_to_update_eval_statistics = True self.action_space = env.action_space self.obs_space = env.observation_space self.env = env if replay_buffer is None: replay_buffer = EnvReplayBuffer( self.replay_buffer_size, self.env, ) self.replay_buffer = replay_buffer self._n_env_steps_total = 0 self._n_train_steps_total = 0 self._n_rollouts_total = 0 self._do_train_time = 0 self._epoch_start_time = None self._algo_start_time = None self._old_table_keys = None self._current_path_builder = PathBuilder() self._exploration_paths = [] self.post_epoch_funcs = []
def __init__( self, env, exploration_policy: ExplorationPolicy, training_env=None, num_epochs=100, num_steps_per_epoch=10000, num_steps_per_eval=1000, num_updates_per_env_step=1, max_num_episodes=None, batch_size=1024, max_path_length=1000, discount=0.99, replay_buffer_size=1000000, reward_scale=1, render=False, save_replay_buffer=False, save_algorithm=False, save_environment=False, save_best=False, save_best_starting_from_epoch=0, eval_sampler=None, eval_policy=None, replay_buffer=None, # for compatibility with deepmind control suite # Right now the semantics is that if observations is not a dictionary # then it means the policy just uses that. If it's a dictionary, it # checks whether policy_uses_pixels to see if it's true or false and # based on that it decides whether the policy takes 'pixels' or 'obs' # from the dictionary policy_uses_pixels=False, freq_saving=1, # for meta-learning policy_uses_task_params=False, # whether the policy uses the task parameters concat_task_params_to_policy_obs=False, # how the policy sees the task parameters # this is useful when you want to generate trajectories from the expert using the # exploration policy do_not_train=False, # some environment like halfcheetah_v2 have a timelimit that defines the terminal # this is used as a minor hack to turn off time limits no_terminal=False, **kwargs ): """ Base class for RL Algorithms :param env: Environment used to evaluate. :param exploration_policy: Policy used to explore :param training_env: Environment used by the algorithm. By default, a copy of `env` will be made. :param num_epochs: :param num_steps_per_epoch: :param num_steps_per_eval: :param num_updates_per_env_step: Used by online training mode. :param num_updates_per_epoch: Used by batch training mode. :param batch_size: :param max_path_length: :param discount: :param replay_buffer_size: :param reward_scale: :param render: :param save_replay_buffer: :param save_algorithm: :param save_environment: :param eval_sampler: :param eval_policy: Policy to evaluate with. :param replay_buffer: """ self.training_env = training_env or pickle.loads(pickle.dumps(env)) # self.training_env = training_env or deepcopy(env) self.exploration_policy = exploration_policy self.num_epochs = num_epochs self.num_env_steps_per_epoch = num_steps_per_epoch self.num_steps_per_eval = num_steps_per_eval self.num_updates_per_train_call = num_updates_per_env_step self.batch_size = batch_size self.max_path_length = max_path_length self.discount = discount self.replay_buffer_size = replay_buffer_size self.reward_scale = reward_scale self.render = render self.save_replay_buffer = save_replay_buffer self.save_algorithm = save_algorithm self.save_environment = save_environment self.save_best = save_best self.save_best_starting_from_epoch = save_best_starting_from_epoch self.policy_uses_pixels = policy_uses_pixels self.policy_uses_task_params = policy_uses_task_params self.concat_task_params_to_policy_obs = concat_task_params_to_policy_obs self.freq_saving = freq_saving if eval_sampler is None: if eval_policy is None: eval_policy = exploration_policy eval_sampler = InPlacePathSampler( env=env, policy=eval_policy, max_samples=self.num_steps_per_eval + self.max_path_length, max_path_length=self.max_path_length, policy_uses_pixels=policy_uses_pixels, policy_uses_task_params=policy_uses_task_params, concat_task_params_to_policy_obs=concat_task_params_to_policy_obs ) self.eval_policy = eval_policy self.eval_sampler = eval_sampler self.action_space = env.action_space self.obs_space = env.observation_space self.env = env if replay_buffer is None: replay_buffer = EnvReplayBuffer( self.replay_buffer_size, self.env, policy_uses_pixels=self.policy_uses_pixels, policy_uses_task_params=self.policy_uses_task_params, concat_task_params_to_policy_obs=self.concat_task_params_to_policy_obs ) self.replay_buffer = replay_buffer self._n_env_steps_total = 0 self._n_train_steps_total = 0 self._n_rollouts_total = 0 self._do_train_time = 0 self._epoch_start_time = None self._algo_start_time = None self._old_table_keys = None self._current_path_builder = PathBuilder() self._exploration_paths = [] self.do_not_train = do_not_train self.num_episodes = 0 self.max_num_episodes = max_num_episodes if max_num_episodes is not None else float('inf') self.no_terminal = no_terminal
def __init__( self, env_sampler, exploration_policy: ExplorationPolicy, neural_process, train_neural_process=False, latent_repr_mode='concat_params', # OR concat_samples num_latent_samples=5, num_epochs=100, num_steps_per_epoch=10000, num_steps_per_eval=1000, num_updates_per_env_step=1, batch_size=1024, max_path_length=1000, discount=0.99, replay_buffer_size=1000000, reward_scale=1, render=False, save_replay_buffer=False, save_algorithm=False, save_environment=False, eval_sampler=None, eval_policy=None, replay_buffer=None, epoch_to_start_training=0): """ Base class for RL Algorithms :param env: Environment used to evaluate. :param exploration_policy: Policy used to explore :param training_env: Environment used by the algorithm. By default, a copy of `env` will be made. :param num_epochs: :param num_steps_per_epoch: :param num_steps_per_eval: :param num_updates_per_env_step: Used by online training mode. :param num_updates_per_epoch: Used by batch training mode. :param batch_size: :param max_path_length: :param discount: :param replay_buffer_size: :param reward_scale: :param render: :param save_replay_buffer: :param save_algorithm: :param save_environment: :param eval_sampler: :param eval_policy: Policy to evaluate with. :param replay_buffer: """ assert not train_neural_process, 'Have not implemented it yet! Remember to set it to train mode when training' self.neural_process = neural_process self.neural_process.set_mode('eval') self.latent_repr_mode = latent_repr_mode self.num_latent_samples = num_latent_samples self.env_sampler = env_sampler env, env_specs = env_sampler() self.training_env, _ = env_sampler(env_specs) # self.training_env = training_env or pickle.loads(pickle.dumps(env)) # self.training_env = training_env or deepcopy(env) self.exploration_policy = exploration_policy self.num_epochs = num_epochs self.num_env_steps_per_epoch = num_steps_per_epoch self.num_steps_per_eval = num_steps_per_eval self.num_updates_per_train_call = num_updates_per_env_step self.batch_size = batch_size self.max_path_length = max_path_length self.discount = discount self.replay_buffer_size = replay_buffer_size self.reward_scale = reward_scale self.render = render self.save_replay_buffer = save_replay_buffer self.save_algorithm = save_algorithm self.save_environment = save_environment self.epoch_to_start_training = epoch_to_start_training if self.latent_repr_mode == 'concat_params': def get_latent_repr(posterior_state): z_mean, z_cov = self.neural_process.get_posterior_params( posterior_state) return np.concatenate([z_mean, z_cov]) self.extra_obs_dim = 2 * self.neural_process.z_dim else: def get_latent_repr(posterior_state): z_mean, z_cov = self.neural_process.get_posterior_params( posterior_state) samples = np.random.multivariate_normal( z_mean, np.diag(z_cov), self.num_latent_samples) samples = samples.flatten() return samples self.extra_obs_dim = self.num_latent_samples * self.neural_process.z_dim self.get_latent_repr = get_latent_repr if eval_sampler is None: if eval_policy is None: eval_policy = exploration_policy eval_sampler = InPlacePathSampler( env=env, policy=eval_policy, max_samples=self.num_steps_per_eval + self.max_path_length, max_path_length=self.max_path_length, neural_process=neural_process, latent_repr_fn=get_latent_repr, reward_scale=reward_scale) self.eval_policy = eval_policy self.eval_sampler = eval_sampler self.action_space = env.action_space self.obs_space = env.observation_space self.env = env obs_space_dim = gym_get_dim(self.obs_space) act_space_dim = gym_get_dim(self.action_space) if replay_buffer is None: replay_buffer = SimpleReplayBuffer( self.replay_buffer_size, obs_space_dim + self.extra_obs_dim, act_space_dim, discrete_action_dim=isinstance(self.action_space, Discrete)) self.replay_buffer = replay_buffer self._n_env_steps_total = 0 self._n_train_steps_total = 0 self._n_rollouts_total = 0 self._do_train_time = 0 self._epoch_start_time = None self._algo_start_time = None self._old_table_keys = None self._current_path_builder = PathBuilder() self._exploration_paths = []
def __init__( self, env, agent, train_goals, wd_goals, ood_goals, replay_buffers, meta_batch_size=64, num_iterations=100, num_train_steps_per_itr=1000, num_tasks=100, num_steps_prior=100, num_steps_posterior=100, num_extra_rl_steps_posterior=100, num_evals=10, num_steps_per_eval=1000, max_path_length=1000, discount=0.99, reward_scale=1, num_exp_traj_eval=1, eval_deterministic=True, render=False, save_replay_buffer=False, save_algorithm=False, save_environment=False, render_eval_paths=False, dump_eval_paths=False, plotter=None, use_same_context=True, recurrent=False, ): """ :param env: training env :param agent: agent that is conditioned on a latent variable z that rl_algorithm is responsible for feeding in :param train_tasks: list of tasks used for training :param eval_tasks: list of tasks used for eval see default experiment config file for descriptions of the rest of the arguments """ self.env = env self.agent = agent self.train_goals = train_goals self.wd_goals = wd_goals self.ood_goals = ood_goals self.replay_buffers = replay_buffers self.num_iterations = num_iterations self.num_train_steps_per_itr = num_train_steps_per_itr self.meta_batch_size = meta_batch_size self.num_evals = num_evals self.num_steps_per_eval = num_steps_per_eval self.max_path_length = max_path_length self.discount = discount self.reward_scale = reward_scale self.num_exp_traj_eval = num_exp_traj_eval self.eval_deterministic = eval_deterministic self.render = render self.save_replay_buffer = save_replay_buffer self.save_algorithm = save_algorithm self.save_environment = save_environment self.use_same_context = use_same_context self.recurrent = recurrent self.eval_statistics = None self.render_eval_paths = render_eval_paths self.dump_eval_paths = dump_eval_paths self.plotter = plotter self.sampler = InPlacePathSampler( env=env, policy=agent, max_path_length=self.max_path_length, ) # separate replay buffers for # - training RL update # - training encoder update self._n_env_steps_total = 0 self._n_train_steps_total = 0 self._n_rollouts_total = 0 self._do_train_time = 0 self._epoch_start_time = None self._algo_start_time = None self._old_table_keys = None self._current_path_builder = PathBuilder() self._exploration_paths = []
def __init__( self, env, agent, train_tasks, eval_tasks, meta_batch=64, num_iterations=100, num_train_steps_per_itr=1000, num_initial_steps=100, num_tasks_sample=100, num_steps_prior=100, num_steps_posterior=100, num_extra_rl_steps_posterior=100, num_evals=10, num_steps_per_eval=1000, batch_size=1024, low_batch_size=2048, #TODO: Tune this batch size embedding_batch_size=1024, embedding_mini_batch_size=1024, max_path_length=1000, discount=0.99, replay_buffer_size=1000000, reward_scale=1, num_exp_traj_eval=1, update_post_train=1, eval_deterministic=True, render=False, save_replay_buffer=False, save_algorithm=False, save_environment=False, render_eval_paths=False, dump_eval_paths=False, plotter=None, use_goals=False): """ :param env: training env :param agent: agent that is conditioned on a latent variable z that rl_algorithm is responsible for feeding in :param train_tasks: list of tasks used for training :param eval_tasks: list of tasks used for eval see default experiment config file for descriptions of the rest of the arguments """ self.env = env self.agent = agent self.use_goals = use_goals assert (agent.use_goals == self.use_goals) self.exploration_agent = agent # Can potentially use a different policy purely for exploration rather than also solving tasks, currently not being used self.train_tasks = train_tasks self.eval_tasks = eval_tasks self.meta_batch = meta_batch self.num_iterations = num_iterations self.num_train_steps_per_itr = num_train_steps_per_itr self.num_initial_steps = num_initial_steps self.num_tasks_sample = num_tasks_sample self.num_steps_prior = num_steps_prior self.num_steps_posterior = num_steps_posterior self.num_extra_rl_steps_posterior = num_extra_rl_steps_posterior self.num_evals = num_evals self.num_steps_per_eval = num_steps_per_eval self.batch_size = batch_size self.embedding_batch_size = embedding_batch_size self.embedding_mini_batch_size = embedding_mini_batch_size self.low_batch_size = low_batch_size self.max_path_length = max_path_length self.discount = discount self.replay_buffer_size = replay_buffer_size self.reward_scale = reward_scale self.update_post_train = update_post_train self.num_exp_traj_eval = num_exp_traj_eval self.eval_deterministic = eval_deterministic self.render = render self.save_replay_buffer = save_replay_buffer self.save_algorithm = save_algorithm self.save_environment = save_environment self.eval_statistics = None self.render_eval_paths = render_eval_paths self.dump_eval_paths = dump_eval_paths self.plotter = plotter obs_dim = int(np.prod(env.observation_space.shape)) action_dim = int(np.prod(env.action_space.shape)) self.sampler = InPlacePathSampler( env=env, policy=agent, max_path_length=self.max_path_length, ) # separate replay buffers for # - training RL update # - training encoder update self.enc_replay_buffer = MultiTaskReplayBuffer( self.replay_buffer_size, env, self.train_tasks, ) if self.use_goals: self.high_buffer = MultiTaskReplayBuffer(self.replay_buffer_size, env, self.train_tasks) #Hacky method for changing the obs and action dimensions for the internal #buffers since they're not the same as the original environment internal_buffers = dict([ (idx, SimpleReplayBuffer( max_replay_buffer_size=self.replay_buffer_size, observation_dim=obs_dim, action_dim=obs_dim, )) for idx in self.train_tasks ]) self.high_buffer.task_buffers = internal_buffers self.low_buffer = SimpleReplayBuffer( max_replay_buffer_size=replay_buffer_size, observation_dim=2 * obs_dim, action_dim=action_dim, ) else: self.replay_buffer = MultiTaskReplayBuffer( self.replay_buffer_size, env, self.train_tasks, ) self._n_env_steps_total = 0 self._n_train_steps_total = 0 self._n_rollouts_total = 0 self._do_train_time = 0 self._epoch_start_time = None self._algo_start_time = None self._old_table_keys = None self._current_path_builder = PathBuilder() self._exploration_paths = []
def experiment(log_dir, variant_overwrite, cpu=False): if not cpu: ptu.set_gpu_mode(True) # optionally set the GPU (default=False) # Load experiment from file. env, _, data, variant = load_experiment(log_dir, variant_overwrite) #assert all([a == b for a, b in zip(print(samples)env.sampled_goal, variant['env_kwargs']['goal_prior'])]) # Set log directory. exp_id = 'eval/ne{}-mpl{}-{}-rs{}/nhp{}'.format( variant['algo_kwargs']['num_episodes'], variant['algo_kwargs']['max_path_length'], ','.join(variant_overwrite['env_kwargs']['shaped_rewards']), variant['algo_kwargs']['reward_scale'], variant['historical_policies_kwargs']['num_historical_policies'], ) exp_id = create_exp_name(exp_id) out_dir = os.path.join(log_dir, exp_id) print('Logging to:', out_dir) setup_logger( log_dir=out_dir, variant=variant, snapshot_mode='none', snapshot_gap=50, ) # Load trained model from file. policy = data['policy'] vf = data['vf'] qf = data['qf'] algorithm = SoftActorCritic( env=env, training_env=env, # can't clone box2d env cause of swig save_environment=False, # can't save box2d env cause of swig policy=policy, qf=qf, vf=vf, **variant['algo_kwargs'], ) # Overwrite algorithm for p(z) adaptation (if model is SMM). if variant['intrinsic_reward'] == 'smm': discriminator = data['discriminator'] density_model = data['density_model'] SMMHook(base_algorithm=algorithm, discriminator=discriminator, density_model=density_model, **variant['smm_kwargs']) # Overwrite algorithm for historical averaging. if variant['historical_policies_kwargs']['num_historical_policies'] > 0: HistoricalPoliciesHook( base_algorithm=algorithm, log_dir=log_dir, **variant['historical_policies_kwargs'], ) algorithm.to(ptu.device) #algorithm.train() samples = algorithm.get_eval_paths() #for path in samples: # print(path['observations']) #plt.figure() #plt.plot(samples[0]['observations'][:, 0], samples[0]['observations'][:, 1]) #plt.plot(3, 2) #plt.show() print(env.reset()) print(samples[0]['observations']) i = 0 for path in samples: np.save('./outtem/out%i.npy' % i, path['observations']) i = i + 1 #print(algorithm.policy.get_action(np.array([0,0]))) from rlkit.samplers.util import rollout from rlkit.samplers.in_place import InPlacePathSampler #path=rollout(env,algorithm.eval_policy,50) eval_sampler = InPlacePathSampler( env=env, policy=algorithm.eval_policy, max_samples=100, max_path_length=50, ) path = algorithm.eval_sampler.obtain_samples() print(path[0]['observations'])
def __init__( self, env, agent, train_tasks, eval_tasks, meta_batch=64, num_iterations=100, num_train_steps_per_itr=1000, num_initial_steps=100, num_tasks_sample=100, num_steps_prior=100, num_steps_posterior=100, num_extra_rl_steps_posterior=100, num_evals=10, num_steps_per_eval=1000, batch_size=1024, embedding_batch_size=1024, embedding_mini_batch_size=1024, max_path_length=1000, discount=0.99, replay_buffer_size=1000000, reward_scale=1, num_exp_traj_eval=1, update_post_train=1, eval_deterministic=True, render=False, save_replay_buffer=False, save_algorithm=False, save_environment=False, render_eval_paths=False, dump_eval_paths=False, plotter=None, dyna=False, dyna_num_train_itr=50, dyna_num_train_steps_per_itr=50, dyna_tandem_train=True, dyna_n_layers=3, dyna_hidden_layer_size=64, dyna_learning_rate=1e-3, ): """ :param env: training env :param agent: agent that is conditioned on a latent variable z that rl_algorithm is responsible for feeding in :param train_tasks: list of tasks used for training :param eval_tasks: list of tasks used for eval see default experiment config file for descriptions of the rest of the arguments """ self.env = env self.agent = agent self.exploration_agent = agent # Can potentially use a different policy purely for exploration rather than also solving tasks, currently not being used self.train_tasks = train_tasks self.eval_tasks = eval_tasks self.meta_batch = meta_batch self.num_iterations = num_iterations self.num_train_steps_per_itr = num_train_steps_per_itr self.num_initial_steps = num_initial_steps self.num_tasks_sample = num_tasks_sample self.num_steps_prior = num_steps_prior self.num_steps_posterior = num_steps_posterior self.num_extra_rl_steps_posterior = num_extra_rl_steps_posterior self.num_evals = num_evals self.num_steps_per_eval = num_steps_per_eval self.batch_size = batch_size self.embedding_batch_size = embedding_batch_size self.embedding_mini_batch_size = embedding_mini_batch_size self.max_path_length = max_path_length self.discount = discount self.replay_buffer_size = replay_buffer_size self.reward_scale = reward_scale self.update_post_train = update_post_train self.num_exp_traj_eval = num_exp_traj_eval self.eval_deterministic = eval_deterministic self.render = render self.save_replay_buffer = save_replay_buffer self.save_algorithm = save_algorithm self.save_environment = save_environment self.eval_statistics = None self.render_eval_paths = render_eval_paths self.dump_eval_paths = dump_eval_paths self.plotter = plotter self.dyna = dyna self.dyna_num_train_itr = dyna_num_train_itr self.dyna_num_train_steps_per_itr = dyna_num_train_steps_per_itr self.dyna_tandem_train = dyna_tandem_train self.dyna_n_layers = dyna_n_layers self.dyna_hidden_layer_size = dyna_hidden_layer_size self.dyna_learning_rate = dyna_learning_rate if dyna: self.sampler = DynamicsSampler( env=env, policy=agent, max_path_length=self.max_path_length, num_train_itr=dyna_num_train_itr, num_train_steps_per_itr=dyna_num_train_steps_per_itr, tandem_train=dyna_tandem_train, n_layers=dyna_n_layers, hidden_layer_size=dyna_hidden_layer_size, learning_rate=dyna_learning_rate, ) else: self.sampler = InPlacePathSampler( env=env, policy=agent, max_path_length=self.max_path_length, ) # separate replay buffers for # - training RL update # - training encoder update self.replay_buffer = MultiTaskReplayBuffer( self.replay_buffer_size, env, self.train_tasks, ) self.enc_replay_buffer = MultiTaskReplayBuffer( self.replay_buffer_size, env, self.train_tasks, ) self._n_env_steps_total = 0 self._n_train_steps_total = 0 self._n_rollouts_total = 0 self._do_train_time = 0 self._epoch_start_time = None self._algo_start_time = None self._old_table_keys = None self._current_path_builder = PathBuilder() self._exploration_paths = []
def __init__( self, env, exploration_policy: ExplorationPolicy, training_env=None, num_epochs=100, num_steps_per_epoch=10000, num_steps_per_eval=1000, num_updates_per_env_step=1, batch_size=1024, max_path_length=1000, discount=0.99, replay_buffer_size=1000000, reward_scale=1, render=False, save_replay_buffer=False, save_algorithm=False, save_environment=True, eval_sampler=None, eval_policy=None, replay_buffer=None, ): """ Base class for RL Algorithms :param env: Environment used to evaluate. :param exploration_policy: Policy used to explore :param training_env: Environment used by the algorithm. By default, a copy of `env` will be made. :param num_epochs: :param num_steps_per_epoch: :param num_steps_per_eval: :param num_updates_per_env_step: Used by online training mode. :param num_updates_per_epoch: Used by batch training mode. :param batch_size: :param max_path_length: :param discount: :param replay_buffer_size: :param reward_scale: :param render: :param save_replay_buffer: :param save_algorithm: :param save_environment: :param eval_sampler: :param eval_policy: Policy to evaluate with. :param replay_buffer: """ self.training_env = training_env or pickle.loads(pickle.dumps(env)) self.exploration_policy = exploration_policy self.num_epochs = num_epochs self.num_env_steps_per_epoch = num_steps_per_epoch self.num_steps_per_eval = num_steps_per_eval self.num_updates_per_train_call = num_updates_per_env_step self.batch_size = batch_size self.max_path_length = max_path_length self.discount = discount self.replay_buffer_size = replay_buffer_size self.reward_scale = reward_scale self.render = render self.save_replay_buffer = save_replay_buffer self.save_algorithm = save_algorithm self.save_environment = save_environment if eval_sampler is None: if eval_policy is None: eval_policy = exploration_policy eval_sampler = InPlacePathSampler( env=env, policy=eval_policy, max_samples=self.num_steps_per_eval + self.max_path_length, max_path_length=self.max_path_length, ) self.eval_policy = eval_policy self.eval_sampler = eval_sampler self.action_space = env.action_space self.obs_space = env.observation_space self.env = env if replay_buffer is None: replay_buffer = EnvReplayBuffer( self.replay_buffer_size, self.env, ) self.replay_buffer = replay_buffer self._n_env_steps_total = 0 self._n_train_steps_total = 0 self._n_rollouts_total = 0 self._do_train_time = 0 self._epoch_start_time = None self._algo_start_time = None self._old_table_keys = None self._current_path_builder = PathBuilder() self._exploration_paths = []