def test_evaluate(self): """Validate the functionality of the _evaluate method. This is done for the following cases: 1. policy = FeedForwardPolicy 2. policy = GoalConditionedPolicy """ # Set the random seeds. random.seed(0) np.random.seed(0) tf.compat.v1.set_random_seed(0) # =================================================================== # # test case 1 # # =================================================================== # # Create the algorithm object. policy_params = self.init_parameters.copy() policy_params['policy'] = FeedForwardPolicy policy_params['eval_env'] = 'MountainCarContinuous-v0' policy_params['nb_eval_episodes'] = 1 policy_params['verbose'] = 2 policy_params['_init_setup_model'] = True alg = OffPolicyRLAlgorithm(**policy_params) # Run the _evaluate operation. ep_rewards, ep_successes, info = alg._evaluate(alg.eval_env) # Test the output from the operation. self.assertEqual(len(ep_rewards), 1) self.assertEqual(len(ep_successes), 0) self.assertEqual(list(info.keys()), ['initial', 'final', 'average']) # Clear memory. del alg # =================================================================== # # test case 2 # # =================================================================== # # Create the algorithm object. policy_params = self.init_parameters.copy() policy_params['policy'] = GoalConditionedPolicy policy_params['eval_env'] = 'MountainCarContinuous-v0' policy_params['nb_eval_episodes'] = 1 policy_params['verbose'] = 2 policy_params['_init_setup_model'] = True alg = OffPolicyRLAlgorithm(**policy_params) # Run the _evaluate operation. ep_rewards, ep_successes, info = alg._evaluate(alg.eval_env) # Test the output from the operation. self.assertEqual(len(ep_rewards), 1) self.assertEqual(len(ep_successes), 0) self.assertEqual(list(info.keys()), ['initial', 'final', 'average']) # Clear memory. del alg
def test_learn_init(self): """Test the non-loop components of the `learn` method.""" # Create the algorithm object. policy_params = self.init_parameters.copy() policy_params['policy'] = GoalConditionedPolicy policy_params['_init_setup_model'] = True alg = OffPolicyRLAlgorithm(**policy_params) # Run the learn operation for zero timesteps. alg.learn(0, log_dir='results', initial_exploration_steps=0) self.assertEqual(alg.episodes, 0) self.assertEqual(alg.total_steps, 0) self.assertEqual(alg.epoch, 0) self.assertEqual(len(alg.episode_rewards_history), 0) self.assertEqual(alg.epoch_episodes, 0) self.assertEqual(len(alg.epoch_actions), 0) self.assertEqual(len(alg.epoch_q1s), 0) self.assertEqual(len(alg.epoch_q2s), 0) self.assertEqual(len(alg.epoch_actor_losses), 0) self.assertEqual(len(alg.epoch_q1_losses), 0) self.assertEqual(len(alg.epoch_q2_losses), 0) self.assertEqual(len(alg.epoch_episode_rewards), 0) self.assertEqual(len(alg.epoch_episode_steps), 0) shutil.rmtree('results') # Test the seeds. alg.learn(0, log_dir='results', seed=1, initial_exploration_steps=0) self.assertEqual(np.random.sample(), 0.417022004702574) self.assertEqual(random.uniform(0, 1), 0.13436424411240122) shutil.rmtree('results')
def run_exp(env, policy, hp, steps, dir_name, evaluate, seed, eval_interval, log_interval, save_interval, initial_exploration_steps): """Run a single training procedure. Parameters ---------- env : str or gym.Env the training/testing environment policy : type [ hbaselines.base_policies.ActorCriticPolicy ] the policy class to use hp : dict additional algorithm hyper-parameters steps : int total number of training steps dir_name : str the location the results files are meant to be stored evaluate : bool whether to include an evaluation environment seed : int specified the random seed for numpy, tensorflow, and random eval_interval : int number of simulation steps in the training environment before an evaluation is performed log_interval : int the number of training steps before logging training results save_interval : int number of simulation steps in the training environment before the model is saved initial_exploration_steps : int number of timesteps that the policy is run before training to initialize the replay buffer with samples """ eval_env = env if evaluate else None alg = OffPolicyRLAlgorithm( policy=policy, env=env, eval_env=eval_env, **hp ) # perform training alg.learn( total_timesteps=steps, log_dir=dir_name, log_interval=log_interval, eval_interval=eval_interval, save_interval=save_interval, initial_exploration_steps=initial_exploration_steps, seed=seed, )
def test_learn_initial_exploration_steps(self): """Test the initial_exploration_steps parameter in the learn method. This is done for the following cases: 1. initial_exploration_steps= = 0 2. initial_exploration_steps= = 100 """ # =================================================================== # # test case 1 # # =================================================================== # # Create the algorithm object. policy_params = self.init_parameters.copy() policy_params['policy'] = FeedForwardPolicy policy_params['_init_setup_model'] = True alg = OffPolicyRLAlgorithm(**policy_params) # Run the learn operation for zero exploration steps. alg.learn(0, log_dir='results', initial_exploration_steps=0) # Check the size of the replay buffer self.assertEqual(len(alg.policy_tf.replay_buffer), 1) # Clear memory. del alg shutil.rmtree('results') # =================================================================== # # test case 2 # # =================================================================== # # Create the algorithm object. policy_params = self.init_parameters.copy() policy_params['policy'] = FeedForwardPolicy policy_params['_init_setup_model'] = True alg = OffPolicyRLAlgorithm(**policy_params) # Run the learn operation for zero exploration steps. alg.learn(0, log_dir='results', initial_exploration_steps=100) # Check the size of the replay buffer self.assertEqual(len(alg.policy_tf.replay_buffer), 100) # Clear memory. del alg shutil.rmtree('results')
def test_fingerprints(self): """Validate the functionality of the fingerprints. When the fingerprint functionality is turned on, the observation within the algorithm (stored under self.obs) should always include the fingerprint element. Policy-specific features of the fingerprint implementation are also tested here. This feature should add a fingerprint dimension to the observation spaces, but NOT the context space of the lower-level or the action space of the higher-level. The intrinsic reward function should also be ignoring the fingerprint elements during its computation. The fingerprint elements are passed by the algorithm, and tested under test_algorithm.py """ # Create the algorithm. policy_params = self.init_parameters.copy() policy_params['policy'] = GoalConditionedPolicy policy_params['nb_rollout_steps'] = 1 policy_params['policy_kwargs'] = {'use_fingerprints': True} alg = OffPolicyRLAlgorithm(**policy_params) # Test the observation spaces of the policies, as well as the context # space of the lower-level policy and action space of the higher-level # policy. self.assertTupleEqual(alg.policy_tf.policy[0].ob_space.shape, (4,)) self.assertTupleEqual(alg.policy_tf.policy[0].ac_space.shape, (2,)) self.assertTupleEqual(alg.policy_tf.policy[-1].ob_space.shape, (4,)) self.assertTupleEqual(alg.policy_tf.policy[-1].co_space.shape, (2,)) # Test intrinsic_reward method within the policy. self.assertAlmostEqual( alg.policy_tf.intrinsic_reward_fn( states=np.array([1, 2, 3]), goals=np.array([0, 0]), next_states=np.array([1, 2, 3])), -np.sqrt(1**2 + 2**2) ) # Validate that observations include the fingerprints elements upon # initializing the `learn` procedure and during a step in the # `_collect_samples` method. alg.learn(1, log_dir='results', log_interval=1, initial_exploration_steps=0) self.assertEqual(len(alg.obs[0]), alg.ob_space.shape[0]) np.testing.assert_almost_equal( alg.obs[0][-alg.policy_tf.fingerprint_dim[0]:], np.array([0, 5])) # Validate that observations include the fingerprints elements during # a reset in the `_collect_samples` method. alg.learn(500, log_dir='results', log_interval=500, initial_exploration_steps=0) self.assertEqual(len(alg.obs[0]), alg.ob_space.shape[0]) np.testing.assert_almost_equal( alg.obs[0][-alg.policy_tf.fingerprint_dim[0]:], np.array([4.99, 0.01])) # Delete generated files. shutil.rmtree('results')
def test_log_eval(self): # Create the algorithm object. policy_params = self.init_parameters.copy() policy_params['policy'] = GoalConditionedPolicy policy_params['_init_setup_model'] = False alg = OffPolicyRLAlgorithm(**policy_params) # test for one evaluation environment rewards = [0, 1, 2] successes = [True, False, False] info = {"test": 5} alg._log_eval(file_path="test_eval.csv", start_time=0, rewards=rewards, successes=successes, info=info) # check that the file was generated self.assertTrue(os.path.exists('test_eval_0.csv')) # import the stored data reader = csv.DictReader(open('test_eval_0.csv', 'r')) results = {"successes": [], "rewards": [], "test": []} for line in reader: results["successes"].append(float(line["success_rate"])) results["rewards"].append(float(line["average_return"])) results["test"].append(float(line["test"])) # test that the data matches expected values self.assertListEqual(results["rewards"], [1]) self.assertListEqual(results["successes"], [1 / 3]) self.assertListEqual(results["test"], [5]) # Delete generated files. os.remove('test_eval_0.csv') # test for one evaluation environment with no successes successes = [] alg._log_eval(file_path="test_eval.csv", start_time=0, rewards=rewards, successes=successes, info=info) # check that the file was generated self.assertTrue(os.path.exists('test_eval_0.csv')) # import the stored data reader = csv.DictReader(open('test_eval_0.csv', 'r')) results = {"successes": []} for line in reader: results["successes"].append(float(line["success_rate"])) # test that the successes are all zero self.assertListEqual(results["successes"], [0]) # Delete generated files. os.remove('test_eval_0.csv')
def test_setup_model_feedforward(self): # Create the algorithm object. policy_params = self.init_parameters.copy() policy_params['policy'] = FeedForwardPolicy policy_params['_init_setup_model'] = True alg = OffPolicyRLAlgorithm(**policy_params) # check the policy_kwargs term policy_kwargs = FEEDFORWARD_PARAMS.copy() policy_kwargs.update(TD3_PARAMS) policy_kwargs['verbose'] = self.init_parameters['verbose'] self.assertDictEqual(alg.policy_kwargs, policy_kwargs) with alg.graph.as_default(): expected_vars = sorted([var.name for var in get_trainable_vars()]) # Check that all trainable variables have been created in the # TensorFlow graph. self.assertListEqual( expected_vars, ['model/pi/fc0/bias:0', 'model/pi/fc0/kernel:0', 'model/pi/fc1/bias:0', 'model/pi/fc1/kernel:0', 'model/pi/output/bias:0', 'model/pi/output/kernel:0', 'model/qf_0/fc0/bias:0', 'model/qf_0/fc0/kernel:0', 'model/qf_0/fc1/bias:0', 'model/qf_0/fc1/kernel:0', 'model/qf_0/qf_output/bias:0', 'model/qf_0/qf_output/kernel:0', 'model/qf_1/fc0/bias:0', 'model/qf_1/fc0/kernel:0', 'model/qf_1/fc1/bias:0', 'model/qf_1/fc1/kernel:0', 'model/qf_1/qf_output/bias:0', 'model/qf_1/qf_output/kernel:0', 'target/pi/fc0/bias:0', 'target/pi/fc0/kernel:0', 'target/pi/fc1/bias:0', 'target/pi/fc1/kernel:0', 'target/pi/output/bias:0', 'target/pi/output/kernel:0', 'target/qf_0/fc0/bias:0', 'target/qf_0/fc0/kernel:0', 'target/qf_0/fc1/bias:0', 'target/qf_0/fc1/kernel:0', 'target/qf_0/qf_output/bias:0', 'target/qf_0/qf_output/kernel:0', 'target/qf_1/fc0/bias:0', 'target/qf_1/fc0/kernel:0', 'target/qf_1/fc1/bias:0', 'target/qf_1/fc1/kernel:0', 'target/qf_1/qf_output/bias:0', 'target/qf_1/qf_output/kernel:0'] )
def run_exp(env, hp, steps, dir_name, evaluate, seed, eval_interval, log_interval, save_interval): """Run a single training procedure. Parameters ---------- env : str or gym.Env the training/testing environment hp : dict additional algorithm hyper-parameters steps : int total number of training steps dir_name : str the location the results files are meant to be stored evaluate : bool whether to include an evaluation environment seed : int specified the random seed for numpy, tensorflow, and random eval_interval : int number of simulation steps in the training environment before an evaluation is performed log_interval : int the number of training steps before logging training results save_interval : int number of simulation steps in the training environment before the model is saved """ eval_env = env if evaluate else None alg = OffPolicyRLAlgorithm( policy=FeedForwardPolicy, env=env, eval_env=eval_env, **hp ) # perform training alg.learn( total_timesteps=steps, log_dir=dir_name, log_interval=log_interval, eval_interval=eval_interval, save_interval=save_interval, seed=seed, )
def test_init(self): """Ensure that the parameters at init are as expected.""" # Create the algorithm object. policy_params = self.init_parameters.copy() policy_params['_init_setup_model'] = False alg = OffPolicyRLAlgorithm(**policy_params) # Test the attribute values. self.assertEqual(alg.policy, self.init_parameters['policy']) self.assertEqual(alg.eval_env, self.init_parameters['eval_env']) self.assertEqual(alg.nb_train_steps, self.init_parameters['nb_train_steps']) self.assertEqual(alg.nb_rollout_steps, self.init_parameters['nb_rollout_steps']) self.assertEqual(alg.nb_eval_episodes, self.init_parameters['nb_eval_episodes']) self.assertEqual(alg.reward_scale, self.init_parameters['reward_scale']) self.assertEqual(alg.render, self.init_parameters['render']) self.assertEqual(alg.render_eval, self.init_parameters['render_eval']) self.assertEqual(alg.verbose, self.init_parameters['verbose'])
def test_setup_model_goal_conditioned(self): # Create the algorithm object. policy_params = self.init_parameters.copy() policy_params['policy'] = GoalConditionedPolicy policy_params['_init_setup_model'] = True alg = OffPolicyRLAlgorithm(**policy_params) # check the policy_kwargs term policy_kwargs = GOAL_CONDITIONED_PARAMS.copy() policy_kwargs.update(TD3_PARAMS) policy_kwargs['verbose'] = self.init_parameters['verbose'] policy_kwargs['env_name'] = self.init_parameters['env'] policy_kwargs['num_envs'] = self.init_parameters['num_envs'] self.assertDictEqual(alg.policy_kwargs, policy_kwargs) with alg.graph.as_default(): expected_vars = sorted([var.name for var in get_trainable_vars()]) # Check that all trainable variables have been created in the # TensorFlow graph. self.assertListEqual( expected_vars, ['level_0/model/pi/fc0/bias:0', 'level_0/model/pi/fc0/kernel:0', 'level_0/model/pi/fc1/bias:0', 'level_0/model/pi/fc1/kernel:0', 'level_0/model/pi/output/bias:0', 'level_0/model/pi/output/kernel:0', 'level_0/model/qf_0/fc0/bias:0', 'level_0/model/qf_0/fc0/kernel:0', 'level_0/model/qf_0/fc1/bias:0', 'level_0/model/qf_0/fc1/kernel:0', 'level_0/model/qf_0/qf_output/bias:0', 'level_0/model/qf_0/qf_output/kernel:0', 'level_0/model/qf_1/fc0/bias:0', 'level_0/model/qf_1/fc0/kernel:0', 'level_0/model/qf_1/fc1/bias:0', 'level_0/model/qf_1/fc1/kernel:0', 'level_0/model/qf_1/qf_output/bias:0', 'level_0/model/qf_1/qf_output/kernel:0', 'level_0/target/pi/fc0/bias:0', 'level_0/target/pi/fc0/kernel:0', 'level_0/target/pi/fc1/bias:0', 'level_0/target/pi/fc1/kernel:0', 'level_0/target/pi/output/bias:0', 'level_0/target/pi/output/kernel:0', 'level_0/target/qf_0/fc0/bias:0', 'level_0/target/qf_0/fc0/kernel:0', 'level_0/target/qf_0/fc1/bias:0', 'level_0/target/qf_0/fc1/kernel:0', 'level_0/target/qf_0/qf_output/bias:0', 'level_0/target/qf_0/qf_output/kernel:0', 'level_0/target/qf_1/fc0/bias:0', 'level_0/target/qf_1/fc0/kernel:0', 'level_0/target/qf_1/fc1/bias:0', 'level_0/target/qf_1/fc1/kernel:0', 'level_0/target/qf_1/qf_output/bias:0', 'level_0/target/qf_1/qf_output/kernel:0', 'level_1/model/pi/fc0/bias:0', 'level_1/model/pi/fc0/kernel:0', 'level_1/model/pi/fc1/bias:0', 'level_1/model/pi/fc1/kernel:0', 'level_1/model/pi/output/bias:0', 'level_1/model/pi/output/kernel:0', 'level_1/model/qf_0/fc0/bias:0', 'level_1/model/qf_0/fc0/kernel:0', 'level_1/model/qf_0/fc1/bias:0', 'level_1/model/qf_0/fc1/kernel:0', 'level_1/model/qf_0/qf_output/bias:0', 'level_1/model/qf_0/qf_output/kernel:0', 'level_1/model/qf_1/fc0/bias:0', 'level_1/model/qf_1/fc0/kernel:0', 'level_1/model/qf_1/fc1/bias:0', 'level_1/model/qf_1/fc1/kernel:0', 'level_1/model/qf_1/qf_output/bias:0', 'level_1/model/qf_1/qf_output/kernel:0', 'level_1/target/pi/fc0/bias:0', 'level_1/target/pi/fc0/kernel:0', 'level_1/target/pi/fc1/bias:0', 'level_1/target/pi/fc1/kernel:0', 'level_1/target/pi/output/bias:0', 'level_1/target/pi/output/kernel:0', 'level_1/target/qf_0/fc0/bias:0', 'level_1/target/qf_0/fc0/kernel:0', 'level_1/target/qf_0/fc1/bias:0', 'level_1/target/qf_0/fc1/kernel:0', 'level_1/target/qf_0/qf_output/bias:0', 'level_1/target/qf_0/qf_output/kernel:0', 'level_1/target/qf_1/fc0/bias:0', 'level_1/target/qf_1/fc0/kernel:0', 'level_1/target/qf_1/fc1/bias:0', 'level_1/target/qf_1/fc1/kernel:0', 'level_1/target/qf_1/qf_output/bias:0', 'level_1/target/qf_1/qf_output/kernel:0'] )
def train_h_baselines(env_name, args, multiagent): """Train policies using SAC and TD3 with h-baselines.""" from hbaselines.algorithms import OffPolicyRLAlgorithm from hbaselines.utils.train import parse_options, get_hyperparameters # Get the command-line arguments that are relevant here args = parse_options(description="", example_usage="", args=args) # the base directory that the logged data will be stored in base_dir = "training_data" for i in range(args.n_training): # value of the next seed seed = args.seed + i # The time when the current experiment started. now = strftime("%Y-%m-%d-%H:%M:%S") # Create a save directory folder (if it doesn't exist). dir_name = os.path.join(base_dir, '{}/{}'.format(args.env_name, now)) ensure_dir(dir_name) # Get the policy class. if args.alg == "TD3": if multiagent: from hbaselines.multi_fcnet.td3 import MultiFeedForwardPolicy policy = MultiFeedForwardPolicy else: from hbaselines.fcnet.td3 import FeedForwardPolicy policy = FeedForwardPolicy elif args.alg == "SAC": if multiagent: from hbaselines.multi_fcnet.sac import MultiFeedForwardPolicy policy = MultiFeedForwardPolicy else: from hbaselines.fcnet.sac import FeedForwardPolicy policy = FeedForwardPolicy else: raise ValueError("Unknown algorithm: {}".format(args.alg)) # Get the hyperparameters. hp = get_hyperparameters(args, policy) # Add the seed for logging purposes. params_with_extra = hp.copy() params_with_extra['seed'] = seed params_with_extra['env_name'] = args.env_name params_with_extra['policy_name'] = policy.__name__ params_with_extra['algorithm'] = args.alg params_with_extra['date/time'] = now # Add the hyperparameters to the folder. with open(os.path.join(dir_name, 'hyperparameters.json'), 'w') as f: json.dump(params_with_extra, f, sort_keys=True, indent=4) # Create the algorithm object. alg = OffPolicyRLAlgorithm( policy=policy, env="flow:{}".format(env_name), eval_env="flow:{}".format(env_name) if args.evaluate else None, **hp) # Perform training. alg.learn( total_steps=args.total_steps, log_dir=dir_name, log_interval=args.log_interval, eval_interval=args.eval_interval, save_interval=args.save_interval, initial_exploration_steps=args.initial_exploration_steps, seed=seed, )
def main(args): """Execute multiple training operations.""" flags = parse_options(args) # get the hyperparameters env_name, policy, hp, seed = get_hyperparameters_from_dir(flags.dir_name) hp['render'] = not flags.no_render # to visualize the policy # create the algorithm object. We will be using the eval environment in # this object to perform the rollout. alg = OffPolicyRLAlgorithm(policy=policy, env=env_name, eval_env=env_name, **hp) # setup the seed value random.seed(seed) np.random.seed(seed) tf.compat.v1.set_random_seed(seed) # get the checkpoint number if flags.ckpt_num is None: filenames = os.listdir(os.path.join(flags.dir_name, "checkpoints")) metafiles = [f[:-5] for f in filenames if f[-5:] == ".meta"] metanum = [int(f.split("-")[-1]) for f in metafiles] ckpt_num = max(metanum) else: ckpt_num = flags.ckpt_num # location to the checkpoint ckpt = os.path.join(flags.dir_name, "checkpoints/itr-{}".format(ckpt_num)) # restore the previous checkpoint alg.saver = tf.compat.v1.train.Saver(alg.trainable_vars) alg.load(ckpt) # some variables that will be needed when replaying the rollout policy = alg.policy_tf env = alg.eval_env # Perform the evaluation procedure. episdoe_rewards = [] for episode_num in range(flags.num_rollouts): # Run a rollout. obs = env.reset() total_reward = 0 while True: context = [env.current_context] \ if hasattr(env, "current_context") else None action = policy.get_action( np.asarray([obs]), context=context, apply_noise=False, random_actions=False, ) obs, reward, done, _ = env.step(action) if not flags.no_render: env.render() total_reward += reward if done: break # Print total returns from a given episode. episdoe_rewards.append(total_reward) print("Round {}, return: {}".format(episode_num, total_reward)) # Print total statistics. print("Average, std return: {}, {}".format(np.mean(episdoe_rewards), np.std(episdoe_rewards)))
def main(args): """Execute multiple training operations.""" flags = parse_options(args) # get the hyperparameters env_name, policy, hp, seed = get_hyperparameters_from_dir(flags.dir_name) hp['render'] = not flags.no_render # to visualize the policy # create the algorithm object. We will be using the eval environment in # this object to perform the rollout. alg = OffPolicyRLAlgorithm(policy=policy, env=env_name, **hp) # setup the seed value if not flags.random_seed: random.seed(seed) np.random.seed(seed) tf.compat.v1.set_random_seed(seed) # get the checkpoint number if flags.ckpt_num is None: filenames = os.listdir(os.path.join(flags.dir_name, "checkpoints")) metafiles = [f[:-5] for f in filenames if f[-5:] == ".meta"] metanum = [int(f.split("-")[-1]) for f in metafiles] ckpt_num = max(metanum) else: ckpt_num = flags.ckpt_num # location to the checkpoint ckpt = os.path.join(flags.dir_name, "checkpoints/itr-{}".format(ckpt_num)) # restore the previous checkpoint alg.saver = tf.compat.v1.train.Saver(alg.trainable_vars) alg.load(ckpt) # some variables that will be needed when replaying the rollout policy = alg.policy_tf env = alg.sampler.env # Perform the evaluation procedure. episdoe_rewards = [] # Add an emission path to Flow environments. if env_name in FLOW_ENV_NAMES: sim_params = deepcopy(env.wrapped_env.sim_params) sim_params.emission_path = "./flow_results" env.wrapped_env.restart_simulation(sim_params, render=not flags.no_render) for episode_num in range(flags.num_rollouts): # Run a rollout. obs = env.reset() total_reward = 0 while True: context = [env.current_context] \ if hasattr(env, "current_context") else None action = policy.get_action( np.asarray([obs]), context=context, apply_noise=False, random_actions=False, ) obs, reward, done, _ = env.step(action[0]) if not flags.no_render: env.render() total_reward += reward if done: break # Print total returns from a given episode. episdoe_rewards.append(total_reward) print("Round {}, return: {}".format(episode_num, total_reward)) # Print total statistics. print("Average, std return: {}, {}".format(np.mean(episdoe_rewards), np.std(episdoe_rewards))) if env_name in FLOW_ENV_NAMES: # wait a short period of time to ensure the xml file is readable time.sleep(0.1) # collect the location of the emission file dir_path = env.wrapped_env.sim_params.emission_path emission_filename = "{0}-emission.xml".format( env.wrapped_env.network.name) emission_path = os.path.join(dir_path, emission_filename) # convert the emission file into a csv emission_to_csv(emission_path) # Delete the .xml version of the emission file. os.remove(emission_path)
def main(args): """Execute multiple training operations.""" flags = parse_options(args) # Run assertions. assert not (flags.no_render and flags.save_video), \ "If saving the rendering, no_render cannot be set to True." # get the hyperparameters env_name, policy, hp, seed = get_hyperparameters_from_dir(flags.dir_name) hp['num_envs'] = 1 hp['render_eval'] = not flags.no_render # to visualize the policy # create the algorithm object. We will be using the eval environment in # this object to perform the rollout. alg = OffPolicyRLAlgorithm(policy=policy, env=env_name, eval_env=env_name, **hp) # setup the seed value if not flags.random_seed: random.seed(seed) np.random.seed(seed) tf.compat.v1.set_random_seed(seed) # get the checkpoint number if flags.ckpt_num is None: filenames = os.listdir(os.path.join(flags.dir_name, "checkpoints")) metafiles = [f[:-5] for f in filenames if f[-5:] == ".meta"] metanum = [int(f.split("-")[-1]) for f in metafiles] ckpt_num = max(metanum) else: ckpt_num = flags.ckpt_num # location to the checkpoint ckpt = os.path.join(flags.dir_name, "checkpoints/itr-{}".format(ckpt_num)) # restore the previous checkpoint alg.saver = tf.compat.v1.train.Saver(alg.trainable_vars) alg.load(ckpt) # some variables that will be needed when replaying the rollout policy = alg.policy_tf env = alg.eval_env # Perform the evaluation procedure. episode_rewards = [] # Add an emission path to Flow environments. if env_name in FLOW_ENV_NAMES: sim_params = deepcopy(env.wrapped_env.sim_params) sim_params.emission_path = "./flow_results" env.wrapped_env.restart_simulation(sim_params, render=not flags.no_render) if not isinstance(env, list): env_list = [env] else: env_list = env for env_num, env in enumerate(env_list): for episode_num in range(flags.num_rollouts): if not flags.no_render and env_name not in FLOW_ENV_NAMES: out = FFmpegWriter("{}_{}_{}.mp4".format( flags.video, env_num, episode_num)) else: out = None obs, total_reward = env.reset(), 0 while True: context = [env.current_context] \ if hasattr(env, "current_context") else None action = policy.get_action( obs=np.asarray([obs]), context=context, apply_noise=False, random_actions=False, ) # Visualize the sub-goals of the hierarchical policy. if hasattr(policy, "_meta_action") \ and policy._meta_action is not None \ and hasattr(env, "set_goal"): goal = policy._meta_action[0][0] + (obs[ policy.goal_indices] if policy.relative_goals else 0) env.set_goal(goal) new_obs, reward, done, _ = env.step(action[0]) if not flags.no_render: if flags.save_video: if alg.env_name == "AntGather": out.writeFrame(env.render(mode='rgb_array')) else: out.writeFrame( env.render(mode='rgb_array', height=1024, width=1024)) else: env.render() total_reward += reward if done: break policy.store_transition( obs0=obs, context0=context[0] if context is not None else None, action=action[0], reward=reward, obs1=new_obs, context1=context[0] if context is not None else None, done=done, is_final_step=done, evaluate=True, ) obs = new_obs # Print total returns from a given episode. episode_rewards.append(total_reward) print("Round {}, return: {}".format(episode_num, total_reward)) # Save the video. if not flags.no_render and env_name not in FLOW_ENV_NAMES \ and flags.save_video: out.close() # Print total statistics. print("Average, std return: {}, {}".format(np.mean(episode_rewards), np.std(episode_rewards)))
def main(args): """Execute multiple training operations.""" flags = parse_options(args) data = { 'name': [], 'step': [], 'distance': [], } for dir_name, name in zip(flags.dir_name, flags.name): # get the hyperparameters env_name, policy, hp, seed = get_hyperparameters_from_dir(dir_name) print(hp.keys()) del hp['algorithm'] del hp['date/time'] # create the algorithm object. We will be using the eval environment in # this object to perform the rollout. alg = OffPolicyRLAlgorithm( policy=policy, env=env_name, eval_env=env_name, **hp) # setup the seed value random.seed(seed) np.random.seed(seed) tf.compat.v1.set_random_seed(seed) filenames = os.listdir(os.path.join(dir_name, "checkpoints")) metafiles = [f[:-5] for f in filenames if f[-5:] == ".meta"] metanum = list(sorted([int(f.split("-")[-1]) for f in metafiles]))[:-1] # get the checkpoint number ckpt_num = max(metanum) # location to the checkpoint ckpt = os.path.join(dir_name, "checkpoints/itr-{}".format(ckpt_num)) # restore the previous checkpoint alg.saver = tf.compat.v1.train.Saver(alg.trainable_vars) alg.load(ckpt) # some variables that will be needed when replaying the rollout policy = alg.policy_tf batches = [] for b in range(flags.num_batches): worker_obs0 = policy.replay_buffer.sample(with_additional=False)[5] batches.append(worker_obs0) for ckpt_num_one, ckpt_num_two in zip(metanum[1:], metanum[:-1]): # get the checkpoint number ckpt_num = ckpt_num_one # location to the checkpoint ckpt = os.path.join(dir_name, "checkpoints/itr-{}".format(ckpt_num)) # restore the previous checkpoint alg.load(ckpt) # some variables that will be needed when replaying the rollout policy = alg.policy_tf mean_one = [] for b in batches: a = policy.policy[-1].get_action(b, None, False, False) mean_one.append(a) # get the checkpoint number ckpt_num = ckpt_num_two # location to the checkpoint ckpt = os.path.join(dir_name, "checkpoints/itr-{}".format(ckpt_num)) # restore the previous checkpoint alg.load(ckpt) # some variables that will be needed when replaying the rollout policy = alg.policy_tf mean_two = [] for b in batches: a = policy.policy[-1].get_action(b, None, False, False) mean_two.append(a) # compute a distance metric between the policies mean_one = np.concatenate(mean_one, axis=0) mean_two = np.concatenate(mean_two, axis=0) kl = np.sum((mean_one - mean_two) ** 2, axis=1).mean() print("{},{},{},{}".format(name, ckpt_num_one, ckpt_num_two, kl)) data['name'].append(name) data['step'].append(ckpt_num_one) data['distance'].append(kl) df = pd.DataFrame(data, columns=['name', 'step', 'distance']) plt.title("Manager MDP Non-Stationarity") ax = sns.lineplot(x='step', y='distance', hue='name', data=df) plt.savefig('ns.png')