def test_cql_compilation(self): """Test whether a CQLTrainer can be built with all frameworks.""" # Learns from a historic-data file. # To generate this data, first run: # $ ./train.py --run=SAC --env=Pendulum-v0 \ # --stop='{"timesteps_total": 50000}' \ # --config='{"output": "/tmp/out"}' rllib_dir = Path(__file__).parent.parent.parent.parent print("rllib dir={}".format(rllib_dir)) data_file = os.path.join(rllib_dir, "tests/data/pendulum/small.json") print("data_file={} exists={}".format(data_file, os.path.isfile(data_file))) config = cql.CQL_DEFAULT_CONFIG.copy() config["env"] = "Pendulum-v0" config["input"] = [data_file] # In the files, we use here for testing, actions have already # been normalized. # This is usually the case when the file was generated by another # RLlib algorithm (e.g. PPO or SAC). config["actions_in_input_normalized"] = False config["clip_actions"] = True config["train_batch_size"] = 2000 config["num_workers"] = 0 # Run locally. config["twin_q"] = True config["learning_starts"] = 0 config["bc_iters"] = 2 # 2 BC iters, 2 CQL iters. config["rollout_fragment_length"] = 1 # Switch on off-policy evaluation. config["input_evaluation"] = ["is"] config["evaluation_interval"] = 2 config["evaluation_num_episodes"] = 10 config["evaluation_config"]["input"] = "sampler" config["evaluation_parallel_to_training"] = False config["evaluation_num_workers"] = 2 num_iterations = 4 # Test for tf/torch frameworks. for fw in framework_iterator(config): trainer = cql.CQLTrainer(config=config) for i in range(num_iterations): results = trainer.train() check_train_results(results) print(results) eval_results = results.get("evaluation") if eval_results: print(f"iter={trainer.iteration} " f"R={eval_results['episode_reward_mean']}") check_compute_single_action(trainer) # Get policy and model. pol = trainer.get_policy() cql_model = pol.model if fw == "tf": pol.get_session().__enter__() # Example on how to do evaluation on the trained Trainer # using the data from CQL's global replay buffer. # Get a sample (MultiAgentBatch -> SampleBatch). from ray.rllib.agents.cql.cql import replay_buffer batch = replay_buffer.replay().policy_batches["default_policy"] if fw == "torch": obs = torch.from_numpy(batch["obs"]) else: obs = batch["obs"] batch["actions"] = batch["actions"].astype(np.float32) # Pass the observations through our model to get the # features, which then to pass through the Q-head. model_out, _ = cql_model({"obs": obs}) # The estimated Q-values from the (historic) actions in the batch. if fw == "torch": q_values_old = cql_model.get_q_values( model_out, torch.from_numpy(batch["actions"])) else: q_values_old = cql_model.get_q_values( tf.convert_to_tensor(model_out), batch["actions"]) # The estimated Q-values for the new actions computed # by our trainer policy. actions_new = pol.compute_actions_from_input_dict({"obs": obs})[0] if fw == "torch": q_values_new = cql_model.get_q_values( model_out, torch.from_numpy(actions_new)) else: q_values_new = cql_model.get_q_values(model_out, actions_new) if fw == "tf": q_values_old, q_values_new = pol.get_session().run( [q_values_old, q_values_new]) print(f"Q-val batch={q_values_old}") print(f"Q-val policy={q_values_new}") if fw == "tf": pol.get_session().__exit__(None, None, None) trainer.stop()
def test_cql_compilation(self): """Test whether a CQLTrainer can be built with all frameworks.""" # Learns from a historic-data file. # To generate this data, first run: # $ ./train.py --run=SAC --env=Pendulum-v0 \ # --stop='{"timesteps_total": 50000}' \ # --config='{"output": "/tmp/out"}' rllib_dir = Path(__file__).parent.parent.parent.parent print("rllib dir={}".format(rllib_dir)) data_file = os.path.join(rllib_dir, "tests/data/pendulum/small.json") print("data_file={} exists={}".format(data_file, os.path.isfile(data_file))) config = cql.CQL_DEFAULT_CONFIG.copy() config["env"] = "Pendulum-v0" config["input"] = [data_file] config["num_workers"] = 0 # Run locally. config["twin_q"] = True config["clip_actions"] = False config["normalize_actions"] = True config["learning_starts"] = 0 config["rollout_fragment_length"] = 1 config["train_batch_size"] = 10 # Switch on off-policy evaluation. config["input_evaluation"] = ["is"] num_iterations = 2 # Test for tf framework (torch not implemented yet). for _ in framework_iterator(config, frameworks=("torch")): trainer = cql.CQLTrainer(config=config) for i in range(num_iterations): print(trainer.train()) check_compute_single_action(trainer) # Get policy, model, and replay-buffer. pol = trainer.get_policy() cql_model = pol.model from ray.rllib.agents.cql.cql import replay_buffer # Example on how to do evaluation on the trained Trainer # using the data from our buffer. # Get a sample (MultiAgentBatch -> SampleBatch). batch = replay_buffer.replay().policy_batches["default_policy"] obs = torch.from_numpy(batch["obs"]) # Pass the observations through our model to get the # features, which then to pass through the Q-head. model_out, _ = cql_model({"obs": obs}) # The estimated Q-values from the (historic) actions in the batch. q_values_old = cql_model.get_q_values( model_out, torch.from_numpy(batch["actions"])) # The estimated Q-values for the new actions computed # by our trainer policy. actions_new = pol.compute_actions_from_input_dict({"obs": obs})[0] q_values_new = cql_model.get_q_values( model_out, torch.from_numpy(actions_new)) print(f"Q-val batch={q_values_old}") print(f"Q-val policy={q_values_new}") trainer.stop()
# If you would like to query CQL's learnt Q-function for arbitrary # (cont.) actions, do the following: obs_batch = torch.from_numpy(np.random.random(size=(5, 3))) action_batch = torch.from_numpy(np.random.random(size=(5, 1))) q_values = cql_model.get_q_values(obs_batch, action_batch) # If you are using the "twin_q", there'll be 2 Q-networks and # we usually consider the min of the 2 outputs, like so: twin_q_values = cql_model.get_twin_q_values(obs_batch, action_batch) final_q_values = torch.min(q_values, twin_q_values) print(final_q_values) # Example on how to do evaluation on the trained Trainer # using the data from our buffer. # Get a sample (MultiAgentBatch -> SampleBatch). batch = replay_buffer.replay().policy_batches["default_policy"] obs = torch.from_numpy(batch["obs"]) # Pass the observations through our model to get the # features, which then to pass through the Q-head. model_out, _ = cql_model({"obs": obs}) # The estimated Q-values from the (historic) actions in the batch. q_values_old = cql_model.get_q_values(model_out, torch.from_numpy(batch["actions"])) # The estimated Q-values for the new actions computed # by our trainer policy. actions_new = pol.compute_actions_from_input_dict({"obs": obs})[0] q_values_new = cql_model.get_q_values(model_out, torch.from_numpy(actions_new)) print(f"Q-val batch={q_values_old}") print(f"Q-val policy={q_values_new}")