def test_default_action_discrete_adapter(self):
ADAPTER_TYPE = adapters.AdapterType.DefaultActionDiscrete
adapter = adapters.adapter_from_type(ADAPTER_TYPE)
interface = adapters.required_interface_from_types(ADAPTER_TYPE)
space = adapters.space_from_type(ADAPTER_TYPE)
AVAILABLE_ACTIONS = [
"keep_lane",
"slow_down",
"change_lane_left",
"change_lane_right",
]
agent, environment = prepare_test_agent_and_environment(
required_interface=interface,
action_adapter=adapter,
)
action_sequence, _, _, _ = run_experiment(agent, environment)
for action in action_sequence:
self.assertIsInstance(action, str)
self.assertIn(action, AVAILABLE_ACTIONS)
self.assertEqual(space.dtype, type(action))
self.assertEqual(space.shape, ())
self.assertTrue(space.contains(action))
def test_default_action_continuous_adapter(self):
ADAPTER_TYPE = adapters.AdapterType.DefaultActionContinuous
adapter = adapters.adapter_from_type(ADAPTER_TYPE)
interface = adapters.required_interface_from_types(ADAPTER_TYPE)
space = adapters.space_from_type(ADAPTER_TYPE)
agent, environment = prepare_test_agent_and_environment(
required_interface=interface,
action_adapter=adapter,
)
action_sequence, _, _, _ = run_experiment(agent, environment)
for action in action_sequence:
self.assertIsInstance(action, np.ndarray)
self.assertEqual(action.dtype, "float32")
self.assertEqual(action.shape, (3, ))
self.assertGreaterEqual(action[0], 0.0)
self.assertLessEqual(action[0], 1.0)
self.assertGreaterEqual(action[1], 0.0)
self.assertLessEqual(action[1], 1.0)
self.assertGreaterEqual(action[2], -1.0)
self.assertLessEqual(action[2], 1.0)
self.assertEqual(space.dtype, action.dtype)
self.assertEqual(space.shape, action.shape)
self.assertTrue(space.contains(action))
def test_default_info_adapter(self):
ADAPTER_TYPE = adapters.AdapterType.DefaultInfo
adapter = adapters.adapter_from_type(ADAPTER_TYPE)
interface = adapters.required_interface_from_types(ADAPTER_TYPE)
agent, environment = prepare_test_agent_and_environment(
required_interface=interface,
info_adapter=adapter,
)
_, infos_sequence, _, _ = run_experiment(agent,
environment,
max_steps=1)
infos = infos_sequence[0]
self.assertIsInstance(infos, dict)
self.assertIn(AGENT_ID, infos)
self.assertIsInstance(infos[AGENT_ID], dict)
self.assertIn("score", infos[AGENT_ID])
self.assertIsInstance(infos[AGENT_ID]["score"], float)
self.assertIn("env_obs", infos[AGENT_ID])
self.assertIsInstance(infos[AGENT_ID]["env_obs"], Observation)
self.assertIn("logs", infos[AGENT_ID])
self.assertIsInstance(infos[AGENT_ID]["logs"], dict)
self.assertIn("position", infos[AGENT_ID]["logs"])
self.assertIsInstance(infos[AGENT_ID]["logs"]["position"], np.ndarray)
self.assertEqual(infos[AGENT_ID]["logs"]["position"].shape, (3, ))
self.assertIn("speed", infos[AGENT_ID]["logs"])
self.assertIsInstance(infos[AGENT_ID]["logs"]["speed"], float)
self.assertIn("steering", infos[AGENT_ID]["logs"])
self.assertIsInstance(infos[AGENT_ID]["logs"]["steering"], float)
self.assertIn("heading", infos[AGENT_ID]["logs"])
self.assertIsInstance(infos[AGENT_ID]["logs"]["heading"], Heading)
self.assertIn("dist_center", infos[AGENT_ID]["logs"])
self.assertIsInstance(infos[AGENT_ID]["logs"]["dist_center"], float)
self.assertIn("start", infos[AGENT_ID]["logs"])
self.assertIsInstance(infos[AGENT_ID]["logs"]["start"], Start)
self.assertIn("goal", infos[AGENT_ID]["logs"])
self.assertIsInstance(infos[AGENT_ID]["logs"]["goal"], PositionalGoal)
self.assertIn("closest_wp", infos[AGENT_ID]["logs"])
self.assertIsInstance(infos[AGENT_ID]["logs"]["closest_wp"], Waypoint)
self.assertIn("events", infos[AGENT_ID]["logs"])
self.assertIsInstance(infos[AGENT_ID]["logs"]["events"], Events)
self.assertIn("ego_num_violations", infos[AGENT_ID]["logs"])
self.assertIsInstance(infos[AGENT_ID]["logs"]["ego_num_violations"],
int)
self.assertIn("social_num_violations", infos[AGENT_ID]["logs"])
self.assertIsInstance(infos[AGENT_ID]["logs"]["social_num_violations"],
int)
self.assertIn("goal_dist", infos[AGENT_ID]["logs"])
self.assertIsInstance(infos[AGENT_ID]["logs"]["goal_dist"], float)
self.assertIn("linear_jerk", infos[AGENT_ID]["logs"])
self.assertIsInstance(infos[AGENT_ID]["logs"]["linear_jerk"], float)
self.assertIn("angular_jerk", infos[AGENT_ID]["logs"])
self.assertIsInstance(infos[AGENT_ID]["logs"]["angular_jerk"], float)
self.assertIn("env_score", infos[AGENT_ID]["logs"])
self.assertIsInstance(infos[AGENT_ID]["logs"]["env_score"], float)
def test_default_observation_vector_adapter(self):
ADAPTER_TYPE = adapters.AdapterType.DefaultObservationVector
adapter = adapters.adapter_from_type(ADAPTER_TYPE)
interface = adapters.required_interface_from_types(ADAPTER_TYPE)
space = adapters.space_from_type(ADAPTER_TYPE)
agent, environment = prepare_test_agent_and_environment(
required_interface=interface,
observation_adapter=adapter,
)
_, _, observations_sequence, _ = run_experiment(agent,
environment,
max_steps=1)
observations = observations_sequence[0]
self.assertIsInstance(observations, dict)
self.assertIn(AGENT_ID, observations)
self.assertIn("low_dim_states", observations[AGENT_ID])
self.assertIn("social_vehicles", observations[AGENT_ID])
self.assertIsInstance(observations[AGENT_ID]["low_dim_states"],
np.ndarray)
self.assertIsInstance(observations[AGENT_ID]["social_vehicles"],
np.ndarray)
self.assertEqual(observations[AGENT_ID]["low_dim_states"].dtype,
"float32")
self.assertEqual(observations[AGENT_ID]["social_vehicles"].dtype,
"float32")
self.assertEqual(observations[AGENT_ID]["low_dim_states"].shape,
(47, ))
self.assertEqual(observations[AGENT_ID]["social_vehicles"].shape,
(10, 4))
self.assertEqual(space.dtype, None)
self.assertEqual(
space["low_dim_states"].dtype,
observations[AGENT_ID]["low_dim_states"].dtype,
)
self.assertEqual(
space["social_vehicles"].dtype,
observations[AGENT_ID]["social_vehicles"].dtype,
)
self.assertEqual(space.shape, None)
self.assertEqual(
space["low_dim_states"].shape,
observations[AGENT_ID]["low_dim_states"].shape,
)
self.assertEqual(
space["social_vehicles"].shape,
observations[AGENT_ID]["social_vehicles"].shape,
)
self.assertTrue(space.contains(observations[AGENT_ID]))
def test_default_reward_adapter(self):
ADAPTER_TYPE = adapters.AdapterType.DefaultReward
adapter = adapters.adapter_from_type(ADAPTER_TYPE)
interface = adapters.required_interface_from_types(ADAPTER_TYPE)
agent, environment = prepare_test_agent_and_environment(
required_interface=interface,
reward_adapter=adapter,
)
_, _, _, rewards_sequence = run_experiment(agent,
environment,
max_steps=1)
rewards = rewards_sequence[0]
self.assertIsInstance(rewards, dict)
self.assertIsInstance(rewards[AGENT_ID], float)
def test_default_observation_image_adapter(self):
ADAPTER_TYPE = adapters.AdapterType.DefaultObservationImage
adapter = adapters.adapter_from_type(ADAPTER_TYPE)
interface = adapters.required_interface_from_types(ADAPTER_TYPE)
space = adapters.space_from_type(ADAPTER_TYPE)
agent, environment = prepare_test_agent_and_environment(
required_interface=interface,
observation_adapter=adapter,
)
_, _, observations_sequence, _ = run_experiment(agent,
environment,
max_steps=1)
observations = observations_sequence[0]
self.assertIsInstance(observations, dict)
self.assertIn(AGENT_ID, observations)
self.assertIsInstance(observations[AGENT_ID], np.ndarray)
self.assertEqual(observations[AGENT_ID].dtype, "float32")
self.assertEqual(observations[AGENT_ID].shape, (4, 64, 64))
self.assertEqual(space.dtype, observations[AGENT_ID].dtype)
self.assertEqual(space.shape, observations[AGENT_ID].shape)
self.assertTrue(space.contains(observations[AGENT_ID]))
def train(
task,
num_episodes,
max_episode_steps,
rollout_fragment_length,
policy,
eval_info,
timestep_sec,
headless,
seed,
train_batch_size,
sgd_minibatch_size,
log_dir,
):
agent_name = policy
policy_params = load_yaml(
f"ultra/baselines/{agent_name}/{agent_name}/params.yaml")
action_type = adapters.type_from_string(policy_params["action_type"])
observation_type = adapters.type_from_string(
policy_params["observation_type"])
reward_type = adapters.type_from_string(policy_params["reward_type"])
if action_type != adapters.AdapterType.DefaultActionContinuous:
raise Exception(
f"RLlib training only supports the "
f"{adapters.AdapterType.DefaultActionContinuous} action type.")
if observation_type != adapters.AdapterType.DefaultObservationVector:
# NOTE: The SMARTS observations adaptation that is done in ULTRA's Gym
# environment is not done in ULTRA's RLlib environment. If other
# observation adapters are used, they may raise an Exception.
raise Exception(
f"RLlib training only supports the "
f"{adapters.AdapterType.DefaultObservationVector} observation type."
)
action_space = adapters.space_from_type(adapter_type=action_type)
observation_space = adapters.space_from_type(adapter_type=observation_type)
action_adapter = adapters.adapter_from_type(adapter_type=action_type)
info_adapter = adapters.adapter_from_type(
adapter_type=adapters.AdapterType.DefaultInfo)
observation_adapter = adapters.adapter_from_type(
adapter_type=observation_type)
reward_adapter = adapters.adapter_from_type(adapter_type=reward_type)
params_seed = policy_params["seed"]
encoder_key = policy_params["social_vehicles"]["encoder_key"]
num_social_features = observation_space["social_vehicles"].shape[1]
social_capacity = observation_space["social_vehicles"].shape[0]
social_policy_hidden_units = int(policy_params["social_vehicles"].get(
"social_policy_hidden_units", 0))
social_policy_init_std = int(policy_params["social_vehicles"].get(
"social_policy_init_std", 0))
social_vehicle_config = get_social_vehicle_configs(
encoder_key=encoder_key,
num_social_features=num_social_features,
social_capacity=social_capacity,
seed=params_seed,
social_policy_hidden_units=social_policy_hidden_units,
social_policy_init_std=social_policy_init_std,
)
ModelCatalog.register_custom_model("fc_model", CustomFCModel)
config = RllibAgent.rllib_default_config(agent_name)
rllib_policies = {
"default_policy": (
None,
observation_space,
action_space,
{
"model": {
"custom_model": "fc_model",
"custom_model_config": {
"social_vehicle_config": social_vehicle_config
},
}
},
)
}
agent_specs = {
"AGENT-007":
AgentSpec(
interface=AgentInterface(
waypoints=Waypoints(lookahead=20),
neighborhood_vehicles=NeighborhoodVehicles(200),
action=ActionSpaceType.Continuous,
rgb=False,
max_episode_steps=max_episode_steps,
debug=True,
),
agent_params={},
agent_builder=None,
action_adapter=action_adapter,
info_adapter=info_adapter,
observation_adapter=observation_adapter,
reward_adapter=reward_adapter,
)
}
tune_config = {
"env": RLlibUltraEnv,
"log_level": "WARN",
"callbacks": Callbacks,
"framework": "torch",
"num_workers": 1,
"train_batch_size": train_batch_size,
"sgd_minibatch_size": sgd_minibatch_size,
"rollout_fragment_length": rollout_fragment_length,
"in_evaluation": True,
"evaluation_num_episodes": eval_info["eval_episodes"],
"evaluation_interval": eval_info[
"eval_rate"], # Evaluation occurs after # of eval-intervals (episodes)
"evaluation_config": {
"env_config": {
"seed": seed,
"scenario_info": task,
"headless": headless,
"eval_mode": True,
"ordered_scenarios": False,
"agent_specs": agent_specs,
"timestep_sec": timestep_sec,
},
"explore": False,
},
"env_config": {
"seed": seed,
"scenario_info": task,
"headless": headless,
"eval_mode": False,
"ordered_scenarios": False,
"agent_specs": agent_specs,
"timestep_sec": timestep_sec,
},
"multiagent": {
"policies": rllib_policies
},
}
config.update(tune_config)
agent = RllibAgent(
agent_name=agent_name,
env=RLlibUltraEnv,
config=tune_config,
logger_creator=log_creator(log_dir),
)
# Iteration value in trainer.py (self._iterations) is the technically the number of episodes
for i in range(num_episodes):
results = agent.train()
agent.log_evaluation_metrics(
results) # Evaluation metrics will now be displayed on Tensorboard
def __new__(
self,
policy_class,
# action_type,
policy_params=None,
checkpoint_dir=None,
# task=None,
max_episode_steps=1200,
experiment_dir=None,
agent_id="",
):
if experiment_dir:
print(
f"Loading spec for {agent_id} from {experiment_dir}/agent_metadata.pkl"
)
with open(f"{experiment_dir}/agent_metadata.pkl",
"rb") as metadata_file:
agent_metadata = dill.load(metadata_file)
spec = agent_metadata["agent_specs"][agent_id]
new_spec = AgentSpec(
interface=spec.interface,
agent_params=dict(
policy_params=spec.agent_params["policy_params"],
checkpoint_dir=checkpoint_dir,
),
agent_builder=spec.agent_builder,
observation_adapter=spec.observation_adapter,
reward_adapter=spec.reward_adapter,
info_adapter=spec.info_adapter,
)
spec = new_spec
else:
# If policy_params is None, then there must be a params.yaml file in the
# same directory as the policy_class module.
if not policy_params:
policy_class_module_file = inspect.getfile(policy_class)
policy_class_module_directory = os.path.dirname(
policy_class_module_file)
policy_params = load_yaml(
os.path.join(policy_class_module_directory, "params.yaml"))
action_type = adapters.type_from_string(
string_type=policy_params["action_type"])
observation_type = adapters.type_from_string(
string_type=policy_params["observation_type"])
reward_type = adapters.type_from_string(
string_type=policy_params["reward_type"])
info_type = adapters.AdapterType.DefaultInfo
adapter_interface_requirements = adapters.required_interface_from_types(
action_type, observation_type, reward_type, info_type)
action_adapter = adapters.adapter_from_type(
adapter_type=action_type)
observation_adapter = adapters.adapter_from_type(
adapter_type=observation_type)
reward_adapter = adapters.adapter_from_type(
adapter_type=reward_type)
info_adapter = adapters.adapter_from_type(adapter_type=info_type)
spec = AgentSpec(
interface=AgentInterface(
**adapter_interface_requirements,
max_episode_steps=max_episode_steps,
debug=True,
),
agent_params=dict(policy_params=policy_params,
checkpoint_dir=checkpoint_dir),
agent_builder=policy_class,
action_adapter=action_adapter,
observation_adapter=observation_adapter,
reward_adapter=reward_adapter,
info_adapter=info_adapter,
)
return spec