def test_evaluate_agent(self):
seed = 2
model = glob.glob("tests/sac_test_models/*/models/0")[0]
log_dir = "tests/output_eval_agent_logs/"
policy_class = "ultra.baselines.sac:sac-v0"
ray.shutdown()
ray.init(ignore_reinit_error=True)
try:
evaluate.remote(
experiment_dir=None,
agent_id="AGENT_001",
policy_class=policy_class,
seed=seed,
itr_count=0,
checkpoint_dir=model,
scenario_info=("00", "eval_test"),
num_episodes=1,
max_episode_steps=2,
timestep_sec=0.1,
headless=True,
log_dir=log_dir,
)
self.assertTrue(True)
except Exception as err:
print(err)
self.assertTrue(False)
def test_evaluate_multiagent(self):
seed = 2
models_directory = glob.glob(
os.path.join(
EvaluateTest.OUTPUT_DIRECTORY, "multiagent_test_models/*/models/"
)
)[0]
log_dir = os.path.join(
EvaluateTest.OUTPUT_DIRECTORY, "output_eval_multiagent_logs/"
)
with open(
os.path.join(models_directory, "../agent_metadata.pkl"), "rb"
) as metadata_file:
agent_metadata = pickle.load(metadata_file)
agent_ids = agent_metadata["agent_ids"]
policy_classes = agent_metadata["agent_classes"]
checkpoint_directories = {
agent_id: sorted(
glob.glob(os.path.join(models_directory, agent_id, "*")),
key=lambda x: int(x.split("/")[-1]),
)
for agent_id in agent_ids
}
ray.shutdown()
ray.init(ignore_reinit_error=True)
try:
evaluate.remote(
experiment_dir=None,
agent_ids=agent_ids,
policy_classes=policy_classes,
seed=seed,
checkpoint_dirs=checkpoint_directories,
scenario_info=("00-multiagent", "eval_test"),
num_episodes=1,
max_episode_steps=2,
timestep_sec=0.1,
headless=True,
log_dir=log_dir,
)
self.assertTrue(True)
except Exception as err:
print(err)
self.assertTrue(False)
def test_evaluate_agent(self):
seed = 2
model = glob.glob("ultra/tests/ppo_models/models/*")[0]
ray.init(ignore_reinit_error=True)
try:
evaluate.remote(
experiment_dir=None,
agent_id="AGENT_001",
policy_class=policy_class,
seed=seed,
itr_count=0,
checkpoint_dir=model,
scenario_info=("00", "easy"),
num_episodes=2,
timestep_sec=0.1,
headless=True,
)
self.assertTrue(True)
except Exception as err:
print(err)
self.assertTrue(False)