def run_policy(results_path,
benchmark_name,
num_episodes,
policy,
seeds=np.arange(10)):
bench = getattr(benchmarks, benchmark_name)()
for s in seeds:
if benchmark_name == "CMAESBenchmark":
experiment_name = f"csa_{s}"
else:
experiment_name = f"optimal_{s}"
logger = Logger(experiment_name=experiment_name,
output_path=results_path / benchmark_name)
env = bench.get_benchmark(seed=s)
env = PerformanceTrackingWrapper(
env, logger=logger.add_module(PerformanceTrackingWrapper))
agent = GenericAgent(env, policy)
logger.add_agent(agent)
logger.add_benchmark(bench)
logger.set_env(env)
run_benchmark(env, agent, num_episodes, logger)
logger.close()
def run_dacbench(results_path,
agent_method,
num_episodes,
bench=None,
seeds=None):
"""
Run all benchmarks for 10 seeds for a given number of episodes with a given agent and save result
Parameters
-------
bench
results_path : str
Path to where results should be saved
agent_method : function
Method that takes an env as input and returns an agent
num_episodes : int
Number of episodes to run for each benchmark
seeds : list[int]
List of seeds to runs all benchmarks for. If None (default) seeds [1, ..., 10] are used.
"""
if bench is None:
bench = map(benchmarks.__dict__.get, benchmarks.__all__)
else:
bench = [getattr(benchmarks, b) for b in bench]
seeds = seeds if seeds is not None else range(10)
for b in bench:
print(f"Evaluating {b.__name__}")
for i in seeds:
print(f"Seed {i}/10")
bench = b()
try:
env = bench.get_benchmark(seed=i)
except:
continue
logger = Logger(
experiment_name=f"seed_{i}",
output_path=Path(results_path) / f"{b.__name__}",
)
perf_logger = logger.add_module(PerformanceTrackingWrapper)
logger.add_benchmark(bench)
logger.set_env(env)
env = PerformanceTrackingWrapper(env, logger=perf_logger)
agent = agent_method(env)
logger.add_agent(agent)
run_benchmark(env, agent, num_episodes, logger)
logger.close()
def run_random(results_path, benchmark_name, num_episodes, seeds, fixed):
bench = getattr(benchmarks, benchmark_name)()
for s in seeds:
if fixed > 1:
experiment_name = f"random_fixed{fixed}_{s}"
else:
experiment_name = f"random_{s}"
logger = Logger(experiment_name=experiment_name,
output_path=results_path / benchmark_name)
env = bench.get_benchmark(seed=s)
env = PerformanceTrackingWrapper(
env, logger=logger.add_module(PerformanceTrackingWrapper))
agent = DynamicRandomAgent(env, fixed)
logger.add_agent(agent)
logger.add_benchmark(bench)
logger.set_env(env)
run_benchmark(env, agent, num_episodes, logger)
logger.close()
def run_optimal(results_path,
benchmark_name,
num_episodes,
seeds=np.arange(10)):
bench = getattr(benchmarks, benchmark_name)()
if benchmark_name == "LubyBenchmark":
policy = optimal_luby
elif benchmark_name == "SigmoidBenchmark":
policy = optimal_sigmoid
elif benchmark_name == "FastDownwardBenchmark":
policy = optimal_fd
elif benchmark_name == "CMAESBenchmark":
policy = csa
else:
print("No comparison policy found for this benchmark")
return
for s in seeds:
if benchmark_name == "CMAESBenchmark":
experiment_name = f"csa_{s}"
else:
experiment_name = f"optimal_{s}"
logger = Logger(experiment_name=experiment_name,
output_path=results_path / benchmark_name)
env = bench.get_benchmark(seed=s)
env = PerformanceTrackingWrapper(
env, logger=logger.add_module(PerformanceTrackingWrapper))
agent = GenericAgent(env, policy)
logger.add_agent(agent)
logger.add_benchmark(bench)
logger.set_env(env)
logger.set_additional_info(seed=s)
run_benchmark(env, agent, num_episodes, logger)
logger.close()
def run_dacbench(results_path, agent_method, num_episodes):
"""
Run all benchmarks for 10 seeds for a given number of episodes with a given agent and save result
Parameters
-------
results_path : str
Path to where results should be saved
agent_method : function
Method that takes an env as input and returns an agent
num_episodes : int
Number of episodes to run for each benchmark
"""
for b in map(benchmarks.__dict__.get, benchmarks.__all__):
print(f"Evaluating {b.__name__}")
for i in range(10):
print(f"Seed {i}/10")
bench = b()
env = bench.get_benchmark(seed=i)
logger = Logger(
experiment_name=f"seed_{i}",
output_path=Path(results_path) / f"{b.__name__}",
)
perf_logger = logger.add_module(PerformanceTrackingWrapper)
logger.add_benchmark(bench)
logger.set_env(env)
logger.set_additional_info(seed=i)
env = PerformanceTrackingWrapper(env, logger=perf_logger)
agent = agent_method(env)
logger.add_agent(agent)
run_benchmark(env, agent, num_episodes, logger)
logger.close()
def run_static(results_path,
benchmark_name,
action,
num_episodes,
seeds=np.arange(10)):
bench = getattr(benchmarks, benchmark_name)()
for s in seeds:
logger = Logger(
experiment_name=f"static_{action}_{s}",
output_path=results_path / benchmark_name,
)
env = bench.get_benchmark(seed=s)
env = PerformanceTrackingWrapper(
env, logger=logger.add_module(PerformanceTrackingWrapper))
agent = StaticAgent(env, action)
logger.add_agent(agent)
logger.add_benchmark(bench)
logger.set_env(env)
logger.set_additional_info(seed=s, action=action)
run_benchmark(env, agent, num_episodes, logger)
logger.close()