def run_policy(results_path, benchmark_name, num_episodes, policy, seeds=np.arange(10)): bench = getattr(benchmarks, benchmark_name)() for s in seeds: if benchmark_name == "CMAESBenchmark": experiment_name = f"csa_{s}" else: experiment_name = f"optimal_{s}" logger = Logger(experiment_name=experiment_name, output_path=results_path / benchmark_name) env = bench.get_benchmark(seed=s) env = PerformanceTrackingWrapper( env, logger=logger.add_module(PerformanceTrackingWrapper)) agent = GenericAgent(env, policy) logger.add_agent(agent) logger.add_benchmark(bench) logger.set_env(env) run_benchmark(env, agent, num_episodes, logger) logger.close()
def run_dacbench(results_path, agent_method, num_episodes, bench=None, seeds=None): """ Run all benchmarks for 10 seeds for a given number of episodes with a given agent and save result Parameters ------- bench results_path : str Path to where results should be saved agent_method : function Method that takes an env as input and returns an agent num_episodes : int Number of episodes to run for each benchmark seeds : list[int] List of seeds to runs all benchmarks for. If None (default) seeds [1, ..., 10] are used. """ if bench is None: bench = map(benchmarks.__dict__.get, benchmarks.__all__) else: bench = [getattr(benchmarks, b) for b in bench] seeds = seeds if seeds is not None else range(10) for b in bench: print(f"Evaluating {b.__name__}") for i in seeds: print(f"Seed {i}/10") bench = b() try: env = bench.get_benchmark(seed=i) except: continue logger = Logger( experiment_name=f"seed_{i}", output_path=Path(results_path) / f"{b.__name__}", ) perf_logger = logger.add_module(PerformanceTrackingWrapper) logger.add_benchmark(bench) logger.set_env(env) env = PerformanceTrackingWrapper(env, logger=perf_logger) agent = agent_method(env) logger.add_agent(agent) run_benchmark(env, agent, num_episodes, logger) logger.close()
def run_random(results_path, benchmark_name, num_episodes, seeds, fixed): bench = getattr(benchmarks, benchmark_name)() for s in seeds: if fixed > 1: experiment_name = f"random_fixed{fixed}_{s}" else: experiment_name = f"random_{s}" logger = Logger(experiment_name=experiment_name, output_path=results_path / benchmark_name) env = bench.get_benchmark(seed=s) env = PerformanceTrackingWrapper( env, logger=logger.add_module(PerformanceTrackingWrapper)) agent = DynamicRandomAgent(env, fixed) logger.add_agent(agent) logger.add_benchmark(bench) logger.set_env(env) run_benchmark(env, agent, num_episodes, logger) logger.close()
def run_optimal(results_path, benchmark_name, num_episodes, seeds=np.arange(10)): bench = getattr(benchmarks, benchmark_name)() if benchmark_name == "LubyBenchmark": policy = optimal_luby elif benchmark_name == "SigmoidBenchmark": policy = optimal_sigmoid elif benchmark_name == "FastDownwardBenchmark": policy = optimal_fd elif benchmark_name == "CMAESBenchmark": policy = csa else: print("No comparison policy found for this benchmark") return for s in seeds: if benchmark_name == "CMAESBenchmark": experiment_name = f"csa_{s}" else: experiment_name = f"optimal_{s}" logger = Logger(experiment_name=experiment_name, output_path=results_path / benchmark_name) env = bench.get_benchmark(seed=s) env = PerformanceTrackingWrapper( env, logger=logger.add_module(PerformanceTrackingWrapper)) agent = GenericAgent(env, policy) logger.add_agent(agent) logger.add_benchmark(bench) logger.set_env(env) logger.set_additional_info(seed=s) run_benchmark(env, agent, num_episodes, logger) logger.close()
def run_dacbench(results_path, agent_method, num_episodes): """ Run all benchmarks for 10 seeds for a given number of episodes with a given agent and save result Parameters ------- results_path : str Path to where results should be saved agent_method : function Method that takes an env as input and returns an agent num_episodes : int Number of episodes to run for each benchmark """ for b in map(benchmarks.__dict__.get, benchmarks.__all__): print(f"Evaluating {b.__name__}") for i in range(10): print(f"Seed {i}/10") bench = b() env = bench.get_benchmark(seed=i) logger = Logger( experiment_name=f"seed_{i}", output_path=Path(results_path) / f"{b.__name__}", ) perf_logger = logger.add_module(PerformanceTrackingWrapper) logger.add_benchmark(bench) logger.set_env(env) logger.set_additional_info(seed=i) env = PerformanceTrackingWrapper(env, logger=perf_logger) agent = agent_method(env) logger.add_agent(agent) run_benchmark(env, agent, num_episodes, logger) logger.close()
def run_static(results_path, benchmark_name, action, num_episodes, seeds=np.arange(10)): bench = getattr(benchmarks, benchmark_name)() for s in seeds: logger = Logger( experiment_name=f"static_{action}_{s}", output_path=results_path / benchmark_name, ) env = bench.get_benchmark(seed=s) env = PerformanceTrackingWrapper( env, logger=logger.add_module(PerformanceTrackingWrapper)) agent = StaticAgent(env, action) logger.add_agent(agent) logger.add_benchmark(bench) logger.set_env(env) logger.set_additional_info(seed=s, action=action) run_benchmark(env, agent, num_episodes, logger) logger.close()