def __init__(self, log_dir=None, log_id=None, use_timestamp=True, parallel=None, slurm=None): """ Constructor. Args: log_dir (str): path to the log directory (Default: ./logs or /work/scratch/$USER) log_id (str): log id (Default: benchmark[_YYYY-mm-dd-HH-MM-SS]) use_timestamp (bool): select if a timestamp should be appended to the log id parallel (dict, None): parameters that are passed to the run_parallel method of the experiment slurm (dict, None): parameters that are passed to the run_slurm method of the experiment """ self._experiment_structure = dict() self._environment_dict = dict() self._agent_list = [] self._parallel = parallel self._slurm = slurm self.logger = BenchmarkLogger(log_dir=log_dir, log_id=log_id, use_timestamp=use_timestamp)
def _create_experiment(self, environment, environment_params, agent_name, agent_builder_params): separator = '.' if separator in environment: environment_name, environment_id = environment.split(separator) environment_params = dict(env_id=environment_id, **environment_params) environment = environment.replace(separator, '_') else: environment_name = environment logger = BenchmarkLogger(log_dir=self.logger.get_path(), log_id='{}/{}'.format(environment, agent_name), use_timestamp=False) try: builder = getattr(mushroom_rl_benchmark.builders, '{}Builder'.format(agent_name)) except AttributeError as e: logger.exception(e) agent_builder = builder.default(**agent_builder_params) env_builder = EnvironmentBuilder(environment_name, environment_params) exp = BenchmarkExperiment(agent_builder, env_builder, logger) return exp
def __init__(self, logger, color_cycle=None, y_limit=None): self._logger = logger path = Path(self._logger.get_path()) self._logger_dict = {} self._color_cycle = dict() if color_cycle is None else color_cycle self._y_limit = dict() if y_limit is None else y_limit alg_count = 0 for env_dir in path.iterdir(): if env_dir.is_dir() and env_dir.name != 'plots': env = env_dir.name self._logger_dict[env] = dict() for alg_dir in env_dir.iterdir(): if alg_dir.is_dir(): alg = alg_dir.name if alg not in self._color_cycle: self._color_cycle[alg] = 'C' + str(alg_count) alg_logger = BenchmarkLogger.from_path(alg_dir) self._logger_dict[env][alg] = alg_logger alg_count += 1
def from_path(cls, path): """ Method to create a BenchmarkVisualizer from a path. """ path = Path(path) return cls(BenchmarkLogger(path.parent, path.name, False))
def __init__(self, log_dir=None, log_id=None, use_timestamp=True, **run_params): """ Constructor. Kwargs: log_dir (str): path to the log directory (Default: ./logs or /work/scratch/$USER) log_id (str): log id (Default: benchmark[_YY-mm-ddTHH:MM:SS.zzz]) use_timestamp (bool): select if a timestamp should be appended to the log id **run_params (dict): parameters that are passed to the run method of the experiment """ self.experiment_structure = dict() self.environment_list = [] self.agent_list = [] self.run_params = run_params self.logger = BenchmarkLogger(log_dir=log_dir, log_id=log_id, use_timestamp=use_timestamp)
class BenchmarkSuite: """ Class to orchestrate the execution of multiple experiments. """ def __init__(self, log_dir=None, log_id=None, use_timestamp=True, **run_params): """ Constructor. Kwargs: log_dir (str): path to the log directory (Default: ./logs or /work/scratch/$USER) log_id (str): log id (Default: benchmark[_YY-mm-ddTHH:MM:SS.zzz]) use_timestamp (bool): select if a timestamp should be appended to the log id **run_params (dict): parameters that are passed to the run method of the experiment """ self.experiment_structure = dict() self.environment_list = [] self.agent_list = [] self.run_params = run_params self.logger = BenchmarkLogger(log_dir=log_dir, log_id=log_id, use_timestamp=use_timestamp) def add_experiment(self, environment_name, environment_builder_params, agent_name, agent_builder_params): """ Add an experiment to the suite. Args: environment_name (str): name of the environment for the experiment (E.g. Gym.Pendulum-v0) environment_builder_params (dict): parameters for the environment builder agent_name (str): name of the agent for the experiment agent_builder_params (dict): parameters for the agent builder """ if environment_name in self.environment_list: if agent_name in self.experiment_structure[environment_name]: raise AttributeError( 'An experiment for environment {} and builders {} already exists.' .format(environment_name, agent_name)) else: self.experiment_structure[environment_name][ agent_name] = self._create_experiment( environment_name, environment_builder_params, agent_name, agent_builder_params) else: self.environment_list.append(environment_name) self.experiment_structure[environment_name] = { agent_name: self._create_experiment(environment_name, environment_builder_params, agent_name, agent_builder_params) } if agent_name not in self.agent_list: self.agent_list.append(agent_name) def _create_experiment(self, environment, environment_params, agent_name, agent_builder_params): separator = '.' if separator in environment: environment_name, environment_id = environment.split(separator) environment_params = dict(env_id=environment_id, **environment_params) environment = environment.replace(separator, '_') else: environment_name = environment logger = BenchmarkLogger(log_dir=self.logger.get_path(), log_id='{}/{}'.format(environment, agent_name), use_timestamp=False) try: builder = getattr(mushroom_rl_benchmark.builders, '{}Builder'.format(agent_name)) except AttributeError as e: logger.exception(e) agent_builder = builder.default(**agent_builder_params) env_builder = EnvironmentBuilder(environment_name, environment_params) exp = BenchmarkExperiment(agent_builder, env_builder, logger) return exp def print_experiments(self): """ Print the experiments in the suite. """ for env, agents in self.experiment_structure.items(): for agent, _ in agents.items(): self.logger.info('Environment: {}\tAgent: {}'.format( env, agent)) def run(self, exec_type='sequential'): """ Run all experiments in the suite. """ for environment, agents in self.experiment_structure.items(): for agent, exp in agents.items(): self.logger.info('Starting Experiment for {} on {}'.format( agent, environment)) exp.run(exec_type=exec_type, **self.run_params)
class BenchmarkSuite: """ Class to orchestrate the execution of multiple experiments. """ def __init__(self, log_dir=None, log_id=None, use_timestamp=True, parallel=None, slurm=None): """ Constructor. Args: log_dir (str): path to the log directory (Default: ./logs or /work/scratch/$USER) log_id (str): log id (Default: benchmark[_YYYY-mm-dd-HH-MM-SS]) use_timestamp (bool): select if a timestamp should be appended to the log id parallel (dict, None): parameters that are passed to the run_parallel method of the experiment slurm (dict, None): parameters that are passed to the run_slurm method of the experiment """ self._experiment_structure = dict() self._environment_dict = dict() self._agent_list = [] self._parallel = parallel self._slurm = slurm self.logger = BenchmarkLogger(log_dir=log_dir, log_id=log_id, use_timestamp=use_timestamp) def add_experiments(self, environment_name, environment_builder_params, agent_names_list, agent_builders_params, **run_params): """ Add a set of experiments for the same environment to the suite. Args: environment_name (str): name of the environment for the experiment (E.g. Gym.Pendulum-v0); environment_builder_params (dict): parameters for the environment builder; agent_names_list (list): list of names of the agents for the experiments; agent_builders_params (list): list of dictionaries containing the parameters for the agent builder; run_params: Parameters that are passed to the run method of the experiment. """ self.add_environment(environment_name, environment_builder_params, **run_params) for agent_name, agent_params in zip(agent_names_list, agent_builders_params): self.add_agent(environment_name, agent_name, agent_params) def add_environment(self, environment_name, environment_builder_params, **run_params): """ Add an environment to the benchmarking suite. Args: environment_name (str): name of the environment for the experiment (E.g. Gym.Pendulum-v0); environment_builder_params (dict): parameters for the environment builder; run_params: Parameters that are passed to the run method of the experiment. """ if environment_name in self._environment_dict: raise AttributeError( 'The environment {} has been already added to the benchmark'. format(environment_name)) self._environment_dict[environment_name] = dict( build_params=environment_builder_params, run_params=run_params) self._experiment_structure[environment_name] = dict() def add_agent(self, environment_name, agent_name, agent_params): """ Add an agent to the benchmarking suite. Args: environment_name (str): name of the environment for the experiment (E.g. Gym.Pendulum-v0); agent_name (str): name of the agent for the experiments; agent_params (list): dictionary containing the parameters for the agent builder. """ assert environment_name in self._environment_dict if agent_name in self._experiment_structure[environment_name]: raise AttributeError( 'An experiment for environment {} and builders {} already exists.' .format(environment_name, agent_name)) self._agent_list.append(agent_name) environment_builder_params = self._environment_dict[environment_name][ 'build_params'] self._experiment_structure[environment_name][agent_name] = \ self._create_experiment(environment_name, environment_builder_params, agent_name, agent_params) def _create_experiment(self, environment, environment_params, agent_name, agent_builder_params): separator = '.' if environment_params is None: environment_params = dict() if separator in environment: environment_name, environment_id = environment.split(separator) environment_params = dict(env_id=environment_id, **environment_params) environment = environment_id else: environment_name = environment logger = BenchmarkLogger(log_dir=self.logger.get_path(), log_id='{}/{}'.format(environment, agent_name), use_timestamp=False) try: builder = getattr(mushroom_rl_benchmark.builders, '{}Builder'.format(agent_name)) except AttributeError as e: logger.exception(e) agent_builder = builder.default(**agent_builder_params) env_builder = EnvironmentBuilder(environment_name, environment_params) exp = BenchmarkExperiment(agent_builder, env_builder, logger) return exp def print_experiments(self): """ Print the experiments in the suite. """ first = True for env, agents in self._experiment_structure.items(): if not first: self.logger.weak_line() first = False self.logger.info('Environment: ' + env) for agent, _ in agents.items(): self.logger.info('- ' + agent) def run(self, exec_type='sequential'): """ Run all experiments in the suite. """ for environment, agents in self._experiment_structure.items(): for agent, exp in agents.items(): self.logger.info('Starting Experiment for {} on {}'.format( agent, environment)) run_params = self._environment_dict[environment]['run_params'] exp.run(exec_type=exec_type, parallel=self._parallel, slurm=self._slurm, **run_params) def save_plots(self, **plot_params): """ Save the result plots to the log directory. Args: **plot_params: parameters to be passed to the suite visualizer. """ visualizer = BenchmarkSuiteVisualizer(self.logger, **plot_params) visualizer.save_reports() def show_plots(self, **plot_params): """ Display the result plots. Args: **plot_params: parameters to be passed to the suite visualizer. """ visualizer = BenchmarkSuiteVisualizer(self.logger, **plot_params) visualizer.show_report()