def _run_multirun_job(cfg: DictConfig) -> float: """Runs a maze job which is part of a Hydra --multirun and returns the maximum mean reward of this run. :param cfg: Hydra configuration for the rollout. :return: The maximum mean reward achieved throughout the training run. """ # required for Hydra --multirun (e.g., sweeper) clear_global_state() try: _run_job(cfg) # when optimizing hyper parameters a single exception # in one job should not break the entire experiment except: return float(np.finfo(np.float32).min) # load tensorboard log and return maximum mean reward # load tensorboard log tf_summary_files = glob.glob("*events.out.tfevents*") assert len( tf_summary_files ) == 1, f"expected exactly 1 tensorflow summary file {tf_summary_files}" events_df = tensorboard_to_pandas(tf_summary_files[0]) # compute maximum mean reward max_mean_reward = np.max( np.asarray(events_df.loc["train_BaseEnvEvents/reward/mean"])) # Add hparams logging to tensorboard metrics = [('train_BaseEnvEvents/reward/mean', max_mean_reward, 'max')] manipulate_hparams_logging_for_exp('.', metrics, clear_hparams=False) return float(max_mean_reward)
def run(self) -> None: """overwrite Process.run()""" # clean any left-over state from the parent process clear_global_state() try: result = self._target(*self._args, **self._kwargs) self._cconn.send(result) except Exception as e: self._cconn.send(e) raise e
def perform_algorithm_seeding_test(hydra_overrides: Dict[str, str]): # Perform base run for comparison ---------------------------------------------------------------------------------- base_dir = os.path.abspath('.') os.mkdir('./base_exp') os.chdir('./base_exp') # run training with Timeout(seconds=60): cfg = run_maze_job(hydra_overrides, config_module="maze.conf", config_name="conf_train") # load tensorboard log tf_summary_files = glob.glob("*events.out.tfevents*") assert len( tf_summary_files ) == 1, f"expected exactly 1 tensorflow summary file {tf_summary_files}" events_df = tensorboard_to_pandas(tf_summary_files[0]) clear_global_state() # Perform comparison run with same seeds --------------------------------------------------------------------------- os.chdir(base_dir) os.mkdir('./exp_pos') os.chdir('./exp_pos') hydra_overrides['seeding.agent_base_seed'] = cfg.seeding.agent_base_seed hydra_overrides['seeding.env_base_seed'] = cfg.seeding.env_base_seed # run training with Timeout(seconds=60): run_maze_job(hydra_overrides, config_module="maze.conf", config_name="conf_train") # load tensorboard log tf_summary_files = glob.glob("*events.out.tfevents*") assert len( tf_summary_files ) == 1, f"expected exactly 1 tensorflow summary file {tf_summary_files}" events_df_2 = tensorboard_to_pandas(tf_summary_files[0]) clear_global_state() del hydra_overrides['seeding.agent_base_seed'] del hydra_overrides['seeding.env_base_seed'] assert len(events_df) == len(events_df_2) for idx, (key, epoch) in enumerate(events_df.index): if 'time' in key: continue assert events_df_2.values[idx] == events_df.values[idx], \ f'Value not equal for key: {key} in epoch: {epoch}' # Perform second comparison run with different seeds --------------------------------------------------------------- os.chdir(base_dir) os.mkdir('./exp_neg') os.chdir('./exp_neg') # run training with Timeout(seconds=60): run_maze_job(hydra_overrides, config_module="maze.conf", config_name="conf_train") # load tensorboard log tf_summary_files = glob.glob("*events.out.tfevents*") assert len( tf_summary_files ) == 1, f"expected exactly 1 tensorflow summary file {tf_summary_files}" events_df_2 = tensorboard_to_pandas(tf_summary_files[0]) all_equal = True for idx, (key, epoch) in enumerate(events_df.index): if 'time' in key: continue all_equal = all_equal and events_df.values[idx] == events_df_2.values[ idx] assert not all_equal, 'The resulting logs should not be all equal'
def fixture_clear_global_state(): """clear global state before any test""" clear_global_state()