def test_get_load_log_paths( patch_search_command, namespace_args, bayes_log_folder, fixture_cleanup_bayes_log_folder ): """Tests that log files are loaded from the Bayesian search log folder. Ensures: - Loaded log file paths are correct - Non-json files are not loaded from the log folder """ bayes_log_folder.mkdir() log_paths = [ bayes_log_folder / 'file1.json', bayes_log_folder / 'file2.json', bayes_log_folder / 'file3.json', bayes_log_folder / 'file4', ] for filename in log_paths: log_file = bayes_log_folder / filename with log_file.open('w') as f: f.write('Test log file') search = PerformBayesianSearch(namespace_args) search.trainer_config_path = Path(__file__) retrieved_log_paths = search.get_load_log_paths() assert len(retrieved_log_paths) == 3 for path in retrieved_log_paths: assert path.suffix == '.json' assert path in log_paths
def test_save_max_to_file( monkeypatch, patch_search_command, patch_perform_bayesian_search, patch_get_last_mean_reward_from_log, patch_get_load_log_paths, patch_get_save_log_path, namespace_args, trainer_config, ): """Tests that a BayesianOptimization object's max property is correctly converted into a trainer configuration dictionary.""" def mock_write_yaml_file(yaml_data, file_path): assert yaml_data == trainer_config monkeypatch.setattr(grimagents.command_util, 'write_yaml_file', mock_write_yaml_file) max = { 'target': 1.595, 'params': { 'batch_size': 144.0682249028942, 'beta': 0.0028687875149226343, 'buffer_size_multiple': 50.017156222601734, }, } search = PerformBayesianSearch(namespace_args) search.save_max_to_file(max)
def main(): configure_logging() if not common.is_pipenv_present(): search_log.error( 'No virtual environment is accessible by Pipenv from this directory, unable to run mlagents-learn' ) sys.exit(1) argv = get_argvs() args = parse_args(argv) if args.edit_config: EditGrimConfigFile(args).execute() elif args.search_count: OutputGridSearchCount(args).execute() elif args.export_index: ExportGridSearchConfiguration(args).execute() elif args.random: PerformRandomSearch(args).execute() elif args.bayesian: PerformBayesianSearch(args).execute() else: PerformGridSearch(args).execute() logging.shutdown()
def test_get_last_mean_reward_from_log(monkeypatch, reward): """Tests for retrieval of the final mean reward of the last training run.""" def mock_get_log_file_path(): return Path() def mock_load_last_lines_from_file(log_file, number_of_lines): return [ r'[2019-09-12 02:02:35,448][INFO] ---------------------------------------------------------------', r'[2019-09-12 02:02:35,450][INFO] Initiating \'3DBall_00-2019-09-12_02-02-34\'', r'[2019-09-12 02:03:14,850][INFO] Exporting brains:', r'[2019-09-12 02:03:14,855][INFO] UnitySDK\Assets\ML-Agents\Examples\3DBall\ImportedModels\3DBallLearning.nn', r'[2019-09-12 02:03:14,855][INFO] ', r'Training run \'3DBall_00-2019-09-12_02-02-34\' ended after 39 seconds', r'[2019-09-12 02:03:14,857][INFO] Training completed successfully', f'[2019-09-12 02:03:14,858][INFO] Final Mean Reward: {reward}', r'[2019-09-12 02:03:14,858][INFO] ---------------------------------------------------------------', ] monkeypatch.setattr(grimagents.settings, 'get_log_file_path', mock_get_log_file_path) monkeypatch.setattr( grimagents.command_util, 'load_last_lines_from_file', mock_load_last_lines_from_file ) assert PerformBayesianSearch.get_last_mean_reward_from_log() == reward
def test_get_save_log_path(monkeypatch, patch_search_command, namespace_args, bayes_log_folder): """Tests for the correct Bayesian search log file path creation.""" def mock_get_timestamp(): return '2019-09-13_03-41-44' def mock_get_log_folder_path(self): return bayes_log_folder monkeypatch.setattr(grimagents.common, 'get_timestamp', mock_get_timestamp) monkeypatch.setattr(PerformBayesianSearch, 'get_log_folder_path', mock_get_log_folder_path) log_path = Path(__file__).parent / '3DBall_bayes/3DBall_2019-09-13_03-41-44.json' search = PerformBayesianSearch(namespace_args) search.trainer_config_path = Path(__file__) assert search.get_save_log_path() == log_path
def test_get_log_folder_path( monkeypatch, patch_search_command, namespace_args, bayes_log_folder, fixture_cleanup_bayes_log_folder, ): """Test for the correct generation of a Bayesian search log folder path and ensure the folder is created if it doesn't exist. Ensures: - The correct log folder path is returned - The log folder is created if it did not previously exist """ search = PerformBayesianSearch(namespace_args) search.trainer_config_path = Path(__file__) assert search.get_log_folder_path() == bayes_log_folder assert bayes_log_folder.exists()
def test_perform_bayes_search( monkeypatch, patch_search_command, patch_perform_bayesian_search, patch_get_load_log_paths, patch_get_last_mean_reward_from_log, patch_get_save_log_path, namespace_args, trainer_config, ): """Tests that PerformBayesianSearch objects correctly perform searches with the specified trainer configurations. Ensures: - The correct configuration file is written for the search - The correct grimagents training command is generated """ def mock_write_yaml_file(yaml_data, file_path): assert yaml_data == trainer_config def mock_run(command): assert command == [ 'pipenv', 'run', 'python', '-m', 'grimagents', str(Path(namespace_args.configuration_file)), '--trainer-config', str(Path('config/search_config.yaml')), '--run-id', '3DBall_00', ] monkeypatch.setattr(grimagents.command_util, 'write_yaml_file', mock_write_yaml_file) monkeypatch.setattr(subprocess, 'run', mock_run) namespace_args.bayesian = [1, 3] search = PerformBayesianSearch(namespace_args) search.perform_bayes_search(batch_size=84, beta=0.002, buffer_size_multiple=88)
def test_perform_bayesian_search_execute( monkeypatch, patch_search_command, patch_perform_bayesian_search, patch_get_optimizer_max, patch_get_last_mean_reward_from_log, patch_get_load_log_paths, patch_get_save_log_path, patch_save_max_to_file, namespace_args, ): """Tests for the correct execution of a Bayesian search. - Ensures observation log loading respects the command line argument - Ensures obseration log saving respects the command line argument - Ensures the correct number of searches desired is communicated to the BayesianOptimization object """ subscribe_counter = Counter() # As we are mocking optimizer.maximize(), perform_bayes_search() will never be called and does not need to be mocked. def mock_optimizer_maximize(self, init_points, n_iter): assert init_points == 2 assert n_iter == 5 monkeypatch.setattr(BayesianOptimization, 'maximize', mock_optimizer_maximize) def mock_bayes_opt_load_logs(optimizer, logs): assert type(logs) is list assert len(logs) == 3 monkeypatch.setattr(bayes_opt.util, 'load_logs', mock_bayes_opt_load_logs) def mock_optimizer_subscribe(self, step, logger): subscribe_counter.increment_counter() monkeypatch.setattr(BayesianOptimization, 'subscribe', mock_optimizer_subscribe) namespace_args.bayesian = [2, 5] namespace_args.bayes_load = True namespace_args.bayes_save = True search = PerformBayesianSearch(namespace_args) search.execute() # The BayesianOptimization object calls subscribe() three times during maximization. A fourth call is made if PerformBayesianSearch object has decided to save optimization logs. As we are mocking the maximize() method, we expect only one call to subescribe(). assert subscribe_counter.count == 1 def mock_bayes_opt_load_logs(optimizer, logs): assert True is False monkeypatch.setattr(bayes_opt.util, 'load_logs', mock_bayes_opt_load_logs) subscribe_counter.reset_counter() namespace_args.bayesian = [2, 5] namespace_args.bayes_load = False namespace_args.bayes_save = False search = PerformBayesianSearch(namespace_args) search.execute() assert subscribe_counter.count == 0
def test_perform_bayesian_search_init(patch_search_command, namespace_args): """Tests for the correct construction of a bayesian search trainer config output path.""" search = PerformBayesianSearch(namespace_args) assert search.output_config_path == Path('config/3DBall_bayes.yaml')