def test_tune_IC_grid_metrics() -> None: """ Runs hyperparameter IC grid search and compares metrics against a saved baseline. """ # Load hyperparameter search config. with open(IC_GRID_CONFIG_PATH, "r") as config_file: config = json.load(config_file) # Modify default training config. config["base_train_config"]["baseline_metrics_filename"] = "tune_IC_grid" # Run training. tune(config)
def test_tune_IC_grid_early_stop_param() -> None: """ Runs hyperparameter IC grid search until an early stop point between params. """ # Load hyperparameter search config. with open(IC_GRID_CONFIG_PATH, "r") as config_file: config = json.load(config_file) # Modify default training config to stop early. config["early_stop"] = {"param_num": 2, "val_num": 0, "trials": 0} # Run training. results = tune(config) # Check results. param_iterations = ( lambda p: p["num_values"] if "num_values" in p else len(p["choices"]) ) expected_iterations = 0 param_names = list(config["search_params"].keys()) for i in range(config["early_stop"]["param_num"]): expected_iterations += param_iterations(config["search_params"][param_names[i]]) assert len(results["iterations"]) == expected_iterations for config_results in results["iterations"]: assert len(config_results["trials"]) == config["trials_per_config"]
def test_tune_grid_values() -> None: """ Runs hyperparameter grid search and makes sure that the correct parameter combinations are used for training. """ # Load hyperparameter search config. with open(GRID_VALUES_CONFIG_PATH, "r") as config_file: tune_config = json.load(config_file) # Construct expected parameter combinations. expected_configs = [] variable_params = [ "initial_lr", "final_lr", "clip_param", "recurrent", ] variable_param_combos = [ [1e-5, 1e-6, 0.6, True], [1e-5, 1e-6, 0.6, False], [1e-4, 1e-6, 0.6, True], [1e-4, 1e-6, 0.6, False], [1e-3, 1e-6, 0.6, True], [1e-3, 1e-6, 0.6, False], ] for variable_param_combo in variable_param_combos: config = dict(tune_config["base_train_config"]) updated_params = dict(zip(variable_params, variable_param_combo)) config = update_config(config, updated_params) expected_configs.append(dict(config)) # Run training and extract actual configs from results. results = tune(tune_config) actual_configs = [ config_results["config"] for config_results in results["iterations"] ] # Compare actual configs with expected configs. We need to do this in a kind of # janky way, since we want to allow the possibility that the list of configs are in # a different order, but we can't make a set of dicts (dicts aren't hashable). We # test that the lists have the same order, that expected_configs contains unique # values, and that each element of expected_configs is an element of actual_configs. assert len(expected_configs) == len(actual_configs) assert all(c1 != c2 for c1, c2 in itertools.combinations(expected_configs, 2)) for expected_config in expected_configs: assert expected_config in actual_configs
def test_tune_IC_grid_values() -> None: """ Runs hyperparameter IC grid search and makes sure that the correct parameter combinations are used for training. """ # Load hyperparameter search config. with open(IC_GRID_CONFIG_PATH, "r") as config_file: config = json.load(config_file) # Construct expected param value intervals. param_intervals = { "initial_lr": [1e-5, 1e-3], "num_layers": [1, 8], "recurrent": [True, False], } # Run training and extract actual configs from results. results = tune(config) actual_configs = [ config_results["config"] for config_results in results["iterations"] ] # Verify configs from training. iteration = 0 best_param_vals: Dict[str, float] = {} for param_name, param_values in param_intervals.items(): param_fitnesses = {} for param_val in param_values: # Check values. for best_param_name, best_param_val in best_param_vals.items(): found, result = dict_search(actual_configs[iteration], best_param_name) assert found assert result == best_param_val # Store fitnesses for comparison. param_fitnesses[param_val] = results["iterations"][iteration]["fitness"] iteration += 1 best_param_val = dict_argmax(param_fitnesses) best_param_vals[param_name] = best_param_val
def test_tune_random_early_stop_iteration() -> None: """ Runs hyperparameter random search until an early stop point between iterations. """ # Load hyperparameter search config. with open(RANDOM_CONFIG_PATH, "r") as config_file: config = json.load(config_file) # Modify default training config to stop early. config["early_stop"] = {"iterations": 2, "trials": 0} # Run training. results = tune(config) # Check results. assert len(results["iterations"]) == config["early_stop"]["iterations"] for config_results in results["iterations"]: assert len(config_results["trials"]) == config["trials_per_config"]
def resume_template( save_name: str, config_path: str, early_stops: List[Dict[str, int]], baseline_name: str, results_name: str, ) -> None: """ Runs while stopping to save/load at a given set of checkpoints, then compares results against non-interrupted version. """ # Load hyperparameter search config. with open(config_path, "r") as config_file: config = json.load(config_file) config["base_train_config"]["save_name"] = save_name # Set baseline to compare against throughout training. config["base_train_config"]["baseline_metrics_filename"] = baseline_name # Ensure that there are no existing saved experiments whose names coincide with the # experiment names used here. We do have to save and load from disk so we want to # make sure that we aren't overwriting any previously existing files. num_param_values = None if config["search_type"] == "IC_grid": num_param_values = get_num_param_values(config["search_params"]) iterations = get_iterations(config["search_type"], config["search_iterations"], config["search_params"]) check_name_uniqueness( save_name, config["search_type"], iterations, config["trials_per_config"], num_param_values=num_param_values, ) # Run until hitting each early stopping point. for stop_index in range(len(early_stops)): # Set early stopping point. config["early_stop"] = early_stops[stop_index] # Set loading point, if necessary. if stop_index > 0: config["load_from"] = save_name # Run partial training. tune(config) # Finish training from checkpoint. config["early_stop"] = None config["load_from"] = save_name resumed_results = tune(config) # Compare resumed results with un-interrupted results. if results_name is not None: results_path = os.path.join(METRICS_DIR, "%s.json" % results_name) with open(results_path, "r") as results_file: correct_results = json.load(results_file) assert tune_results_equal(resumed_results, correct_results) # Clean up saved results. experiment_names = get_experiment_names( save_name, config["search_type"], iterations, config["trials_per_config"], num_param_values=num_param_values, ) for name in experiment_names: save_dir = save_dir_from_name(name) if os.path.isdir(save_dir): rmtree(save_dir)
if __name__ == "__main__": # Parse config filename from command line arguments. parser = argparse.ArgumentParser() parser.add_argument( "command", type=str, help="Command to run. Either 'train' or 'tune'.", ) parser.add_argument( "config_filename", type=str, help="Name of config file to load from.", ) args = parser.parse_args() # Load config file. with open(args.config_filename, "r") as config_file: config = json.load(config_file) # Run specified command. if args.command == "train": train(config) elif args.command == "tune": tune(config) elif args.command == "meta_train": meta_train(config) else: raise ValueError("Unsupported command: '%s'" % args.command)