def test_tune_IC_grid_metrics() -> None:
"""
Runs hyperparameter IC grid search and compares metrics against a saved baseline.
"""
# Load hyperparameter search config.
with open(IC_GRID_CONFIG_PATH, "r") as config_file:
config = json.load(config_file)
# Modify default training config.
config["base_train_config"]["baseline_metrics_filename"] = "tune_IC_grid"
# Run training.
tune(config)
def test_tune_IC_grid_early_stop_param() -> None:
"""
Runs hyperparameter IC grid search until an early stop point between params.
"""
# Load hyperparameter search config.
with open(IC_GRID_CONFIG_PATH, "r") as config_file:
config = json.load(config_file)
# Modify default training config to stop early.
config["early_stop"] = {"param_num": 2, "val_num": 0, "trials": 0}
# Run training.
results = tune(config)
# Check results.
param_iterations = (
lambda p: p["num_values"] if "num_values" in p else len(p["choices"])
)
expected_iterations = 0
param_names = list(config["search_params"].keys())
for i in range(config["early_stop"]["param_num"]):
expected_iterations += param_iterations(config["search_params"][param_names[i]])
assert len(results["iterations"]) == expected_iterations
for config_results in results["iterations"]:
assert len(config_results["trials"]) == config["trials_per_config"]
def test_tune_grid_values() -> None:
"""
Runs hyperparameter grid search and makes sure that the correct parameter
combinations are used for training.
"""
# Load hyperparameter search config.
with open(GRID_VALUES_CONFIG_PATH, "r") as config_file:
tune_config = json.load(config_file)
# Construct expected parameter combinations.
expected_configs = []
variable_params = [
"initial_lr",
"final_lr",
"clip_param",
"recurrent",
]
variable_param_combos = [
[1e-5, 1e-6, 0.6, True],
[1e-5, 1e-6, 0.6, False],
[1e-4, 1e-6, 0.6, True],
[1e-4, 1e-6, 0.6, False],
[1e-3, 1e-6, 0.6, True],
[1e-3, 1e-6, 0.6, False],
]
for variable_param_combo in variable_param_combos:
config = dict(tune_config["base_train_config"])
updated_params = dict(zip(variable_params, variable_param_combo))
config = update_config(config, updated_params)
expected_configs.append(dict(config))
# Run training and extract actual configs from results.
results = tune(tune_config)
actual_configs = [
config_results["config"] for config_results in results["iterations"]
]
# Compare actual configs with expected configs. We need to do this in a kind of
# janky way, since we want to allow the possibility that the list of configs are in
# a different order, but we can't make a set of dicts (dicts aren't hashable). We
# test that the lists have the same order, that expected_configs contains unique
# values, and that each element of expected_configs is an element of actual_configs.
assert len(expected_configs) == len(actual_configs)
assert all(c1 != c2
for c1, c2 in itertools.combinations(expected_configs, 2))
for expected_config in expected_configs:
assert expected_config in actual_configs
def test_tune_IC_grid_values() -> None:
"""
Runs hyperparameter IC grid search and makes sure that the correct parameter
combinations are used for training.
"""
# Load hyperparameter search config.
with open(IC_GRID_CONFIG_PATH, "r") as config_file:
config = json.load(config_file)
# Construct expected param value intervals.
param_intervals = {
"initial_lr": [1e-5, 1e-3],
"num_layers": [1, 8],
"recurrent": [True, False],
}
# Run training and extract actual configs from results.
results = tune(config)
actual_configs = [
config_results["config"] for config_results in results["iterations"]
]
# Verify configs from training.
iteration = 0
best_param_vals: Dict[str, float] = {}
for param_name, param_values in param_intervals.items():
param_fitnesses = {}
for param_val in param_values:
# Check values.
for best_param_name, best_param_val in best_param_vals.items():
found, result = dict_search(actual_configs[iteration], best_param_name)
assert found
assert result == best_param_val
# Store fitnesses for comparison.
param_fitnesses[param_val] = results["iterations"][iteration]["fitness"]
iteration += 1
best_param_val = dict_argmax(param_fitnesses)
best_param_vals[param_name] = best_param_val
def test_tune_random_early_stop_iteration() -> None:
"""
Runs hyperparameter random search until an early stop point between iterations.
"""
# Load hyperparameter search config.
with open(RANDOM_CONFIG_PATH, "r") as config_file:
config = json.load(config_file)
# Modify default training config to stop early.
config["early_stop"] = {"iterations": 2, "trials": 0}
# Run training.
results = tune(config)
# Check results.
assert len(results["iterations"]) == config["early_stop"]["iterations"]
for config_results in results["iterations"]:
assert len(config_results["trials"]) == config["trials_per_config"]
def resume_template(
save_name: str,
config_path: str,
early_stops: List[Dict[str, int]],
baseline_name: str,
results_name: str,
) -> None:
"""
Runs while stopping to save/load at a given set of checkpoints, then compares
results against non-interrupted version.
"""
# Load hyperparameter search config.
with open(config_path, "r") as config_file:
config = json.load(config_file)
config["base_train_config"]["save_name"] = save_name
# Set baseline to compare against throughout training.
config["base_train_config"]["baseline_metrics_filename"] = baseline_name
# Ensure that there are no existing saved experiments whose names coincide with the
# experiment names used here. We do have to save and load from disk so we want to
# make sure that we aren't overwriting any previously existing files.
num_param_values = None
if config["search_type"] == "IC_grid":
num_param_values = get_num_param_values(config["search_params"])
iterations = get_iterations(config["search_type"],
config["search_iterations"],
config["search_params"])
check_name_uniqueness(
save_name,
config["search_type"],
iterations,
config["trials_per_config"],
num_param_values=num_param_values,
)
# Run until hitting each early stopping point.
for stop_index in range(len(early_stops)):
# Set early stopping point.
config["early_stop"] = early_stops[stop_index]
# Set loading point, if necessary.
if stop_index > 0:
config["load_from"] = save_name
# Run partial training.
tune(config)
# Finish training from checkpoint.
config["early_stop"] = None
config["load_from"] = save_name
resumed_results = tune(config)
# Compare resumed results with un-interrupted results.
if results_name is not None:
results_path = os.path.join(METRICS_DIR, "%s.json" % results_name)
with open(results_path, "r") as results_file:
correct_results = json.load(results_file)
assert tune_results_equal(resumed_results, correct_results)
# Clean up saved results.
experiment_names = get_experiment_names(
save_name,
config["search_type"],
iterations,
config["trials_per_config"],
num_param_values=num_param_values,
)
for name in experiment_names:
save_dir = save_dir_from_name(name)
if os.path.isdir(save_dir):
rmtree(save_dir)
if __name__ == "__main__":
# Parse config filename from command line arguments.
parser = argparse.ArgumentParser()
parser.add_argument(
"command",
type=str,
help="Command to run. Either 'train' or 'tune'.",
)
parser.add_argument(
"config_filename",
type=str,
help="Name of config file to load from.",
)
args = parser.parse_args()
# Load config file.
with open(args.config_filename, "r") as config_file:
config = json.load(config_file)
# Run specified command.
if args.command == "train":
train(config)
elif args.command == "tune":
tune(config)
elif args.command == "meta_train":
meta_train(config)
else:
raise ValueError("Unsupported command: '%s'" % args.command)