def test_load_tuning_config():
testdict = {
"engines": [
{
"command": "lc0",
"fixed_parameters": {
"CPuctBase": 13232,
"Threads": 2
}
},
{
"command": "sf",
"fixed_parameters": {
"Threads": 8
}
},
],
"parameter_ranges": {
"CPuct": "Real(0.0, 1.0)"
},
"gp_samples":
100,
}
json_dict, commands, fixed_params, param_ranges = load_tuning_config(
testdict)
assert len(json_dict) == 1
assert "gp_samples" in json_dict
assert len(commands) == 2
assert len(fixed_params) == 2
assert len(param_ranges) == 1
def local( # noqa: C901
tuning_config,
acq_function="mes",
acq_function_samples=1,
confidence=0.9,
data_path=None,
gp_burnin=5,
gp_samples=300,
gp_initial_burnin=100,
gp_initial_samples=300,
logfile="log.txt",
n_initial_points=30,
n_points=500,
plot_every=5,
plot_path="plots",
random_seed=0,
result_every=5,
resume=True,
verbose=False,
):
"""Run a local tune.
Parameters defined in the `tuning_config` file always take precedence.
"""
json_dict = json.load(tuning_config)
settings, commands, fixed_params, param_ranges = load_tuning_config(json_dict)
log_level = logging.DEBUG if verbose else logging.INFO
log_format = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
root_logger = logging.getLogger()
root_logger.setLevel(log_level)
file_logger = logging.FileHandler(settings.get("logfile", logfile))
file_logger.setFormatter(log_format)
root_logger.addHandler(file_logger)
console_logger = logging.StreamHandler(sys.stdout)
console_logger.setFormatter(log_format)
root_logger.addHandler(console_logger)
logging.debug(f"Got the following tuning settings:\n{json_dict}")
# 1. Create seed sequence
ss = np.random.SeedSequence(settings.get("random_seed", random_seed))
# 2. Create kernel
# 3. Create optimizer
random_state = np.random.RandomState(np.random.MT19937(ss.spawn(1)[0]))
opt = Optimizer(
dimensions=list(param_ranges.values()),
n_points=settings.get("n_points", n_points),
n_initial_points=settings.get("n_initial_points", n_initial_points),
# gp_kernel=kernel, # TODO: Let user pass in different kernels
gp_kwargs=dict(normalize_y=True),
# gp_priors=priors, # TODO: Let user pass in priors
acq_func=settings.get("acq_function", acq_function),
acq_func_kwargs=dict(alpha="inf", n_thompson=20),
random_state=random_state,
)
X = []
y = []
noise = []
iteration = 0
# 3.1 Resume from existing data:
if data_path is None:
data_path = "data.npz"
if resume:
path = pathlib.Path(data_path)
if path.exists():
with np.load(path) as importa:
X = importa["arr_0"].tolist()
y = importa["arr_1"].tolist()
noise = importa["arr_2"].tolist()
if len(X[0]) != opt.space.n_dims:
logging.error(
"The number of parameters are not matching the number of "
"dimensions. Rename the existing data file or ensure that the "
"parameter ranges are correct."
)
sys.exit(1)
reduction_needed, X_reduced, y_reduced, noise_reduced = reduce_ranges(
X, y, noise, opt.space
)
if reduction_needed:
backup_path = path.parent / (
path.stem + f"_backup_{int(time.time())}" + path.suffix
)
logging.warning(
f"The parameter ranges are smaller than the existing data. "
f"Some points will have to be discarded. "
f"The original {len(X)} data points will be saved to "
f"{backup_path}"
)
np.savez_compressed(
backup_path, np.array(X), np.array(y), np.array(noise)
)
X = X_reduced
y = y_reduced
noise = noise_reduced
iteration = len(X)
logging.info(
f"Importing {iteration} existing datapoints. This could take a while..."
)
opt.tell(
X,
y,
noise_vector=noise,
gp_burnin=settings.get("gp_initial_burnin", gp_initial_burnin),
gp_samples=settings.get("gp_initial_samples", gp_initial_samples),
n_samples=settings.get("n_samples", 1),
progress=True,
)
logging.info("Importing finished.")
# 4. Main optimization loop:
while True:
logging.info("Starting iteration {}".format(iteration))
result_every_n = settings.get("result_every", result_every)
if (
result_every_n > 0
and iteration % result_every_n == 0
and opt.gp.chain_ is not None
):
result_object = create_result(Xi=X, yi=y, space=opt.space, models=[opt.gp])
try:
best_point, best_value = expected_ucb(result_object, alpha=0.0)
best_point_dict = dict(zip(param_ranges.keys(), best_point))
logging.info(f"Current optimum:\n{best_point_dict}")
logging.info(f"Estimated value: {best_value}")
confidence_val = settings.get("confidence", confidence)
confidence_out = confidence_intervals(
optimizer=opt,
param_names=list(param_ranges.keys()),
hdi_prob=confidence_val,
opt_samples=1000,
multimodal=False,
)
logging.info(
f"{confidence_val*100}% confidence intervals:\n{confidence_out}"
)
except ValueError:
logging.info(
"Computing current optimum was not successful. "
"This can happen in rare cases and running the "
"tuner again usually works."
)
plot_every_n = settings.get("plot_every", plot_every)
if (
plot_every_n > 0
and iteration % plot_every_n == 0
and opt.gp.chain_ is not None
):
logging.getLogger("matplotlib.font_manager").disabled = True
if opt.space.n_dims == 1:
logging.warning(
"Plotting for only 1 parameter is not supported yet."
)
else:
logging.debug("Starting to compute the next plot.")
result_object = create_result(
Xi=X, yi=y, space=opt.space, models=[opt.gp]
)
plt.style.use("dark_background")
fig, ax = plt.subplots(
nrows=opt.space.n_dims,
ncols=opt.space.n_dims,
figsize=(3 * opt.space.n_dims, 3 * opt.space.n_dims),
)
fig.patch.set_facecolor("#36393f")
for i in range(opt.space.n_dims):
for j in range(opt.space.n_dims):
ax[i, j].set_facecolor("#36393f")
timestr = time.strftime("%Y%m%d-%H%M%S")
plot_objective(
result_object, dimensions=list(param_ranges.keys()), fig=fig, ax=ax
)
plotpath = pathlib.Path(settings.get("plot_path", plot_path))
plotpath.mkdir(parents=True, exist_ok=True)
full_plotpath = plotpath / f"{timestr}-{iteration}.png"
plt.savefig(
full_plotpath,
pad_inches=0.1,
dpi=300,
bbox_inches="tight",
facecolor="#36393f",
)
logging.info(f"Saving a plot to {full_plotpath}.")
plt.close(fig)
point = opt.ask()
point_dict = dict(zip(param_ranges.keys(), point))
logging.info("Testing {}".format(point_dict))
engine_json = prepare_engines_json(commands=commands, fixed_params=fixed_params)
logging.debug(f"engines.json is prepared:\n{engine_json}")
write_engines_json(engine_json, point_dict)
logging.info("Start experiment")
now = datetime.now()
out_exp, out_exp_err = run_match(**settings)
later = datetime.now()
difference = (later - now).total_seconds()
logging.info(f"Experiment finished ({difference}s elapsed).")
logging.debug(f"Raw result:\n{out_exp}\n{out_exp_err}")
score, error = parse_experiment_result(out_exp, **settings)
logging.info("Got score: {} +- {}".format(score, error))
logging.info("Updating model")
while True:
try:
now = datetime.now()
# We fetch kwargs manually here to avoid collisions:
n_samples = settings.get("acq_function_samples", acq_function_samples)
gp_burnin = settings.get("gp_burnin", gp_burnin)
gp_samples = settings.get("gp_samples", gp_samples)
if opt.gp.chain_ is None:
gp_burnin = settings.get("gp_initial_burnin", gp_initial_burnin)
gp_samples = settings.get("gp_initial_samples", gp_initial_samples)
opt.tell(
point,
score,
n_samples=n_samples,
gp_samples=gp_samples,
gp_burnin=gp_burnin,
)
else:
opt.tell(
point,
score,
n_samples=n_samples,
gp_samples=gp_samples,
gp_burnin=gp_burnin,
)
later = datetime.now()
difference = (later - now).total_seconds()
logging.info(f"GP sampling finished ({difference}s)")
logging.debug(f"GP kernel: {opt.gp.kernel_}")
except ValueError:
logging.warning(
"Error encountered during fitting. Trying to sample chain a bit. "
"If this problem persists, restart the tuner to reinitialize."
)
opt.gp.sample(n_burnin=5, priors=opt.gp_priors)
else:
break
X.append(point)
y.append(score)
noise.append(error)
iteration = len(X)
with AtomicWriter(data_path, mode="wb", overwrite=True).open() as f:
np.savez_compressed(f, np.array(X), np.array(y), np.array(noise))
def local( # noqa: C901
tuning_config,
acq_function="mes",
acq_function_samples=1,
acq_function_lcb_alpha=1.96,
confidence=0.9,
data_path=None,
gp_burnin=5,
gp_samples=300,
gp_initial_burnin=100,
gp_initial_samples=300,
#kernel_lengthscale_prior_lower_bound=0.1,
#kernel_lengthscale_prior_upper_bound=0.5,
#kernel_lengthscale_prior_lower_steepness=2.0,
#kernel_lengthscale_prior_upper_steepness=1.0,
gp_signal_prior_scale=4.0,
gp_noise_prior_scale=0.0006,
gp_lengthscale_prior_lb=0.1,
gp_lengthscale_prior_ub=0.5,
normalize_y=True,
noise_scaling_coefficient=1,
logfile="log.txt",
n_initial_points=16,
n_points=500,
plot_every=1,
plot_path="plots",
plot_on_resume=False,
random_seed=0,
result_every=1,
resume=True,
fast_resume=True,
model_path="model.pkl",
point=None,
reset=False,
verbose=0,
warp_inputs=True,
rounds=10,
):
"""Run a local tune.
Parameters defined in the `tuning_config` file always take precedence.
"""
json_dict = json.load(tuning_config)
settings, commands, fixed_params, param_ranges = load_tuning_config(json_dict)
root_logger = setup_logger(
verbose=verbose, logfile=settings.get("logfile", logfile)
)
root_logger.debug(f"Got the following tuning settings:\n{json_dict}")
root_logger.debug(
f"Acquisition function: {acq_function}, Acquisition function samples: {acq_function_samples}, Acquisition function lcb alpha: {acq_function_lcb_alpha}, GP burnin: {gp_burnin}, GP samples: {gp_samples}, GP initial burnin: {gp_initial_burnin}, GP initial samples: {gp_initial_samples}, GP signal prior scale: {gp_signal_prior_scale}, GP noise prior scale: {gp_noise_prior_scale}, GP lengthscale prior lower bound: {gp_lengthscale_prior_lb}, GP lengthscale prior upper bound: {gp_lengthscale_prior_ub}, Warp inputs: {warp_inputs}, Normalize y: {normalize_y}, Noise scaling coefficient: {noise_scaling_coefficient}, Initial points: {n_initial_points}, Next points: {n_points}, Random seed: {random_seed}"
)
#root_logger.debug(
#f"Acquisition function: {acq_function}, Acquisition function samples: {acq_function_samples}, GP burnin: {gp_burnin}, GP samples: {gp_samples}, GP initial burnin: {gp_initial_burnin}, GP initial samples: {gp_initial_samples}, Kernel lengthscale prior lower bound: {kernel_lengthscale_prior_lower_bound}, Kernel lengthscale prior upper bound: {kernel_lengthscale_prior_upper_bound}, Kernel lengthscale prior lower steepness: {kernel_lengthscale_prior_lower_steepness}, Kernel lengthscale prior upper steepness: {kernel_lengthscale_prior_upper_steepness}, Warp inputs: {warp_inputs}, Normalize y: {normalize_y}, Noise scaling coefficient: {noise_scaling_coefficient}, Initial points: {n_initial_points}, Next points: {n_points}, Random seed: {random_seed}"
#)
root_logger.debug(
f"Chess Tuning Tools version: {importlib.metadata.version('chess-tuning-tools')}, Bayes-skopt version: {importlib.metadata.version('bask')}, Scikit-optimize version: {importlib.metadata.version('scikit-optimize')}, Scikit-learn version: {importlib.metadata.version('scikit-learn')}, SciPy version: {importlib.metadata.version('scipy')}"
)
#root_logger.debug(
#f"Chess Tuning Tools version: {pkg_resources.get_distribution('chess-tuning-tools').parsed_version}"
#)
# Initialize/import data structures:
if data_path is None:
data_path = "data.npz"
intermediate_data_path = data_path.replace(".", "_intermediate.", 1)
try:
X, y, noise, iteration, round, counts_array, point = initialize_data(
parameter_ranges=list(param_ranges.values()),
resume=resume,
data_path=data_path,
intermediate_data_path=intermediate_data_path,
)
except ValueError:
root_logger.error(
"The number of parameters are not matching the number of "
"dimensions. Rename the existing data file or ensure that the "
"parameter ranges are correct."
)
sys.exit(1)
# Initialize Optimizer object and if applicable, resume from existing
# data/optimizer:
gp_priors = create_priors(
n_parameters=len(param_ranges),
signal_scale=settings.get("gp_signal_prior_scale", gp_signal_prior_scale),
lengthscale_lower_bound=settings.get(
"gp_lengthscale_prior_lb", gp_lengthscale_prior_lb
),
lengthscale_upper_bound=settings.get(
"gp_lengthscale_prior_ub", gp_lengthscale_prior_ub
),
noise_scale=settings.get("gp_noise_prior_scale", gp_noise_prior_scale),
)
opt = initialize_optimizer(
X=X,
y=y,
noise=noise,
parameter_ranges=list(param_ranges.values()),
noise_scaling_coefficient=noise_scaling_coefficient,
random_seed=settings.get("random_seed", random_seed),
warp_inputs=settings.get("warp_inputs", warp_inputs),
normalize_y=settings.get("normalize_y", normalize_y),
#kernel_lengthscale_prior_lower_bound=settings.get("kernel_lengthscale_prior_lower_bound", kernel_lengthscale_prior_lower_bound),
#kernel_lengthscale_prior_upper_bound=settings.get("kernel_lengthscale_prior_upper_bound", kernel_lengthscale_prior_upper_bound),
#kernel_lengthscale_prior_lower_steepness=settings.get("kernel_lengthscale_prior_lower_steepness", kernel_lengthscale_prior_lower_steepness),
#kernel_lengthscale_prior_upper_steepness=settings.get("kernel_lengthscale_prior_upper_steepness", kernel_lengthscale_prior_upper_steepness),
n_points=settings.get("n_points", n_points),
n_initial_points=settings.get("n_initial_points", n_initial_points),
acq_function=settings.get("acq_function", acq_function),
acq_function_samples=settings.get("acq_function_samples", acq_function_samples),
acq_function_lcb_alpha=settings.get(
"acq_function_lcb_alpha", acq_function_lcb_alpha
),
resume=resume,
fast_resume=fast_resume,
model_path=model_path,
gp_initial_burnin=settings.get("gp_initial_burnin", gp_initial_burnin),
gp_initial_samples=settings.get("gp_initial_samples", gp_initial_samples),
gp_priors=gp_priors,
)
is_first_iteration_after_program_start = True
# Main optimization loop:
while True:
if round == 0:
root_logger.info("Starting iteration {}".format(iteration))
else:
root_logger.info("Resuming iteration {}".format(iteration))
# If a model has been fit, print/plot results so far:
if len(y) > 0 and opt.gp.chain_ is not None:
result_object = create_result(Xi=X, yi=y, space=opt.space, models=[opt.gp])
#root_logger.debug(f"result_object:\n{result_object}")
result_every_n = settings.get("result_every", result_every)
if result_every_n > 0 and iteration % result_every_n == 0:
print_results(
optimizer=opt,
result_object=result_object,
parameter_names=list(param_ranges.keys()),
confidence=settings.get("confidence", confidence),
)
plot_every_n = settings.get("plot_every", plot_every)
if (
plot_every_n > 0
and iteration % plot_every_n == 0
and (not is_first_iteration_after_program_start or plot_on_resume)
):
plot_results(
optimizer=opt,
result_object=result_object,
plot_path=settings.get("plot_path", plot_path),
parameter_names=list(param_ranges.keys()),
)
if point is None:
round = 0 # If previous tested point is not present, start over iteration.
counts_array = np.array([0, 0, 0, 0, 0])
if round == 0:
point = opt.ask() # Ask optimizer for next point.
point_dict = dict(zip(param_ranges.keys(), point))
root_logger.info("Testing {}".format(point_dict))
if len(y) > 0 and opt.gp.chain_ is not None:
testing_current_value = opt.gp.predict(opt.space.transform([point]))
with opt.gp.noise_set_to_zero():
_, testing_current_std = opt.gp.predict(
opt.space.transform([point]), return_std=True
)
root_logger.debug(
f"Predicted Elo: {np.around(-testing_current_value[0] * 100, 4)} +- "
f"{np.around(testing_current_std * 100, 4).item()}"
)
confidence_mult = erfinv(confidence) * np.sqrt(2)
lower_bound = np.around(
-testing_current_value * 100
- confidence_mult * testing_current_std * 100,
4,
).item()
upper_bound = np.around(
-testing_current_value * 100
+ confidence_mult * testing_current_std * 100,
4,
).item()
root_logger.debug(
f"{confidence * 100}% confidence interval of the Elo value: "
f"({lower_bound}, "
f"{upper_bound})"
)
root_logger.info("Start experiment")
else:
point_dict = dict(zip(param_ranges.keys(), point))
root_logger.info("Testing {}".format(point_dict))
if len(y) > 0 and opt.gp.chain_ is not None:
testing_current_value = opt.gp.predict(opt.space.transform([point]))
with opt.gp.noise_set_to_zero():
_, testing_current_std = opt.gp.predict(
opt.space.transform([point]), return_std=True
)
root_logger.debug(
f"Predicted Elo: {np.around(-testing_current_value[0] * 100, 4)} +- "
f"{np.around(testing_current_std * 100, 4).item()}"
)
confidence_mult = erfinv(confidence) * np.sqrt(2)
lower_bound = np.around(
-testing_current_value * 100
- confidence_mult * testing_current_std * 100,
4,
).item()
upper_bound = np.around(
-testing_current_value * 100
+ confidence_mult * testing_current_std * 100,
4,
).item()
root_logger.debug(
f"{confidence * 100}% confidence interval of the Elo value: "
f"({lower_bound}, "
f"{upper_bound})"
)
root_logger.info("Continue experiment")
# Run experiment:
now = datetime.now()
#settings["debug_mode"] = settings.get(
#"debug_mode", False if verbose <= 1 else True
#)
while round < settings.get("rounds", rounds):
round += 1
if round > 1:
root_logger.debug(
f"WW, WD, WL/DD, LD, LL experiment counts: {counts_array}"
)
score, error_variance = counts_to_penta(counts=counts_array)
root_logger.info(
"Experiment Elo so far: {} +- {}".format(
-score * 100, np.sqrt(error_variance) * 100
)
)
root_logger.debug(f"Round: {round}")
settings, commands, fixed_params, param_ranges = load_tuning_config(
json_dict
)
# Prepare engines.json file for cutechess-cli:
engine_json = prepare_engines_json(
commands=commands, fixed_params=fixed_params
)
root_logger.debug(f"engines.json is prepared:\n{engine_json}")
write_engines_json(engine_json, point_dict)
out_exp = []
out_all = []
for output_line in run_match(
**settings, tuning_config_name=tuning_config.name
):
line = output_line.rstrip()
is_debug = is_debug_log(line)
if is_debug and verbose > 2:
root_logger.debug(line)
if not is_debug:
out_exp.append(line)
out_all.append(line)
check_log_for_errors(cutechess_output=out_all)
out_exp = "\n".join(out_exp)
(
match_score,
match_error_variance,
match_counts_array,
) = parse_experiment_result(out_exp, **settings)
counts_array += match_counts_array
with AtomicWriter(
intermediate_data_path, mode="wb", overwrite=True
).open() as f:
np.savez_compressed(f, np.array(round), counts_array, point)
later = datetime.now()
difference = (later - now).total_seconds()
root_logger.info(f"Experiment finished ({difference}s elapsed).")
# Parse cutechess-cli output and report results (Elo and standard deviation):
root_logger.debug(f"WW, WD, WL/DD, LD, LL experiment counts: {counts_array}")
score, error_variance = counts_to_penta(counts=counts_array)
root_logger.info(
"Got Elo: {} +- {}".format(-score * 100, np.sqrt(error_variance) * 100)
)
X.append(point)
y.append(score)
noise.append(error_variance)
# Update data structures and persist to disk:
with AtomicWriter(data_path, mode="wb", overwrite=True).open() as f:
np.savez_compressed(f, np.array(X), np.array(y), np.array(noise))
with AtomicWriter(model_path, mode="wb", overwrite=True).open() as f:
dill.dump(opt, f)
round = 0
counts_array = np.array([0, 0, 0, 0, 0])
with AtomicWriter(
intermediate_data_path, mode="wb", overwrite=True
).open() as f:
np.savez_compressed(f, np.array(round), counts_array, point)
# Update model with the new data:
if reset:
root_logger.info("Deleting the model and generating a new one.")
# Reset optimizer.
del opt
if acq_function == "rand":
current_acq_func = random.choice(["mes", "pvrs", "ei", "lcb", "ts"])
root_logger.debug(
f"Current random acquisition function: {current_acq_func}"
)
else:
current_acq_func = acq_function
opt = initialize_optimizer(
X=X,
y=y,
noise=noise,
parameter_ranges=list(param_ranges.values()),
noise_scaling_coefficient=noise_scaling_coefficient,
random_seed=settings.get("random_seed", random_seed),
warp_inputs=settings.get("warp_inputs", warp_inputs),
normalize_y=settings.get("normalize_y", normalize_y),
#kernel_lengthscale_prior_lower_bound=settings.get("kernel_lengthscale_prior_lower_bound", kernel_lengthscale_prior_lower_bound),
#kernel_lengthscale_prior_upper_bound=settings.get("kernel_lengthscale_prior_upper_bound", kernel_lengthscale_prior_upper_bound),
#kernel_lengthscale_prior_lower_steepness=settings.get("kernel_lengthscale_prior_lower_steepness", kernel_lengthscale_prior_lower_steepness),
#kernel_lengthscale_prior_upper_steepness=settings.get("kernel_lengthscale_prior_upper_steepness", kernel_lengthscale_prior_upper_steepness),
n_points=settings.get("n_points", n_points),
n_initial_points=settings.get("n_initial_points", n_initial_points),
acq_function=current_acq_func,
acq_function_samples=settings.get(
"acq_function_samples", acq_function_samples
),
acq_function_lcb_alpha=settings.get(
"acq_function_lcb_alpha", acq_function_lcb_alpha
),
resume=True,
fast_resume=False,
model_path=None,
gp_initial_burnin=settings.get("gp_burnin", gp_burnin),
gp_initial_samples=settings.get("gp_samples", gp_samples),
)
else:
root_logger.info("Updating model.")
if acq_function == "rand":
opt.acq_func = ACQUISITION_FUNC[
random.choice(["mes", "pvrs", "ei", "lcb", "ts"])
]
root_logger.debug(
f"Current random acquisition function: {opt.acq_func}"
)
update_model(
optimizer=opt,
point=point,
score=score,
variance=error_variance,
noise_scaling_coefficient=noise_scaling_coefficient,
acq_function_samples=settings.get(
"acq_function_samples", acq_function_samples
),
acq_function_lcb_alpha=settings.get(
"acq_function_lcb_alpha", acq_function_lcb_alpha
),
gp_burnin=settings.get("gp_burnin", gp_burnin),
gp_samples=settings.get("gp_samples", gp_samples),
gp_initial_burnin=settings.get("gp_initial_burnin", gp_initial_burnin),
gp_initial_samples=settings.get(
"gp_initial_samples", gp_initial_samples
),
)
iteration = len(X)
is_first_iteration_after_program_start = False
#with AtomicWriter(data_path, mode="wb", overwrite=True).open() as f:
#np.savez_compressed(f, np.array(X), np.array(y), np.array(noise))
with AtomicWriter(model_path, mode="wb", overwrite=True).open() as f:
dill.dump(opt, f)