def CreateOptimizer(self,
request: OptimizerService_pb2.CreateOptimizerRequest,
context): # pylint: disable=unused-argument
self.logger.info("Creating Optimizer")
print("CREATING OPTIMIZER")
optimization_problem = OptimizationProblem.from_protobuf(
optimization_problem_pb2=request.OptimizationProblem)
optimizer_config_json = request.OptimizerConfig
if optimizer_config_json is not None and len(
optimizer_config_json) > 0:
optimizer_config = Point.from_json(optimizer_config_json)
else:
optimizer_config = bayesian_optimizer_config_store.default
optimizer = BayesianOptimizer(
optimization_problem=optimization_problem,
optimizer_config=optimizer_config)
optimizer_id = self.get_next_optimizer_id()
# To avoid a race condition we acquire the lock before inserting the lock and the optimizer into their respective
# dictionaries. Otherwise we could end up with a situation where a lock is in the dictionary, but the optimizer
# is not.
optimizer_lock = self._lock_manager.RLock()
with optimizer_lock:
self._optimizer_locks_by_optimizer_id[
optimizer_id] = optimizer_lock
self._optimizers_by_id[optimizer_id] = optimizer
self._ordered_ids.append(optimizer_id)
self.logger.info(f"Created optimizer {optimizer_id}.")
return OptimizerService_pb2.OptimizerHandle(Id=optimizer_id)
def connect_to_existing_remote_optimizer(self, optimizer_info: OptimizerInfo) -> BayesianOptimizerProxy:
"""Connects to an existing optimizer.
Parameters
----------
optimizer_info : OptimizerInfo
Returns
-------
BayesianOptimizerProxy
"""
return BayesianOptimizerProxy(
grpc_channel=self._grpc_channel,
optimization_problem=OptimizerServiceDecoder.decode_optimization_problem(optimizer_info.OptimizationProblem),
optimizer_config=Point.from_json(optimizer_info.OptimizerConfigJsonString),
id=optimizer_info.OptimizerHandle.Id,
logger=self.logger
)
def CreateOptimizer(self, request: CreateOptimizerRequest, context): # pylint: disable=unused-argument
self.logger.info("Creating Optimizer")
optimization_problem = OptimizerServiceDecoder.decode_optimization_problem(
optimization_problem_pb2=request.OptimizationProblem)
optimizer_config_json = request.OptimizerConfig
if optimizer_config_json is not None and len(
optimizer_config_json) > 0:
optimizer_config = Point.from_json(optimizer_config_json)
else:
optimizer_config = bayesian_optimizer_config_store.default
optimizer = BayesianOptimizer(
optimization_problem=optimization_problem,
optimizer_config=optimizer_config)
optimizer_id = self._bayesian_optimizer_store.get_next_optimizer_id()
self._bayesian_optimizer_store.add_optimizer(optimizer_id=optimizer_id,
optimizer=optimizer)
self.logger.info(
f"Created optimizer {optimizer_id} with config: {optimizer.optimizer_config.to_json(indent=2)}"
)
return OptimizerHandle(Id=optimizer_id)
def read_from_disk(target_folder):
"""Mirrors write_to_disk by reading into memory the contents of an OptimizerEvaluationReport from disk."""
optimizer_evaluation_report = OptimizerEvaluationReport()
optimizer_config_file = os.path.join(target_folder,
"optimizer_config.json")
with open(optimizer_config_file, 'r') as in_file:
optimizer_evaluation_report.optimizer_configuration = Point.from_json(
in_file.read())
objective_function_config_file = os.path.join(
target_folder, "objective_function_config.json")
with open(objective_function_config_file, 'r') as in_file:
optimizer_evaluation_report.objective_function_configuration = Point.from_json(
in_file.read())
pickled_optimizers_dir = os.path.join(target_folder,
"pickled_optimizers")
if os.path.exists(pickled_optimizers_dir):
for file_name in os.listdir(pickled_optimizers_dir):
iteration_number, file_extension = file_name.split(".")
assert file_extension == "pickle"
iteration_number = int(iteration_number)
with open(os.path.join(pickled_optimizers_dir, file_name),
'rb') as in_file:
optimizer_evaluation_report.pickled_optimizers_over_time[
iteration_number] = in_file.read()
objective_function_initial_state_file_path = os.path.join(
target_folder, "objective_function_initial_state.pickle")
if os.path.exists(objective_function_initial_state_file_path):
with open(objective_function_initial_state_file_path,
'rb') as in_file:
optimizer_evaluation_report.pickled_objective_function_initial_state = in_file.read(
)
objective_function_final_state_file_path = os.path.join(
target_folder, "objective_function_final_state.pickle")
if os.path.exists(objective_function_final_state_file_path):
with open(objective_function_final_state_file_path,
'rb') as in_file:
optimizer_evaluation_report.pickled_objective_function_final_state = in_file.read(
)
gof_file_path = os.path.join(
target_folder, "regression_model_goodness_of_fit_state.pickle")
if os.path.exists(gof_file_path):
with open(gof_file_path, 'rb') as in_file:
optimizer_evaluation_report.regression_model_fit_state = pickle.load(
in_file)
optima_over_time_file_path = os.path.join(target_folder,
"optima_over_time.pickle")
if os.path.exists(optima_over_time_file_path):
with open(optima_over_time_file_path, 'rb') as in_file:
optimizer_evaluation_report.optima_over_time = pickle.load(
in_file)
pareto_over_time_file_path = os.path.join(target_folder,
"pareto_over_time.pickle")
if os.path.exists(pareto_over_time_file_path):
with open(pareto_over_time_file_path, "rb") as in_file:
optimizer_evaluation_report.pareto_over_time = pickle.load(
in_file)
pareto_volume_over_time_file_path = os.path.join(
target_folder, "pareto_volume_over_time.json")
if os.path.exists(pareto_volume_over_time_file_path):
with open(pareto_volume_over_time_file_path, 'r') as in_file:
optimizer_evaluation_report.pareto_volume_over_time = json.load(
in_file)
execution_info_file_path = os.path.join(target_folder,
"execution_info.json")
if os.path.exists(execution_info_file_path):
with open(execution_info_file_path, 'r') as in_file:
execution_info_dict = json.load(in_file)
optimizer_evaluation_report.success = execution_info_dict[
'success']
optimizer_evaluation_report.num_optimization_iterations = execution_info_dict[
'num_optimization_iterations']
optimizer_evaluation_report.evaluation_frequency = execution_info_dict[
'evaluation_frequency']
optimizer_evaluation_report.exception = execution_info_dict[
'exception']
optimizer_evaluation_report.exception_traceback = execution_info_dict[
'exception_stack_trace']
optimizer_evaluation_report.start_time = datetime.strptime(
execution_info_dict['start_time'],
OptimizerEvaluationReport.DATETIME_FORMAT)
optimizer_evaluation_report.end_time = datetime.strptime(
execution_info_dict['end_time'],
OptimizerEvaluationReport.DATETIME_FORMAT)
return optimizer_evaluation_report
def test_defaults(self):
"""Tests default optimizer configurations against default objective functions."""
optimizer_evaluator_config = optimizer_evaluator_config_store.default
optimizer_evaluator_config.num_iterations = 100
# We want to test this functionality so let's make sure that nobody accidentally disables it in the default config.
#
self.assertTrue(
optimizer_evaluator_config.include_pickled_optimizer_in_report)
self.assertTrue(optimizer_evaluator_config.
include_pickled_objective_function_in_report)
self.assertTrue(
optimizer_evaluator_config.report_regression_model_goodness_of_fit)
self.assertTrue(optimizer_evaluator_config.report_optima_over_time)
self.assertTrue(
optimizer_evaluator_config.include_execution_trace_in_report)
optimizer_config = bayesian_optimizer_config_store.default
objective_function_config = objective_function_config_store.default
print(optimizer_evaluator_config.to_json(indent=2))
print(optimizer_config.to_json(indent=2))
print(objective_function_config.to_json(indent=2))
optimizer_evaluator = OptimizerEvaluator(
optimizer_evaluator_config=optimizer_evaluator_config,
objective_function_config=objective_function_config,
optimizer_config=optimizer_config)
optimizer_evaluation_report = optimizer_evaluator.evaluate_optimizer()
self.assertTrue(optimizer_evaluation_report.success)
with pd.option_context('display.max_columns', 100):
print(
optimizer_evaluation_report.
regression_model_goodness_of_fit_state.
get_goodness_of_fit_dataframe(DataSetType.TRAIN).tail())
for optimum_name, optimum_over_time in optimizer_evaluation_report.optima_over_time.items(
):
print(
"#####################################################################################################"
)
print(optimum_name)
print(optimum_over_time.get_dataframe().tail(10))
print(
"#####################################################################################################"
)
# Now let's do it again with the unpickled optimizer.
#
unpickled_objective_function = pickle.loads(
optimizer_evaluation_report.
pickled_objective_function_initial_state)
unpickled_optimizer = pickle.loads(
optimizer_evaluation_report.pickled_optimizer_initial_state)
optimizer_evaluator_2 = OptimizerEvaluator(
optimizer_evaluator_config=optimizer_evaluator_config,
objective_function=unpickled_objective_function,
optimizer=unpickled_optimizer)
optimizer_evaluation_report_2 = optimizer_evaluator_2.evaluate_optimizer(
)
with pd.option_context('display.max_columns', 100):
print(
optimizer_evaluation_report_2.
regression_model_goodness_of_fit_state.
get_goodness_of_fit_dataframe(DataSetType.TRAIN).tail())
for optimum_name, optimum_over_time in optimizer_evaluation_report_2.optima_over_time.items(
):
print(
"#####################################################################################################"
)
print(optimum_name)
print(optimum_over_time.get_dataframe().tail(10))
print(
"#####################################################################################################"
)
optimizer_evaluation_report.write_to_disk(target_folder=self.temp_dir)
# Validate that all files were created as expected.
#
with open(os.path.join(self.temp_dir,
"execution_info.json")) as in_file:
execution_info = json.load(in_file)
self.assertTrue(execution_info["success"])
self.assertTrue(execution_info["num_optimization_iterations"] ==
optimizer_evaluator_config.num_iterations)
self.assertTrue(execution_info["evaluation_frequency"] ==
optimizer_evaluator_config.evaluation_frequency)
self.assertTrue(execution_info["exception"] is None)
self.assertTrue(execution_info["exception_stack_trace"] is None)
with open(os.path.join(self.temp_dir,
"execution_trace.json")) as in_file:
trace = json.load(in_file)
self.assertTrue(len(trace) > 100)
self.assertTrue(
all(key in trace[0]
for key in ["ts", "name", "ph", "cat", "pid", "tid", "args"]))
with open(os.path.join(self.temp_dir, "goodness_of_fit.pickle"),
"rb") as in_file:
unpickled_gof = pickle.load(in_file)
gof_df = unpickled_gof.get_goodness_of_fit_dataframe()
self.assertTrue(len(gof_df.index) > 0)
self.assertTrue(
all((col_name in gof_df.columns.values
or col_name == "data_set_type")
for col_name in GoodnessOfFitMetrics._fields))
self.assertTrue(
all(gof_df[col_name].is_monotonic for col_name in [
"last_refit_iteration_number", "observation_count",
"prediction_count"
]))
with open(os.path.join(self.temp_dir, "optima_over_time.pickle"),
"rb") as in_file:
unpickled_optima_over_time = pickle.load(in_file)
for key, optimum_over_time in unpickled_optima_over_time.items():
self.assertTrue(optimizer_evaluation_report.optima_over_time[key].
get_dataframe().equals(
optimum_over_time.get_dataframe()))
with open(os.path.join(self.temp_dir,
"optimizer_config.json")) as in_file:
optimizer_config_json_str = in_file.read()
deserialized_optimizer_config = Point.from_json(
optimizer_config_json_str)
self.assertTrue(deserialized_optimizer_config == optimizer_config)
with open(os.path.join(self.temp_dir,
"objective_function_config.json")) as in_file:
objective_function_json_str = in_file.read()
deserialized_objective_function_config = Point.from_json(
objective_function_json_str)
self.assertTrue(deserialized_objective_function_config ==
objective_function_config)
objective_function_pickle_file_names = [
"objective_function_final_state.pickle",
"objective_function_initial_state.pickle"
]
for file_name in objective_function_pickle_file_names:
with open(os.path.join(self.temp_dir, file_name), "rb") as in_file:
unpickled_objective_function = pickle.load(in_file)
self.assertTrue(
unpickled_objective_function.objective_function_config ==
objective_function_config.polynomial_objective_config)
for _ in range(10):
random_pt = unpickled_objective_function.parameter_space.random(
)
self.assertTrue(
unpickled_objective_function.evaluate_point(random_pt)
in unpickled_objective_function.output_space)
# Lastly let's double check the pickled optimizers
#
pickled_optimizers_dir = os.path.join(self.temp_dir,
"pickled_optimizers")
num_pickled_optimizers = len([
name for name in os.listdir(pickled_optimizers_dir)
if os.path.isfile(os.path.join(pickled_optimizers_dir, name))
])
self.assertTrue(num_pickled_optimizers == 11)
# Finally, let's make sure that the optimizers serialized to disk are usable.
#
with open(os.path.join(pickled_optimizers_dir, "99.pickle"),
"rb") as in_file:
final_optimizer_from_disk = pickle.load(in_file)
final_optimizer_from_report = pickle.loads(
optimizer_evaluation_report.pickled_optimizers_over_time[99])
for _ in range(100):
self.assertTrue(final_optimizer_from_disk.suggest() ==
final_optimizer_from_report.suggest())