def __init__(self,
optimizer_evaluator_config: Point,
optimizer: OptimizerBase = None,
optimizer_config: Point = None,
objective_function: ObjectiveFunctionBase = None,
objective_function_config: Point = None):
assert optimizer_evaluator_config in optimizer_evaluator_config_store.parameter_space
assert (optimizer is None) != (
optimizer_config is None
), "A valid optimizer XOR a valid optimizer_config must be supplied."
assert (objective_function is None) != (objective_function_config is None),\
"A valid objective_function XOR a valid objective_function_config must be specified"
self.optimizer_evaluator_config = optimizer_evaluator_config
self.objective_function_config = None
self.objective_function = None
self.optimizer_config = None
self.optimizer = None
# Let's get the objective function assigned to self's fields.
#
if (objective_function_config
is not None) and (objective_function is None):
assert objective_function_config in objective_function_config_store.parameter_space
self.objective_function_config = objective_function_config
self.objective_function = ObjectiveFunctionFactory.create_objective_function(
objective_function_config)
elif (objective_function
is not None) and (objective_function_config is None):
self.objective_function_config = objective_function.objective_function_config
self.objective_function = objective_function
else:
# The assert above should have caught it but just in case someone removes or changes it.
#
assert False, "A valid objective_function XOR a valid objective_function_config must be specified"
# Let's get the optimizer and its config assigned to self's fields.
#
if (optimizer_config is not None) and (optimizer is None):
assert optimizer_config in bayesian_optimizer_config_store.parameter_space
optimization_problem = self.objective_function.default_optimization_problem
self.optimizer_config = optimizer_config
self.optimizer = BayesianOptimizerFactory().create_local_optimizer(
optimizer_config=optimizer_config,
optimization_problem=optimization_problem)
elif (optimizer is not None) and (optimizer_config is None):
# TODO: assert that the optimization problem in the optimizer matches the objective function.
# But this requires Hypergrid.__eq__.
#
self.optimizer_config = optimizer.optimizer_config
self.optimizer = optimizer
else:
# Again, the assert at the beginning of the constructor should have caught this. But more asserts => less bugs.
#
assert False, "A valid optimizer XOR a valid optimizer_config must be supplied."
def __init__(self, grpc_channel, logger=None):
self.logger = logger if logger is not None else create_logger(
"OptimizerMonitor")
self._grpc_channel = grpc_channel
self._optimizer_monitoring_stub = OptimizerMonitoringServiceStub(
channel=self._grpc_channel)
self._optimizer_factory = BayesianOptimizerFactory(
grpc_channel=self._grpc_channel, logger=self.logger)
def setUp(self):
mlos_globals.init_mlos_global_context()
mlos_globals.mlos_global_context.start_clock()
self.logger = create_logger('TestSmartCacheWithRemoteOptimizer')
self.logger.level = logging.DEBUG
# Start up the gRPC service.
#
self.server = OptimizerMicroserviceServer(port=50051, num_threads=10)
self.server.start()
self.optimizer_service_grpc_channel = grpc.insecure_channel('localhost:50051')
self.bayesian_optimizer_factory = BayesianOptimizerFactory(grpc_channel=self.optimizer_service_grpc_channel, logger=self.logger)
self.mlos_agent = MlosAgent(
logger=self.logger,
communication_channel=mlos_globals.mlos_global_context.communication_channel,
shared_config=mlos_globals.mlos_global_context.shared_config,
bayesian_optimizer_grpc_channel=self.optimizer_service_grpc_channel
)
self.mlos_agent_thread = Thread(target=self.mlos_agent.run)
self.mlos_agent_thread.start()
global_values.declare_singletons() # TODO: having both globals and global_values is a problem
# Let's add the allowed component types
self.mlos_agent.add_allowed_component_type(SmartCache)
self.mlos_agent.add_allowed_component_type(SmartCacheWorkloadGenerator)
self.mlos_agent.set_configuration(
component_type=SmartCacheWorkloadGenerator,
new_config_values=Point(
workload_type='cyclical_key_from_range',
cyclical_key_from_range_config=Point(
min=0,
range_width=2048
)
)
)
# Let's create the workload
self.smart_cache_workload = SmartCacheWorkloadGenerator(logger=self.logger)
self.optimizer = None
self.working_set_size_estimator = WorkingSetSizeEstimator()
self.hit_rate_monitor = HitRateMonitor()
self.smart_cache_experiment = MlosExperiment(
smart_component_types=[SmartCache],
telemetry_aggregators=[self.working_set_size_estimator, self.hit_rate_monitor]
)
self.optimization_problem = OptimizationProblem(
parameter_space=SmartCache.parameter_search_space,
objective_space=SimpleHypergrid(name="objectives", dimensions=[ContinuousDimension(name="hit_rate", min=0, max=1)]),
objectives=[Objective(name="hit_rate", minimize=False)]
)
def setUp(self):
self.logger = create_logger(self.__class__.__name__)
# Start up the gRPC service.
#
self.server = OptimizerMicroserviceServer(port=50051, num_threads=10)
self.server.start()
self.optimizer_service_channel = grpc.insecure_channel('localhost:50051')
self.bayesian_optimizer_factory = BayesianOptimizerFactory(grpc_channel=self.optimizer_service_channel, logger=self.logger)
self.optimizer_monitor = OptimizerMonitor(grpc_channel=self.optimizer_service_channel, logger=self.logger)
objective_function_config = objective_function_config_store.get_config_by_name('2d_quadratic_concave_up')
self.objective_function = ObjectiveFunctionFactory.create_objective_function(objective_function_config)
self.optimization_problem = OptimizationProblem(
parameter_space=self.objective_function.parameter_space,
objective_space=self.objective_function.output_space,
objectives=[Objective(name='y', minimize=True)]
)
class OptimizerMonitor:
"""Enables monitoring optimizers existing within the OptimizerMicroservice.
"""
def __init__(self, grpc_channel, logger=None):
self.logger = logger if logger is not None else create_logger(
"OptimizerMonitor")
self._grpc_channel = grpc_channel
self._optimizer_monitoring_stub = OptimizerMonitoringServiceStub(
channel=self._grpc_channel)
self._optimizer_factory = BayesianOptimizerFactory(
grpc_channel=self._grpc_channel, logger=self.logger)
def __repr__(self):
return f"OptimizerMonitor(grpc_channel='{self._grpc_channel._channel.target().decode()}')" # pylint: disable=protected-access
def get_existing_optimizers(self):
"""Returns proxies to all existing optimizers.
:return:
"""
request = Empty()
optimizer_list = self._optimizer_monitoring_stub.ListExistingOptimizers(
request)
optimizer_proxies = [
self._optimizer_factory.connect_to_existing_remote_optimizer(
optimizer_info) for optimizer_info in optimizer_list.Optimizers
]
return optimizer_proxies
def get_optimizer_by_id(self, optimizer_id):
"""Returns a proxy to an optimizer with a specified Id.
:param optimizer_id:
:return:
"""
optimizer_handle = OptimizerHandle(Id=optimizer_id)
optimizer_info = self._optimizer_monitoring_stub.GetOptimizerInfo(
optimizer_handle)
optimizer_proxy = self._optimizer_factory.connect_to_existing_remote_optimizer(
optimizer_info)
return optimizer_proxy
def setup_method(self, method):
self.logger = create_logger(self.__class__.__name__)
# Start up the gRPC service. Try a bunch of times before giving up.
#
max_num_tries = 100
num_tries = 0
for port in range(50051, 50051 + max_num_tries):
num_tries += 1
try:
self.server = OptimizerServicesServer(port=port,
num_threads=10)
self.server.start()
self.port = port
break
except:
self.logger.info(
f"Failed to create OptimizerMicroserviceServer on port {port}"
)
if num_tries == max_num_tries:
raise
self.optimizer_service_channel = grpc.insecure_channel(
f'localhost:{self.port}')
self.bayesian_optimizer_factory = BayesianOptimizerFactory(
grpc_channel=self.optimizer_service_channel, logger=self.logger)
self.optimizer_monitor = OptimizerMonitor(
grpc_channel=self.optimizer_service_channel, logger=self.logger)
objective_function_config = objective_function_config_store.get_config_by_name(
'2d_quadratic_concave_up')
self.objective_function = ObjectiveFunctionFactory.create_objective_function(
objective_function_config)
self.optimization_problem = OptimizationProblem(
parameter_space=self.objective_function.parameter_space,
objective_space=self.objective_function.output_space,
objectives=[Objective(name='y', minimize=True)])
def setup_class(cls):
""" Sets up all the singletons needed to test the BayesianOptimizer.
"""
warnings.simplefilter("error")
global_values.declare_singletons()
global_values.tracer = Tracer(actor_id=cls.__name__, thread_id=0)
cls.logger = create_logger(logger_name=cls.__name__)
cls.logger.setLevel(logging.DEBUG)
cls.port = None
# Start up the gRPC service. Try a bunch of ports, before giving up so we can do several in parallel.
#
max_num_tries = 100
num_tries = 0
for port in range(50051, 50051 + max_num_tries):
num_tries += 1
try:
cls.server = OptimizerMicroserviceServer(port=port,
num_threads=10,
logger=cls.logger)
cls.server.start()
cls.port = port
break
except:
cls.logger.info(
f"Failed to create OptimizerMicroserviceServer on port {port}."
)
if num_tries == max_num_tries:
raise
cls.optimizer_service_channel = grpc.insecure_channel(
f'localhost:{cls.port}')
cls.bayesian_optimizer_factory = BayesianOptimizerFactory(
grpc_channel=cls.optimizer_service_channel, logger=cls.logger)
cls.temp_dir = os.path.join(os.getcwd(), "temp")
if not os.path.exists(cls.temp_dir):
os.mkdir(cls.temp_dir)
cls.trace_output_path = os.path.join(
cls.temp_dir, "TestBayesianOptimizerTrace.json")
try:
os.remove(cls.trace_output_path)
except OSError:
pass
def setUpClass(cls):
""" Sets up all the singletons needed to test the BayesianOptimizer.
"""
warnings.simplefilter("error")
global_values.declare_singletons()
global_values.tracer = Tracer(actor_id=cls.__name__, thread_id=0)
cls.logger = create_logger(logger_name=cls.__name__)
# Start up the gRPC service.
#
cls.server = OptimizerMicroserviceServer(port=50051, num_threads=10)
cls.server.start()
cls.optimizer_service_channel = grpc.insecure_channel('localhost:50051')
cls.bayesian_optimizer_factory = BayesianOptimizerFactory(grpc_channel=cls.optimizer_service_channel, logger=cls.logger)
class TestBayesianOptimizerGrpcClient(unittest.TestCase):
""" Tests the E2E Grpc Client-Service workflow.
"""
@classmethod
def setUpClass(cls):
warnings.simplefilter("error")
global_values.declare_singletons()
def setUp(self):
self.logger = create_logger(self.__class__.__name__)
# Start up the gRPC service.
#
self.server = OptimizerMicroserviceServer(port=50051, num_threads=10)
self.server.start()
self.optimizer_service_channel = grpc.insecure_channel('localhost:50051')
self.bayesian_optimizer_factory = BayesianOptimizerFactory(grpc_channel=self.optimizer_service_channel, logger=self.logger)
self.optimizer_monitor = OptimizerMonitor(grpc_channel=self.optimizer_service_channel, logger=self.logger)
objective_function_config = objective_function_config_store.get_config_by_name('2d_quadratic_concave_up')
self.objective_function = ObjectiveFunctionFactory.create_objective_function(objective_function_config)
self.optimization_problem = OptimizationProblem(
parameter_space=self.objective_function.parameter_space,
objective_space=self.objective_function.output_space,
objectives=[Objective(name='y', minimize=True)]
)
def tearDown(self):
""" We need to tear down the gRPC server here.
:return:
"""
self.server.stop(grace=None)
def test_echo(self):
optimizer_service_stub = OptimizerServiceStub(channel=self.optimizer_service_channel)
response = optimizer_service_stub.Echo(Empty())
self.assertTrue(isinstance(response, Empty))
def test_optimizer_with_default_config(self):
pre_existing_optimizers = {optimizer.id: optimizer for optimizer in self.optimizer_monitor.get_existing_optimizers()}
print(bayesian_optimizer_config_store.default)
bayesian_optimizer = self.bayesian_optimizer_factory.create_remote_optimizer(
optimization_problem=self.optimization_problem,
optimizer_config=bayesian_optimizer_config_store.default
)
post_existing_optimizers = {optimizer.id: optimizer for optimizer in self.optimizer_monitor.get_existing_optimizers()}
new_optimizers = {
optimizer_id: optimizer
for optimizer_id, optimizer in post_existing_optimizers.items()
if optimizer_id not in pre_existing_optimizers
}
self.assertTrue(len(new_optimizers) == 1)
new_optimizer_id = list(new_optimizers.keys())[0]
new_optimizer = new_optimizers[new_optimizer_id]
self.assertTrue(new_optimizer_id == bayesian_optimizer.id)
self.assertTrue(new_optimizer.optimizer_config == bayesian_optimizer.optimizer_config)
num_iterations = 100
registered_features_df, registered_objectives_df = self.optimize_quadratic(optimizer=bayesian_optimizer, num_iterations=num_iterations)
# Apparently the to_json/from_json loses precision so we explicitly lose it here so that we can do the comparison.
#
registered_features_json = registered_features_df.to_json(orient='index', double_precision=15)
registered_objectives_json = registered_objectives_df.to_json(orient='index', double_precision=15)
# Apparently the jitter is too good and we actually have to use the json strings or they will be optimized away.
#
assert len(registered_features_json) > 0
assert len(registered_objectives_json) > 0
registered_features_df = pd.read_json(registered_features_json, orient='index')
registered_objectives_df = pd.read_json(registered_objectives_json, orient='index')
observed_features_df, observed_objectives_df = bayesian_optimizer.get_all_observations()
self.assertTrue((np.abs(registered_features_df - observed_features_df) < 0.00000001).all().all())
self.assertTrue((np.abs(registered_objectives_df - observed_objectives_df) < 0.00000001).all().all())
# Let's look at the goodness of fit.
#
random_forest_gof_metrics = bayesian_optimizer.compute_surrogate_model_goodness_of_fit()
# The model might not have used all of the samples, but should have used a majority of them (I expect about 90%), but 70% is a good sanity check
# and should make this test not very flaky.
self.assertTrue(random_forest_gof_metrics.last_refit_iteration_number > 0.7 * num_iterations)
# The invariants below should be true for all surrogate models: the random forest, and all constituent decision trees. So let's iterate over them all.
models_gof_metrics = [random_forest_gof_metrics]
for model_gof_metrics in models_gof_metrics:
self.assertTrue(0 <= model_gof_metrics.relative_absolute_error <= 1) # This could fail if the models are really wrong. Not expected in this unit test though.
self.assertTrue(0 <= model_gof_metrics.relative_squared_error <= 1)
# There is an invariant linking mean absolute error (MAE), root mean squared error (RMSE) and number of observations (n) let's assert it.
n = model_gof_metrics.last_refit_iteration_number
self.assertTrue(model_gof_metrics.mean_absolute_error <= model_gof_metrics.root_mean_squared_error <= math.sqrt(n) * model_gof_metrics.mean_absolute_error)
# We know that the sample confidence interval is wider (or equal to) prediction interval. So hit rates should be ordered accordingly.
self.assertTrue(model_gof_metrics.sample_90_ci_hit_rate >= model_gof_metrics.prediction_90_ci_hit_rate)
self.assertTrue(0 <= model_gof_metrics.coefficient_of_determination <= 1)
def test_optimizer_with_random_config(self):
num_random_restarts = 10
for i in range(num_random_restarts):
optimizer_config = bayesian_optimizer_config_store.parameter_space.random()
optimizer_config.min_samples_required_for_guided_design_of_experiments = min(optimizer_config.min_samples_required_for_guided_design_of_experiments, 100)
if optimizer_config.surrogate_model_implementation == "HomogeneousRandomForestRegressionModel":
rf_config = optimizer_config.homogeneous_random_forest_regression_model_config
rf_config.n_estimators = min(rf_config.n_estimators, 20)
print(f"[{i+1}/{num_random_restarts}] Creating a bayesian optimizer with config: {optimizer_config}")
bayesian_optimizer = self.bayesian_optimizer_factory.create_remote_optimizer(
optimization_problem=self.optimization_problem,
optimizer_config=optimizer_config
)
registered_features_df, registered_objectives_df = self.optimize_quadratic(optimizer=bayesian_optimizer, num_iterations=12)
# Apparently the to_json/from_json loses precision so we explicitly lose it here so that we can do the comparison.
#
registered_features_json = registered_features_df.to_json(orient='index', double_precision=15)
registered_objectives_json = registered_objectives_df.to_json(orient='index', double_precision=15)
# Apparently the jitter is too good and we actually have to use the json strings or they will be optimized away.
#
assert len(registered_features_json) > 0
assert len(registered_objectives_json) > 0
registered_features_df = pd.read_json(registered_features_json, orient='index')
registered_objectives_df = pd.read_json(registered_objectives_json, orient='index')
observed_features_df, observed_objectives_df = bayesian_optimizer.get_all_observations()
self.assertTrue((np.abs(registered_features_df - observed_features_df) < 0.00000001).all().all())
self.assertTrue((np.abs(registered_objectives_df - observed_objectives_df) < 0.00000001).all().all())
@unittest.skip(reason="Not implemented yet.")
def test_optimizer_with_named_config(self):
...
def optimize_quadratic(self, optimizer, num_iterations):
registered_features_df = None
registered_objectives_df = None
old_optimum = np.inf
for i in range(num_iterations):
suggested_params = optimizer.suggest()
suggested_params_df = suggested_params.to_dataframe()
y = self.objective_function.evaluate_point(suggested_params)
optimizer.register(suggested_params_df, y.to_dataframe())
if registered_features_df is None:
registered_features_df = suggested_params_df
else:
registered_features_df = registered_features_df.append(suggested_params_df, ignore_index=True)
if registered_objectives_df is None:
registered_objectives_df = y.to_dataframe()
else:
registered_objectives_df = registered_objectives_df.append(y.to_dataframe(), ignore_index=True)
best_params, optimum = optimizer.optimum()
# ensure current optimum doesn't go up
assert optimum.y <= old_optimum
old_optimum = optimum.y
print(f"[{i+1}/{num_iterations}]Best Params: {best_params}, Best Value: {optimum.y}")
return registered_features_df, registered_objectives_df
class OptimizerEvaluator:
"""Evaluates optimizers against objective functions.
This class is responsible for:
1. instantiating an optimizer
2. instantiating an objective function
3. launching an optimizer against that objective function
4. keeping track of the goodness of fit, and of the various optima over time
5. producing an instance of OptimizerEvaluationReport.
"""
def __init__(
self,
optimizer_evaluator_config: Point,
optimizer: OptimizerBase = None,
optimizer_config: Point = None,
objective_function: ObjectiveFunctionBase = None,
objective_function_config: Point = None,
logger=None
):
assert optimizer_evaluator_config in optimizer_evaluator_config_store.parameter_space
assert (optimizer is None) != (optimizer_config is None), "A valid optimizer XOR a valid optimizer_config must be supplied."
assert (objective_function is None) != (objective_function_config is None),\
"A valid objective_function XOR a valid objective_function_config must be specified"
if logger is None:
logger = create_logger("OptimizerEvaluator")
self.logger = logger
self.optimizer_evaluator_config = optimizer_evaluator_config
self.objective_function_config = None
self.objective_function = None
self.optimizer_config = None
self.optimizer = None
# Let's get the objective function assigned to self's fields.
#
if (objective_function_config is not None) and (objective_function is None):
assert objective_function_config in objective_function_config_store.parameter_space
self.objective_function_config = objective_function_config
self.objective_function = ObjectiveFunctionFactory.create_objective_function(objective_function_config)
elif (objective_function is not None) and (objective_function_config is None):
self.objective_function_config = objective_function.objective_function_config
self.objective_function = objective_function
else:
# The assert above should have caught it but just in case someone removes or changes it.
#
assert False, "A valid objective_function XOR a valid objective_function_config must be specified"
# Let's get the optimizer and its config assigned to self's fields.
#
if (optimizer_config is not None) and (optimizer is None):
assert optimizer_config in bayesian_optimizer_config_store.parameter_space
optimization_problem = self.objective_function.default_optimization_problem
self.optimizer_config = optimizer_config
self.optimizer = BayesianOptimizerFactory().create_local_optimizer(
optimizer_config=optimizer_config,
optimization_problem=optimization_problem
)
elif (optimizer is not None) and (optimizer_config is None):
# TODO: assert that the optimization problem in the optimizer matches the objective function.
# But this requires Hypergrid.__eq__.
#
self.optimizer_config = optimizer.optimizer_config
self.optimizer = optimizer
else:
# Again, the assert at the beginning of the constructor should have caught this. But more asserts => less bugs.
#
assert False, "A valid optimizer XOR a valid optimizer_config must be supplied."
@trace()
def evaluate_optimizer(self) -> OptimizerEvaluationReport: # pylint: disable=too-many-statements,too-many-branches
evaluation_report = OptimizerEvaluationReport(
optimizer_configuration=self.optimizer_config,
objective_function_configuration=self.objective_function_config,
num_optimization_iterations=self.optimizer_evaluator_config.num_iterations,
evaluation_frequency=self.optimizer_evaluator_config.evaluation_frequency
)
if self.optimizer_evaluator_config.include_execution_trace_in_report:
mlos.global_values.declare_singletons()
if mlos.global_values.tracer is None:
mlos.global_values.tracer = Tracer()
mlos.global_values.tracer.clear_events()
if self.optimizer_evaluator_config.include_pickled_objective_function_in_report:
evaluation_report.pickled_objective_function_initial_state = pickle.dumps(self.objective_function)
if self.optimizer_evaluator_config.include_pickled_optimizer_in_report:
evaluation_report.pickled_optimizer_initial_state = pickle.dumps(self.optimizer)
multi_objective_regression_model_fit_state = MultiObjectiveRegressionModelFitState(objective_names=self.optimizer.optimization_problem.objective_names)
for objective_name in self.optimizer.optimization_problem.objective_names:
multi_objective_regression_model_fit_state[objective_name] = RegressionModelFitState()
optima_over_time = {}
optima_over_time[OptimumDefinition.BEST_OBSERVATION.value] = OptimumOverTime(
optimization_problem=self.optimizer.optimization_problem,
optimum_definition=OptimumDefinition.BEST_OBSERVATION
)
optima_over_time[OptimumDefinition.PREDICTED_VALUE_FOR_OBSERVED_CONFIG.value] = OptimumOverTime(
optimization_problem=self.optimizer.optimization_problem,
optimum_definition=OptimumDefinition.PREDICTED_VALUE_FOR_OBSERVED_CONFIG
)
optima_over_time[f"{OptimumDefinition.UPPER_CONFIDENCE_BOUND_FOR_OBSERVED_CONFIG.value}_99"] = OptimumOverTime(
optimization_problem=self.optimizer.optimization_problem,
optimum_definition=OptimumDefinition.UPPER_CONFIDENCE_BOUND_FOR_OBSERVED_CONFIG,
alpha=0.01
)
optima_over_time[f"{OptimumDefinition.LOWER_CONFIDENCE_BOUND_FOR_OBSERVED_CONFIG.value}_99"] = OptimumOverTime(
optimization_problem=self.optimizer.optimization_problem,
optimum_definition=OptimumDefinition.LOWER_CONFIDENCE_BOUND_FOR_OBSERVED_CONFIG,
alpha=0.01
)
#####################################################################################################
evaluation_report.start_time = datetime.utcnow()
i = 0
try:
with traced(scope_name="optimization_loop"):
for i in range(self.optimizer_evaluator_config.num_iterations):
parameters = self.optimizer.suggest()
objectives = self.objective_function.evaluate_point(parameters)
self.optimizer.register(parameters.to_dataframe(), objectives.to_dataframe())
if i % self.optimizer_evaluator_config.evaluation_frequency == 0:
self.logger.info(f"[{i + 1}/{self.optimizer_evaluator_config.num_iterations}]")
with traced(scope_name="evaluating_optimizer"):
if self.optimizer_evaluator_config.include_pickled_optimizer_in_report:
evaluation_report.add_pickled_optimizer(iteration=i, pickled_optimizer=pickle.dumps(self.optimizer))
if self.optimizer.trained:
multi_objective_gof_metrics = self.optimizer.compute_surrogate_model_goodness_of_fit()
for objective_name, gof_metrics in multi_objective_gof_metrics:
multi_objective_regression_model_fit_state[objective_name].set_gof_metrics(
data_set_type=DataSetType.TRAIN,
gof_metrics=gof_metrics
)
for optimum_name, optimum_over_time in optima_over_time.items():
try:
config, value = self.optimizer.optimum(
optimum_definition=optimum_over_time.optimum_definition,
alpha=optimum_over_time.alpha
)
optima_over_time[optimum_name].add_optimum_at_iteration(
iteration=i,
optimum_config=config,
optimum_value=value
)
except ValueError as e:
self.logger.info(f"Failed to get {optimum_name} optimum.", exc_info=True)
if self.optimizer_evaluator_config.report_pareto_over_time:
evaluation_report.pareto_over_time[i] = copy.deepcopy(self.optimizer.optimization_problem)
if self.optimizer_evaluator_config.report_pareto_volume_over_time:
volume_estimator = self.optimizer.pareto_frontier.approximate_pareto_volume()
ci99_on_volume = volume_estimator.get_two_sided_confidence_interval_on_pareto_volume(alpha=0.01)
evaluation_report.pareto_volume_over_time[i] = ci99_on_volume
evaluation_report.success = True
except Exception as e:
evaluation_report.success = False
evaluation_report.exception = e
evaluation_report.exception_traceback = traceback.format_exc()
evaluation_report.end_time = datetime.utcnow()
with traced(scope_name="evaluating_optimizer"):
# Once the optimization is done, we perform a final evaluation of the optimizer.
if self.optimizer.trained:
multi_objective_gof_metrics = self.optimizer.compute_surrogate_model_goodness_of_fit()
for objective_name, gof_metrics in multi_objective_gof_metrics:
multi_objective_regression_model_fit_state[objective_name].set_gof_metrics(data_set_type=DataSetType.TRAIN, gof_metrics=gof_metrics)
for optimum_name, optimum_over_time in optima_over_time.items():
try:
config, value = self.optimizer.optimum(optimum_definition=optimum_over_time.optimum_definition, alpha=optimum_over_time.alpha)
optima_over_time[optimum_name].add_optimum_at_iteration(
iteration=self.optimizer_evaluator_config.num_iterations,
optimum_config=config,
optimum_value=value
)
except Exception as e:
self.logger.info(f"Failed to get {optimum_name} optimum.", exc_info=True)
if self.optimizer_evaluator_config.report_pareto_over_time:
evaluation_report.pareto_over_time[i] = copy.deepcopy(self.optimizer.optimization_problem)
if self.optimizer_evaluator_config.report_pareto_volume_over_time:
volume_estimator = self.optimizer.pareto_frontier.approximate_pareto_volume()
ci99_on_volume = volume_estimator.get_two_sided_confidence_interval_on_pareto_volume(alpha=0.01)
evaluation_report.pareto_volume_over_time[i] = ci99_on_volume
if self.optimizer_evaluator_config.include_execution_trace_in_report:
evaluation_report.execution_trace = mlos.global_values.tracer.trace_events
mlos.global_values.tracer.clear_events()
if self.optimizer_evaluator_config.include_pickled_optimizer_in_report:
evaluation_report.add_pickled_optimizer(iteration=i, pickled_optimizer=pickle.dumps(self.optimizer))
if self.optimizer_evaluator_config.include_pickled_objective_function_in_report:
evaluation_report.pickled_objective_function_final_state = pickle.dumps(self.objective_function)
if self.optimizer_evaluator_config.report_regression_model_goodness_of_fit:
evaluation_report.regression_model_fit_state = multi_objective_regression_model_fit_state
if self.optimizer_evaluator_config.report_optima_over_time:
evaluation_report.optima_over_time = optima_over_time
return evaluation_report
class TestSmartCacheWithRemoteOptimizer:
""" Tests SmartCache that's being tuned by the remote optimizer.
This test will:
1. Instantiate a SmartCache.
2. Create an MlosExperiment that connects to a remote or in-process optimizer.
3. Optimize the SmartCache with the help of the remote or in-process optimizer.
"""
def setup_method(self, method):
mlos_globals.init_mlos_global_context()
mlos_globals.mlos_global_context.start_clock()
self.logger = create_logger('TestSmartCacheWithRemoteOptimizer')
self.logger.level = logging.DEBUG
# Start up the gRPC service. Try a bunch of times before giving up.
#
max_num_tries = 100
num_tries = 0
for port in range(50051, 50051 + max_num_tries):
num_tries += 1
try:
self.server = OptimizerServicesServer(port=port,
num_threads=10)
self.server.start()
self.port = port
break
except:
self.logger.info(
f"Failed to create OptimizerMicroserviceServer on port {port}"
)
if num_tries == max_num_tries:
raise
self.optimizer_service_channel = grpc.insecure_channel(
f'localhost:{self.port}')
self.bayesian_optimizer_factory = BayesianOptimizerFactory(
grpc_channel=self.optimizer_service_channel, logger=self.logger)
self.mlos_agent = MlosAgent(
logger=self.logger,
communication_channel=mlos_globals.mlos_global_context.
communication_channel,
shared_config=mlos_globals.mlos_global_context.shared_config,
bayesian_optimizer_grpc_channel=self.optimizer_service_channel)
self.mlos_agent_thread = Thread(target=self.mlos_agent.run)
self.mlos_agent_thread.start()
global_values.declare_singletons(
) # TODO: having both globals and global_values is a problem
# Let's add the allowed component types
self.mlos_agent.add_allowed_component_type(SmartCache)
self.mlos_agent.add_allowed_component_type(SmartCacheWorkloadGenerator)
self.mlos_agent.set_configuration(
component_type=SmartCacheWorkloadGenerator,
new_config_values=Point(workload_type='cyclical_key_from_range',
cyclical_key_from_range_config=Point(
min=0, range_width=2048)))
# Let's create the workload
self.smart_cache_workload = SmartCacheWorkloadGenerator(
logger=self.logger)
self.optimizer = None
self.working_set_size_estimator = WorkingSetSizeEstimator()
self.hit_rate_monitor = HitRateMonitor()
self.smart_cache_experiment = MlosExperiment(
smart_component_types=[SmartCache],
telemetry_aggregators=[
self.working_set_size_estimator, self.hit_rate_monitor
])
self.optimization_problem = OptimizationProblem(
parameter_space=SmartCache.parameter_search_space,
objective_space=SimpleHypergrid(name="objectives",
dimensions=[
ContinuousDimension(
name="hit_rate",
min=0,
max=1)
]),
objectives=[Objective(name="hit_rate", minimize=False)])
def teardown_method(self, method):
mlos_globals.mlos_global_context.stop_clock()
self.mlos_agent.stop_all()
self.server.stop(grace=None).wait(timeout=1)
self.server.wait_for_termination(timeout=1)
self.optimizer_service_channel.close()
def test_smart_cache_with_remote_optimizer_on_a_timer(self):
""" Periodically invokes the optimizer to improve cache performance.
"""
optimizer_config = bayesian_optimizer_config_store.default
optimizer_config.homogeneous_random_forest_regression_model_config.decision_tree_regression_model_config.n_new_samples_before_refit = 5
self.optimizer = self.bayesian_optimizer_factory.create_remote_optimizer(
optimization_problem=self.optimization_problem,
optimizer_config=optimizer_config)
self.mlos_agent.start_experiment(self.smart_cache_experiment)
num_iterations = 101
for i in range(num_iterations):
smart_cache_workload_thread = Thread(
target=self.smart_cache_workload.run, args=(0.1, ))
smart_cache_workload_thread.start()
smart_cache_workload_thread.join()
current_cache_config = self.mlos_agent.get_configuration(
component_type=SmartCache)
features_df = current_cache_config.to_dataframe()
hit_rate = self.hit_rate_monitor.get_hit_rate()
num_requests = self.hit_rate_monitor.num_requests
working_set_size_estimate = self.working_set_size_estimator.estimate_working_set_size(
)
objectives_df = pd.DataFrame({'hit_rate': [hit_rate]})
self.optimizer.register(features_df, objectives_df)
new_config_values = self.optimizer.suggest()
self.mlos_agent.set_configuration(
component_type=SmartCache, new_config_values=new_config_values)
self.hit_rate_monitor.reset()
self.logger.info(
f"Previous config: {current_cache_config.to_json()}")
self.logger.info(
f"Estimated working set size: {working_set_size_estimate.chapman_estimator}. Hit rate: {hit_rate:.2f}. Num requests: {num_requests} "
)
self.mlos_agent.stop_experiment(self.smart_cache_experiment)
# Let's look at the goodness of fit.
#
multi_objective_gof_metrics = self.optimizer.compute_surrogate_model_goodness_of_fit(
)
for objective_name, random_forest_gof_metrics in multi_objective_gof_metrics:
# The model might not have used all of the samples, but should have used a majority of them (I expect about 90%), but 70% is a good sanity check
# and should make this test not very flaky.
assert random_forest_gof_metrics.last_refit_iteration_number > 0.5 * num_iterations
# Those relative errors should generally be between 0 and 1 unless the model's predictions are worse than predicting average...
# This unit tests occasionally doesn't have enough data to get us down to 1 so we'll pass the test if its less than 2.
# Note, the point of this test is to check sanity. We'll use a separate suite to evaluate models' performance from an ML standpoint.
self.logger.info(
f"Relative absolute error: {random_forest_gof_metrics.relative_absolute_error}"
)
self.logger.info(
f"Relative squared error: {random_forest_gof_metrics.relative_squared_error}"
)
assert random_forest_gof_metrics.relative_absolute_error is None or (
0 <= random_forest_gof_metrics.relative_absolute_error <= 2)
assert random_forest_gof_metrics.relative_squared_error is None or (
0 <= random_forest_gof_metrics.relative_squared_error <= 2)
# There is an invariant linking mean absolute error (MAE), root mean squared error (RMSE) and number of observations (n) let's assert it.
n = random_forest_gof_metrics.last_refit_iteration_number
self.logger.info(f"Last refit iteration number: {n}")
self.logger.info(
f"Mean absolute error: {random_forest_gof_metrics.mean_absolute_error}"
)
self.logger.info(
f"Root mean squared error: {random_forest_gof_metrics.root_mean_squared_error}"
)
assert random_forest_gof_metrics.mean_absolute_error <= random_forest_gof_metrics.root_mean_squared_error <= math.sqrt(
n) * random_forest_gof_metrics.mean_absolute_error
# We know that the sample confidence interval is wider (or equal to) prediction interval. So hit rates should be ordered accordingly.
assert random_forest_gof_metrics.sample_90_ci_hit_rate >= random_forest_gof_metrics.prediction_90_ci_hit_rate
class TestBayesianOptimizerGrpcClient:
""" Tests the E2E Grpc Client-Service workflow.
"""
@classmethod
def setup_class(cls):
warnings.simplefilter("error")
global_values.declare_singletons()
def setup_method(self, method):
self.logger = create_logger(self.__class__.__name__)
# Start up the gRPC service. Try a bunch of times before giving up.
#
max_num_tries = 100
num_tries = 0
for port in range(50051, 50051 + max_num_tries):
num_tries += 1
try:
self.server = OptimizerServicesServer(port=port,
num_threads=10)
self.server.start()
self.port = port
break
except:
self.logger.info(
f"Failed to create OptimizerMicroserviceServer on port {port}"
)
if num_tries == max_num_tries:
raise
self.optimizer_service_channel = grpc.insecure_channel(
f'localhost:{self.port}')
self.bayesian_optimizer_factory = BayesianOptimizerFactory(
grpc_channel=self.optimizer_service_channel, logger=self.logger)
self.optimizer_monitor = OptimizerMonitor(
grpc_channel=self.optimizer_service_channel, logger=self.logger)
objective_function_config = objective_function_config_store.get_config_by_name(
'2d_quadratic_concave_up')
self.objective_function = ObjectiveFunctionFactory.create_objective_function(
objective_function_config)
self.optimization_problem = OptimizationProblem(
parameter_space=self.objective_function.parameter_space,
objective_space=self.objective_function.output_space,
objectives=[Objective(name='y', minimize=True)])
def teardown_method(self, method):
""" We need to tear down the gRPC server here.
:return:
"""
self.server.stop(grace=None).wait(timeout=1)
self.server.wait_for_termination(timeout=1)
self.optimizer_service_channel.close()
def test_echo(self):
optimizer_service_stub = OptimizerServiceStub(
channel=self.optimizer_service_channel)
response = optimizer_service_stub.Echo(Empty())
assert isinstance(response, Empty)
def test_optimizer_with_default_config(self):
pre_existing_optimizers = {
optimizer.id: optimizer
for optimizer in self.optimizer_monitor.get_existing_optimizers()
}
print(bayesian_optimizer_config_store.default)
bayesian_optimizer = self.bayesian_optimizer_factory.create_remote_optimizer(
optimization_problem=self.optimization_problem,
optimizer_config=bayesian_optimizer_config_store.default)
post_existing_optimizers = {
optimizer.id: optimizer
for optimizer in self.optimizer_monitor.get_existing_optimizers()
}
new_optimizers = {
optimizer_id: optimizer
for optimizer_id, optimizer in post_existing_optimizers.items()
if optimizer_id not in pre_existing_optimizers
}
assert len(new_optimizers) == 1
new_optimizer_id = list(new_optimizers.keys())[0]
new_optimizer = new_optimizers[new_optimizer_id]
assert new_optimizer_id == bayesian_optimizer.id
assert new_optimizer.optimizer_config == bayesian_optimizer.optimizer_config
num_iterations = 100
registered_features_df, registered_objectives_df = self.optimize_objective_function(
optimizer=bayesian_optimizer,
objective_function=self.objective_function,
num_iterations=num_iterations)
# Apparently the to_json/from_json loses precision so we explicitly lose it here so that we can do the comparison.
#
registered_features_json = registered_features_df.to_json(
orient='index', double_precision=15)
registered_objectives_json = registered_objectives_df.to_json(
orient='index', double_precision=15)
# Apparently the jitter is too good and we actually have to use the json strings or they will be optimized away.
#
assert len(registered_features_json) > 0
assert len(registered_objectives_json) > 0
registered_features_df = pd.read_json(registered_features_json,
orient='index')
registered_objectives_df = pd.read_json(registered_objectives_json,
orient='index')
observed_features_df, observed_objectives_df, _ = bayesian_optimizer.get_all_observations(
)
assert (np.abs(registered_features_df - observed_features_df) <
0.00000001).all().all()
assert (np.abs(registered_objectives_df - observed_objectives_df) <
0.00000001).all().all()
# Assert that the observations and predictions are returned in the right order from the remote optimizer
#
parameters_df, objectives_df, _ = bayesian_optimizer.get_all_observations(
)
predictions_df = bayesian_optimizer.predict(
parameter_values_pandas_frame=parameters_df).get_dataframe()
assert parameters_df.index.intersection(predictions_df.index).equals(
predictions_df.index)
# Let's look at the goodness of fit.
#
multi_objective_gof_metrics = bayesian_optimizer.compute_surrogate_model_goodness_of_fit(
)
for objective_name, random_forest_gof_metrics in multi_objective_gof_metrics:
# The model might not have used all of the samples, but should have used a majority of them (I expect about 90%), but 50% is a good sanity check
# and should make this test not very flaky.
assert random_forest_gof_metrics.last_refit_iteration_number > 0.5 * num_iterations
# The invariants below should be true for all surrogate models: the random forest, and all constituent decision trees. So let's iterate over them all.
models_gof_metrics = [random_forest_gof_metrics]
for model_gof_metrics in models_gof_metrics:
assert 0 <= model_gof_metrics.relative_absolute_error <= 1 # This could fail if the models are really wrong. Not expected in this unit test though.
assert 0 <= model_gof_metrics.relative_squared_error <= 1
# There is an invariant linking mean absolute error (MAE), root mean squared error (RMSE) and number of observations (n) let's assert it.
n = model_gof_metrics.last_refit_iteration_number
assert model_gof_metrics.mean_absolute_error <= model_gof_metrics.root_mean_squared_error <= math.sqrt(
n) * model_gof_metrics.mean_absolute_error
# We know that the sample confidence interval is wider (or equal to) prediction interval. So hit rates should be ordered accordingly.
assert model_gof_metrics.sample_90_ci_hit_rate >= model_gof_metrics.prediction_90_ci_hit_rate
assert 0 <= model_gof_metrics.coefficient_of_determination <= 1
@pytest.mark.parametrize("i", [i for i in range(10)])
def test_optimizer_with_random_config(self, i):
optimizer_config = bayesian_optimizer_config_store.parameter_space.random(
)
optimizer_config.min_samples_required_for_guided_design_of_experiments = max(
min(
optimizer_config.
min_samples_required_for_guided_design_of_experiments, 100),
20)
if optimizer_config.surrogate_model_implementation == "HomogeneousRandomForestRegressionModel":
rf_config = optimizer_config.homogeneous_random_forest_regression_model_config
rf_config.n_estimators = min(rf_config.n_estimators, 20)
if optimizer_config.surrogate_model_implementation == MultiObjectiveRegressionEnhancedRandomForest.__name__:
optimizer_config.min_samples_required_for_guided_design_of_experiments = 25
rerf_model_config = optimizer_config.regression_enhanced_random_forest_regression_model_config
rerf_model_config.max_basis_function_degree = min(
rerf_model_config.max_basis_function_degree, 2)
# increased polynomial degree requires more data to estimate model parameters (poly term coefficients)
optimizer_config.min_samples_required_for_guided_design_of_experiments += 25 * (
rerf_model_config.max_basis_function_degree - 1)
rf_model_config = rerf_model_config.sklearn_random_forest_regression_model_config
rf_model_config.perform_initial_random_forest_hyper_parameter_search = False
rf_model_config.max_depth = min(rf_model_config.max_depth, 10)
rf_model_config.n_jobs = min(rf_model_config.n_jobs, 4)
print(
f"[{i+1}] Creating a bayesian optimizer with config: {optimizer_config}"
)
bayesian_optimizer = self.bayesian_optimizer_factory.create_remote_optimizer(
optimization_problem=self.optimization_problem,
optimizer_config=optimizer_config)
registered_features_df, registered_objectives_df = self.optimize_objective_function(
optimizer=bayesian_optimizer,
objective_function=self.objective_function,
num_iterations=12)
# Apparently the to_json/from_json loses precision so we explicitly lose it here so that we can do the comparison.
#
registered_features_json = registered_features_df.to_json(
orient='index', double_precision=15)
registered_objectives_json = registered_objectives_df.to_json(
orient='index', double_precision=15)
# Apparently the jitter is too good and we actually have to use the json strings or they will be optimized away.
#
assert len(registered_features_json) > 0
assert len(registered_objectives_json) > 0
registered_features_df = pd.read_json(registered_features_json,
orient='index')
registered_objectives_df = pd.read_json(registered_objectives_json,
orient='index')
observed_features_df, observed_objectives_df, _ = bayesian_optimizer.get_all_observations(
)
assert (np.abs(registered_features_df - observed_features_df) <
0.00000001).all().all()
assert (np.abs(registered_objectives_df - observed_objectives_df) <
0.00000001).all().all()
@pytest.mark.parametrize("i", [i for i in range(10)])
def test_optimizer_with_random_config_random_objective(self, i):
objective_function_config = objective_function_config_store.parameter_space.random(
)
objective_function = ObjectiveFunctionFactory.create_objective_function(
objective_function_config)
optimization_problem = objective_function.default_optimization_problem
optimizer_config = bayesian_optimizer_config_store.parameter_space.random(
)
optimizer_config.min_samples_required_for_guided_design_of_experiments = max(
min(
optimizer_config.
min_samples_required_for_guided_design_of_experiments, 100),
20)
if optimizer_config.surrogate_model_implementation == "HomogeneousRandomForestRegressionModel":
rf_config = optimizer_config.homogeneous_random_forest_regression_model_config
rf_config.n_estimators = min(rf_config.n_estimators, 20)
if optimizer_config.surrogate_model_implementation == MultiObjectiveRegressionEnhancedRandomForest.__name__:
optimizer_config.min_samples_required_for_guided_design_of_experiments = 25
rerf_model_config = optimizer_config.regression_enhanced_random_forest_regression_model_config
rerf_model_config.max_basis_function_degree = min(
rerf_model_config.max_basis_function_degree, 2)
# increased polynomial degree requires more data to estimate model parameters (poly term coefficients)
optimizer_config.min_samples_required_for_guided_design_of_experiments += 25 * (
rerf_model_config.max_basis_function_degree - 1)
rf_model_config = rerf_model_config.sklearn_random_forest_regression_model_config
rf_model_config.perform_initial_random_forest_hyper_parameter_search = False
rf_model_config.max_depth = min(rf_model_config.max_depth, 10)
rf_model_config.n_jobs = min(rf_model_config.n_jobs, 4)
print(
f"[{i+1}] Creating a bayesian optimizer with config: {optimizer_config} \n\n\nObjective function config: {objective_function_config}"
)
bayesian_optimizer = self.bayesian_optimizer_factory.create_remote_optimizer(
optimization_problem=optimization_problem,
optimizer_config=optimizer_config)
registered_params_df, registered_objectives_df = self.optimize_objective_function(
optimizer=bayesian_optimizer,
objective_function=objective_function,
num_iterations=20)
# Apparently the to_json/from_json loses precision so we explicitly lose it here so that we can do the comparison.
#
registered_features_json = registered_params_df.to_json(
orient='index', double_precision=15)
registered_objectives_json = registered_objectives_df.to_json(
orient='index', double_precision=15)
# Apparently the jitter is too good and we actually have to use the json strings or they will be optimized away.
#
assert len(registered_features_json) > 0
assert len(registered_objectives_json) > 0
registered_params_df = pd.read_json(registered_features_json,
orient='index')
registered_objectives_df = pd.read_json(registered_objectives_json,
orient='index')
observed_params_df, observed_objectives_df, _ = bayesian_optimizer.get_all_observations(
)
numeric_params_names = [
dimension.name
for dimension in optimization_problem.parameter_space.dimensions
if (isinstance(dimension, (ContinuousDimension, DiscreteDimension))
or (isinstance(dimension, CategoricalDimension)
and dimension.is_numeric)) and (
dimension.name in registered_params_df.columns) and (
dimension.name in observed_params_df.columns)
]
numeric_params_df = registered_params_df[numeric_params_names]
observed_numeric_params_df = observed_params_df[numeric_params_names]
assert (np.abs(
numeric_params_df.fillna(0) - observed_numeric_params_df.fillna(0))
< 0.00000001).all().all()
assert (np.abs(registered_objectives_df - observed_objectives_df) <
0.00000001).all().all()
@staticmethod
def optimize_objective_function(optimizer, objective_function,
num_iterations):
registered_features_df = None
registered_objectives_df = None
# Let's make sure that the optimum for the first objective doesn't get worse throughout the optimization loop.
#
first_objective = optimizer.optimization_problem.objectives[0]
if first_objective.minimize:
old_optimum = np.inf
else:
old_optimum = -np.inf
for i in range(num_iterations):
suggested_params = optimizer.suggest()
suggested_params_df = suggested_params.to_dataframe()
objective_values = objective_function.evaluate_point(
suggested_params)
optimizer.register(suggested_params_df,
objective_values.to_dataframe())
if registered_features_df is None:
registered_features_df = suggested_params_df
else:
registered_features_df = registered_features_df.append(
suggested_params_df, ignore_index=True)
if registered_objectives_df is None:
registered_objectives_df = objective_values.to_dataframe()
else:
registered_objectives_df = registered_objectives_df.append(
objective_values.to_dataframe(), ignore_index=True)
best_params, optimum = optimizer.optimum()
# ensure current optimum doesn't go up
#
if first_objective.minimize:
assert optimum[first_objective.name] <= old_optimum
else:
assert optimum[first_objective.name] >= old_optimum
old_optimum = optimum[first_objective.name]
print(
f"[{i+1}/{num_iterations}]Best Params: {best_params}, Best Value: {optimum[first_objective.name]}"
)
return registered_features_df, registered_objectives_df