def setUpClass(cls) -> None:
global_values.declare_singletons()
cls.slope = 10
cls.y_intercept = 10
cls.input_values = np.linspace(start=0, stop=100, num=1000, endpoint=True)
cls.output_values = cls.input_values * cls.slope + cls.y_intercept
cls.input_space = SimpleHypergrid(
name="input",
dimensions=[ContinuousDimension(name="x", min=0, max=100)]
)
cls.output_space = SimpleHypergrid(
name="output",
dimensions=[ContinuousDimension(name="y", min=-math.inf, max=math.inf)]
)
cls.input_pandas_dataframe = pd.DataFrame({"x": cls.input_values})
cls.output_pandas_dataframe = pd.DataFrame({"y": cls.output_values})
cls.model_config = HomogeneousRandomForestRegressionModelConfig()
cls.model = HomogeneousRandomForestRegressionModel(
model_config=cls.model_config,
input_space=cls.input_space,
output_space=cls.output_space
)
cls.model.fit(cls.input_pandas_dataframe, cls.output_pandas_dataframe, iteration_number=len(cls.input_pandas_dataframe.index))
cls.sample_inputs = {'x': np.linspace(start=-10, stop=110, num=13, endpoint=True)}
cls.sample_inputs_pandas_dataframe = pd.DataFrame(cls.sample_inputs)
cls.sample_predictions = cls.model.predict(cls.sample_inputs_pandas_dataframe)
def setUpClass(cls):
"""Sets up all the singletons needed to test the BayesianOptimizer.
"""
warnings.simplefilter("error")
global_values.declare_singletons()
global_values.tracer = Tracer(actor_id=cls.__name__, thread_id=0)
cls.logger = create_logger(logger_name=cls.__name__)
def setUp(self):
mlos_globals.init_mlos_global_context()
mlos_globals.mlos_global_context.start_clock()
self.logger = create_logger('TestSmartCacheWithRemoteOptimizer')
self.logger.level = logging.DEBUG
# Start up the gRPC service.
#
self.server = OptimizerMicroserviceServer(port=50051, num_threads=10)
self.server.start()
self.optimizer_service_grpc_channel = grpc.insecure_channel('localhost:50051')
self.bayesian_optimizer_factory = BayesianOptimizerFactory(grpc_channel=self.optimizer_service_grpc_channel, logger=self.logger)
self.mlos_agent = MlosAgent(
logger=self.logger,
communication_channel=mlos_globals.mlos_global_context.communication_channel,
shared_config=mlos_globals.mlos_global_context.shared_config,
bayesian_optimizer_grpc_channel=self.optimizer_service_grpc_channel
)
self.mlos_agent_thread = Thread(target=self.mlos_agent.run)
self.mlos_agent_thread.start()
global_values.declare_singletons() # TODO: having both globals and global_values is a problem
# Let's add the allowed component types
self.mlos_agent.add_allowed_component_type(SmartCache)
self.mlos_agent.add_allowed_component_type(SmartCacheWorkloadGenerator)
self.mlos_agent.set_configuration(
component_type=SmartCacheWorkloadGenerator,
new_config_values=Point(
workload_type='cyclical_key_from_range',
cyclical_key_from_range_config=Point(
min=0,
range_width=2048
)
)
)
# Let's create the workload
self.smart_cache_workload = SmartCacheWorkloadGenerator(logger=self.logger)
self.optimizer = None
self.working_set_size_estimator = WorkingSetSizeEstimator()
self.hit_rate_monitor = HitRateMonitor()
self.smart_cache_experiment = MlosExperiment(
smart_component_types=[SmartCache],
telemetry_aggregators=[self.working_set_size_estimator, self.hit_rate_monitor]
)
self.optimization_problem = OptimizationProblem(
parameter_space=SmartCache.parameter_search_space,
objective_space=SimpleHypergrid(name="objectives", dimensions=[ContinuousDimension(name="hit_rate", min=0, max=1)]),
objectives=[Objective(name="hit_rate", minimize=False)]
)
def setup_class(cls):
""" Set's up all the objects needed to test the RandomSearchOptimizer
To test the RandomSearchOptimizer we need to first construct:
* an optimization problem
* a utility function
To construct a utility function we need the same set up as in the TestConfidenceBoundUtilityFunction
test.
:return:
"""
global_values.declare_singletons()
global_values.tracer = Tracer(actor_id=cls.__name__, thread_id=0)
objective_function_config = objective_function_config_store.get_config_by_name(
'2d_quadratic_concave_up')
objective_function = ObjectiveFunctionFactory.create_objective_function(
objective_function_config=objective_function_config)
cls.input_space = objective_function.parameter_space
cls.output_space = objective_function.output_space
cls.input_values_dataframe = objective_function.parameter_space.random_dataframe(
num_samples=2500)
cls.output_values_dataframe = objective_function.evaluate_dataframe(
cls.input_values_dataframe)
cls.model_config = homogeneous_random_forest_config_store.default
print(cls.model_config)
cls.model = MultiObjectiveHomogeneousRandomForest(
model_config=cls.model_config,
input_space=cls.input_space,
output_space=cls.output_space)
cls.model.fit(cls.input_values_dataframe,
cls.output_values_dataframe,
iteration_number=len(cls.input_values_dataframe.index))
cls.utility_function_config = Point(
utility_function_name="upper_confidence_bound_on_improvement",
alpha=0.05)
cls.optimization_problem = OptimizationProblem(
parameter_space=cls.input_space,
objective_space=cls.output_space,
objectives=[Objective(name='y', minimize=True)])
cls.utility_function = ConfidenceBoundUtilityFunction(
function_config=cls.utility_function_config,
surrogate_model=cls.model,
minimize=cls.optimization_problem.objectives[0].minimize)
def test_tracer(self):
global_values.declare_singletons()
global_values.tracer = Tracer(actor_id='1', thread_id='0')
try:
fib_number = self.fibonacci(n=3)
except:
pass
trace_events = global_values.tracer.trace_events
reformatted_events = Tracer.reformat_events(trace_events)
assert len(trace_events) > 0
print(json.dumps(reformatted_events, indent=2))
def setUp(self):
mlos_globals.init_mlos_global_context()
mlos_globals.mlos_global_context.start_clock()
self.logger = create_logger('TestSmartCacheWithRemoteOptimizer')
self.logger.level = logging.INFO
self.mlos_agent = MlosAgent(
logger=self.logger,
communication_channel=mlos_globals.mlos_global_context.
communication_channel,
shared_config=mlos_globals.mlos_global_context.shared_config,
)
self.mlos_agent_thread = Thread(target=self.mlos_agent.run)
self.mlos_agent_thread.start()
global_values.declare_singletons(
) # TODO: having both globals and global_values is a problem
self.workload_duration_s = 5
# Let's add the allowed component types
self.mlos_agent.add_allowed_component_type(SmartCache)
self.mlos_agent.add_allowed_component_type(SmartCacheWorkloadGenerator)
# Let's create the workload
self.smart_cache_workload = SmartCacheWorkloadGenerator(
logger=self.logger)
self.optimizer = None
self.working_set_size_estimator = WorkingSetSizeEstimator()
self.cache_config_timer = Timer(
timeout_ms=200,
observer_callback=self._set_new_cache_configuration)
self.smart_cache_experiment = MlosExperiment(
smart_component_types=[SmartCache],
telemetry_aggregators=[
self.cache_config_timer, self.working_set_size_estimator
])
self.optimization_problem = OptimizationProblem(
parameter_space=SmartCache.parameter_search_space,
objective_space=SimpleHypergrid(name="objectives",
dimensions=[
ContinuousDimension(
name="miss_rate",
min=0,
max=1)
]),
context_space=None, # TODO: add working set size estimate
objectives=[Objective(name="miss_rate", minimize=True)])
def setup_class(cls):
""" Set's up all the objects needed to test the UtilityFunctionOptimizers
To test the UtilityFunctionOptimizers we need to first construct:
* an objective function for the model to approximate and its corresponding parameter and output spaces
* an optimization problem
* a regression model, then train it on some random parameters to the objective function
* a utility function that utilizes the model
* a pareto frontier over the random parameters
And only then do we get to test our utility function optimizers. This is a lot of work and a somewhat cleaner approach
would be to simply create an instance of the BayesianOptimizer to do all of the above for us, but then we might not be able
to test the utility function optimizers as thoroughly as we need to.
:return:
"""
global_values.declare_singletons()
global_values.tracer = Tracer(actor_id=cls.__name__, thread_id=0)
cls.logger = create_logger("TestUtilityFunctionOptimizers")
cls.model_config = multi_objective_pass_through_model_config_store.default
cls.model = MultiObjectivePassThroughModelForTesting(
model_config=cls.model_config,
logger=cls.logger
)
cls.objective_function = cls.model.objective_function
cls.parameter_space = cls.objective_function.parameter_space
cls.objective_space = cls.objective_function.output_space
cls.optimization_problem = cls.objective_function.default_optimization_problem
cls.utility_function_config = Point(utility_function_name="upper_confidence_bound_on_improvement", alpha=0.05)
cls.utility_function = ConfidenceBoundUtilityFunction(
function_config=cls.utility_function_config,
surrogate_model=cls.model,
minimize=cls.optimization_problem.objectives[0].minimize,
logger=cls.logger
)
# To make the pareto frontier we have to generate some random points.
#
cls.parameters_df = cls.objective_function.parameter_space.random_dataframe(1000)
cls.objectives_df = cls.objective_function.evaluate_dataframe(cls.parameters_df)
cls.pareto_frontier = ParetoFrontier(
optimization_problem=cls.optimization_problem,
objectives_df=cls.objectives_df,
parameters_df=cls.parameters_df
)
def setup_class(cls):
""" Sets up all the singletons needed to test the BayesianOptimizer.
"""
warnings.simplefilter("error")
global_values.declare_singletons()
global_values.tracer = Tracer(actor_id=cls.__name__, thread_id=0)
cls.logger = create_logger(logger_name=cls.__name__)
cls.logger.setLevel(logging.DEBUG)
cls.port = None
# Start up the gRPC service. Try a bunch of ports, before giving up so we can do several in parallel.
#
max_num_tries = 100
num_tries = 0
for port in range(50051, 50051 + max_num_tries):
num_tries += 1
try:
cls.server = OptimizerMicroserviceServer(port=port,
num_threads=10,
logger=cls.logger)
cls.server.start()
cls.port = port
break
except:
cls.logger.info(
f"Failed to create OptimizerMicroserviceServer on port {port}."
)
if num_tries == max_num_tries:
raise
cls.optimizer_service_channel = grpc.insecure_channel(
f'localhost:{cls.port}')
cls.bayesian_optimizer_factory = BayesianOptimizerFactory(
grpc_channel=cls.optimizer_service_channel, logger=cls.logger)
cls.temp_dir = os.path.join(os.getcwd(), "temp")
if not os.path.exists(cls.temp_dir):
os.mkdir(cls.temp_dir)
cls.trace_output_path = os.path.join(
cls.temp_dir, "TestBayesianOptimizerTrace.json")
try:
os.remove(cls.trace_output_path)
except OSError:
pass
def setUpClass(cls):
""" Sets up all the singletons needed to test the BayesianOptimizer.
"""
warnings.simplefilter("error")
global_values.declare_singletons()
global_values.tracer = Tracer(actor_id=cls.__name__, thread_id=0)
cls.logger = create_logger(logger_name=cls.__name__)
# Start up the gRPC service.
#
cls.server = OptimizerMicroserviceServer(port=50051, num_threads=10)
cls.server.start()
cls.optimizer_service_channel = grpc.insecure_channel('localhost:50051')
cls.bayesian_optimizer_factory = BayesianOptimizerFactory(grpc_channel=cls.optimizer_service_channel, logger=cls.logger)
def main():
args = parse_command_line_arguments()
server = OptimizerMicroserviceServer(port=args.port, num_threads=args.num_threads)
def ctrl_c_handler(_, __):
print("Received CTRL-C: shutting down.")
if server.started:
print("Shutting down server.")
server.stop(grace=None)
print("Server stopped.")
global_values.declare_singletons()
signal.signal(signal.SIGINT, ctrl_c_handler)
signal.signal(signal.SIGTERM, ctrl_c_handler)
print("Starting Optimizer Microservice ...")
server.start()
server.wait_for_termination()
def setUpClass(cls):
""" Set's up all the objects needed to test the RandomSearchOptimizer
To test the RandomSearchOptimizer we need to first construct:
* an optimization problem
* a utility function
To construct a utility function we need the same set up as in the TestConfidenceBoundUtilityFunction
test.
:return:
"""
cls.temp_dir = os.path.join(os.getcwd(), "temp")
if not os.path.exists(cls.temp_dir):
os.mkdir(cls.temp_dir)
global_values.declare_singletons()
global_values.tracer = Tracer(actor_id=cls.__name__, thread_id=0)
cls.logger = create_logger(logger_name=cls.__name__)
def setup_class(cls) -> None:
global_values.declare_singletons()
global_values.tracer = Tracer(actor_id=cls.__name__, thread_id=0)
def setUpClass(cls):
global_values.declare_singletons()
def setUpClass(cls):
warnings.simplefilter("error")
global_values.declare_singletons()
def setup_class(cls) -> None:
global_values.declare_singletons()
#
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.
#
import pytest
import numpy as np
import pandas as pd
from mlos.Optimizers.RegressionModels.Prediction import Prediction
import mlos.global_values as global_values
global_values.declare_singletons()
class TestPrediction:
class MockValidRegressionModelPrediction(Prediction):
all_prediction_fields = Prediction.LegalColumnNames
OUTPUTS = [
all_prediction_fields.SAMPLE_MEAN,
all_prediction_fields.SAMPLE_VARIANCE,
all_prediction_fields.SAMPLE_SIZE
]
def __init__(self, objective_name: str):
super().__init__(objective_name=objective_name,
predictor_outputs=TestPrediction.
MockValidRegressionModelPrediction.OUTPUTS)
@classmethod
def classSetUp(cls):
global_values.declare_singletons()
def classSetUp(cls):
global_values.declare_singletons()
def setUpClass(cls):
""" Set's up all the objects needed to test the RandomSearchOptimizer
To test the RandomSearchOptimizer we need to first construct:
* an optimization problem
* a utility function
To construct a utility function we need the same set up as in the TestConfidenceBoundUtilityFunction
test.
:return:
"""
global_values.declare_singletons()
cls.input_space = SimpleHypergrid(name="input",
dimensions=[
ContinuousDimension(name='x_1',
min=-100,
max=100),
ContinuousDimension(name='x_2',
min=-100,
max=100)
])
cls.output_space = SimpleHypergrid(name="output",
dimensions=[
ContinuousDimension(
name='y',
min=-math.inf,
max=math.inf)
])
x_1, x_2 = np.meshgrid(cls.input_space['x_1'].linspace(num=201),
cls.input_space['x_2'].linspace(num=201))
y = -quadratic(x_1=x_1, x_2=x_2)
cls.input_values_dataframe = pd.DataFrame({
'x_1': x_1.reshape(-1),
'x_2': x_2.reshape(-1)
})
cls.output_values_dataframe = pd.DataFrame({'y': y.reshape(-1)})
cls.model_config = HomogeneousRandomForestRegressionModelConfig()
cls.model = HomogeneousRandomForestRegressionModel(
model_config=cls.model_config,
input_space=cls.input_space,
output_space=cls.output_space)
cls.model.fit(cls.input_values_dataframe,
cls.output_values_dataframe,
iteration_number=len(cls.input_values_dataframe.index))
cls.utility_function_config = ConfidenceBoundUtilityFunctionConfig(
utility_function_name="upper_confidence_bound_on_improvement",
alpha=0.05)
cls.optimization_problem = OptimizationProblem(
parameter_space=cls.input_space,
objective_space=cls.output_space,
objectives=[Objective(name='y', minimize=True)])
cls.utility_function = ConfidenceBoundUtilityFunction(
function_config=cls.utility_function_config,
surrogate_model=cls.model,
minimize=cls.optimization_problem.objectives[0].minimize)
