def predict(self,
parameter_values_pandas_frame,
t=None,
context_values_pandas_frame=None): # pylint: disable=unused-argument
# TODO: make this streaming and/or using arrow.
#
if context_values_pandas_frame is not None:
raise NotImplementedError(
"Context not currently supported on remote optimizers")
feature_values_dict = parameter_values_pandas_frame.to_dict(
orient='list')
prediction_request = OptimizerService_pb2.PredictRequest(
OptimizerHandle=self.optimizer_handle,
Features=OptimizerService_pb2.Features(
FeaturesJsonString=json.dumps(feature_values_dict)))
prediction_response = self._optimizer_stub.Predict(prediction_request)
# To be compliant with the OptimizerBase, we need to recover a single Prediction object and return it.
#
objective_predictions_pb2 = prediction_response.ObjectivePredictions
assert len(objective_predictions_pb2) == 1
only_prediction_pb2 = objective_predictions_pb2[0]
objective_name = only_prediction_pb2.ObjectiveName
valid_predictions_df = Prediction.dataframe_from_json(
only_prediction_pb2.PredictionDataFrameJsonString)
prediction = Prediction.create_prediction_from_dataframe(
objective_name=objective_name, dataframe=valid_predictions_df)
return prediction
def encode_optimization_problem(optimization_problem: OptimizationProblem) -> OptimizerService_pb2.OptimizationProblem:
return OptimizerService_pb2.OptimizationProblem(
ParameterSpace=OptimizerService_pb2.Hypergrid(HypergridJsonString=json.dumps(optimization_problem.parameter_space, cls=HypergridJsonEncoder)),
ObjectiveSpace=OptimizerService_pb2.Hypergrid(HypergridJsonString=json.dumps(optimization_problem.objective_space, cls=HypergridJsonEncoder)),
Objectives=[OptimizerService_pb2.Objective(Name=objective.name, Minimize=objective.minimize) for objective in optimization_problem.objectives],
ContextSpace=None if optimization_problem.context_space is None
else OptimizerService_pb2.Hypergrid(HypergridJsonString=json.dumps(optimization_problem.context_space, cls=HypergridJsonEncoder))
)
def register(self, feature_values_pandas_frame, target_values_pandas_frame):
register_request = OptimizerService_pb2.RegisterObservationsRequest(
OptimizerHandle=self.optimizer_handle,
Observations=OptimizerService_pb2.Observations(
Features=OptimizerService_pb2.Features(FeaturesJsonString=feature_values_pandas_frame.to_json(orient='index', double_precision=15)),
ObjectiveValues=OptimizerService_pb2.ObjectiveValues(
ObjectiveValuesJsonString=target_values_pandas_frame.to_json(orient='index', double_precision=15)
)
)
)
self._optimizer_stub.RegisterObservations(register_request)
def encode_primitive_value(value: Union[int, float, bool, str]) -> OptimizerService_pb2.PrimitiveValue:
assert isinstance(value, (int, float, bool, str))
if isinstance(value, bool):
return OptimizerService_pb2.PrimitiveValue(BoolValue=value)
if isinstance(value, int):
return OptimizerService_pb2.PrimitiveValue(IntValue=value)
if isinstance(value, float):
return OptimizerService_pb2.PrimitiveValue(DoubleValue=value)
if isinstance(value, str):
return OptimizerService_pb2.PrimitiveValue(StringValue=value)
raise TypeError(f"{value} is of type: {type(value)} but must be one of (int, float, bool, str)")
def register(self, parameter_values_pandas_frame, target_values_pandas_frame, context_values_pandas_frame=None):
if context_values_pandas_frame is not None:
raise NotImplementedError("Context not currently supported on remote optimizers")
feature_values_pandas_frame = parameter_values_pandas_frame
register_request = OptimizerService_pb2.RegisterObservationsRequest(
OptimizerHandle=self.optimizer_handle_for_optimizer_service,
Observations=OptimizerService_pb2.Observations(
Features=OptimizerService_pb2.Features(FeaturesJsonString=feature_values_pandas_frame.to_json(orient='index', double_precision=15)),
ObjectiveValues=OptimizerService_pb2.ObjectiveValues(
ObjectiveValuesJsonString=target_values_pandas_frame.to_json(orient='index', double_precision=15)
)
)
)
self._optimizer_stub.RegisterObservations(register_request) # TODO: we should be using the optimizer_stub for this.
def encode_ordinal_dimension(dimension: OrdinalDimension) -> OptimizerService_pb2.OrdinalDimension:
assert isinstance(dimension, OrdinalDimension)
return OptimizerService_pb2.OrdinalDimension(
Name=dimension.name,
Ascending=dimension.ascending,
OrderedValues=[OptimizerServiceEncoder.encode_primitive_value(value) for value in dimension.values]
)
def CreateOptimizer(self,
request: OptimizerService_pb2.CreateOptimizerRequest,
context): # pylint: disable=unused-argument
self.logger.info("Creating Optimizer")
print("CREATING OPTIMIZER")
optimization_problem = OptimizationProblem.from_protobuf(
optimization_problem_pb2=request.OptimizationProblem)
optimizer_config_json = request.OptimizerConfig
if optimizer_config_json is not None and len(
optimizer_config_json) > 0:
optimizer_config = Point.from_json(optimizer_config_json)
else:
optimizer_config = bayesian_optimizer_config_store.default
optimizer = BayesianOptimizer(
optimization_problem=optimization_problem,
optimizer_config=optimizer_config)
optimizer_id = self.get_next_optimizer_id()
# To avoid a race condition we acquire the lock before inserting the lock and the optimizer into their respective
# dictionaries. Otherwise we could end up with a situation where a lock is in the dictionary, but the optimizer
# is not.
optimizer_lock = self._lock_manager.RLock()
with optimizer_lock:
self._optimizer_locks_by_optimizer_id[
optimizer_id] = optimizer_lock
self._optimizers_by_id[optimizer_id] = optimizer
self._ordered_ids.append(optimizer_id)
self.logger.info(f"Created optimizer {optimizer_id}.")
return OptimizerService_pb2.OptimizerHandle(Id=optimizer_id)
def encode_discrete_dimension(
dimension: DiscreteDimension
) -> OptimizerService_pb2.DiscreteDimension:
assert isinstance(dimension, DiscreteDimension)
return OptimizerService_pb2.DiscreteDimension(Name=dimension.name,
Min=dimension.min,
Max=dimension.max)
def encode_optimization_problem(
optimization_problem: OptimizationProblem
) -> OptimizerService_pb2.OptimizationProblem:
return OptimizerService_pb2.OptimizationProblem(
ParameterSpace=OptimizerServiceEncoder.encode_hypergrid(
optimization_problem.parameter_space),
ObjectiveSpace=OptimizerServiceEncoder.encode_hypergrid(
optimization_problem.objective_space),
Objectives=[
OptimizerService_pb2.Objective(Name=objective.name,
Minimize=objective.minimize)
for objective in optimization_problem.objectives
],
ContextSpace=None if optimization_problem.context_space is None
else OptimizerServiceEncoder.encode_hypergrid(
optimization_problem.context_space))
def Predict(self, request, context): # pylint: disable=unused-argument
features_dict = json.loads(request.Features.FeaturesJsonString)
features_df = pd.DataFrame(features_dict)
with self.exclusive_optimizer(
optimizer_id=request.OptimizerHandle.Id) as optimizer:
prediction = optimizer.predict(features_df)
assert isinstance(prediction, Prediction)
response = OptimizerService_pb2.PredictResponse(ObjectivePredictions=[
OptimizerService_pb2.SingleObjectivePrediction(
ObjectiveName=prediction.objective_name,
PredictionDataFrameJsonString=prediction.dataframe_to_json())
])
return response
def suggest(self, random=False, context=None): # pylint: disable=unused-argument
suggestion_request = OptimizerService_pb2.SuggestRequest(
OptimizerHandle=self.optimizer_handle, Random=random)
suggestion_response = self._optimizer_stub.Suggest(suggestion_request)
suggested_params_dict = json.loads(
suggestion_response.ParametersJsonString)
return Point(**suggested_params_dict)
def encode_empty_dimension(
dimension: EmptyDimension) -> OptimizerService_pb2.EmptyDimension:
assert isinstance(dimension, EmptyDimension)
return OptimizerService_pb2.EmptyDimension(
Name=dimension.name,
DimensionType=OptimizerServiceEncoder.dimension_types_to_pb2_types[
dimension.type])
def encode_subgrid(
subgrid: SimpleHypergrid.JoinedSubgrid
) -> OptimizerService_pb2.GuestSubgrid:
assert isinstance(subgrid, SimpleHypergrid.JoinedSubgrid)
return OptimizerService_pb2.GuestSubgrid(
Subgrid=OptimizerServiceEncoder.encode_hypergrid(subgrid.subgrid),
ExternalPivotDimension=OptimizerServiceEncoder.encode_dimension(
subgrid.join_dimension))
def encode_continuous_dimension(dimension: ContinuousDimension) -> OptimizerService_pb2.ContinuousDimension:
assert isinstance(dimension, ContinuousDimension)
return OptimizerService_pb2.ContinuousDimension(
Name=dimension.name,
Min=dimension.min,
Max=dimension.max,
IncludeMin=dimension.include_min,
IncludeMax=dimension.include_max
)
def encode_categorical_dimension(
dimension: CategoricalDimension
) -> OptimizerService_pb2.CategoricalDimension:
assert isinstance(dimension, CategoricalDimension)
return OptimizerService_pb2.CategoricalDimension(
Name=dimension.name,
Values=[
OptimizerServiceEncoder.encode_primitive_value(value)
for value in dimension.values
])
def Suggest(self, request, context): # pylint: disable=unused-argument
# TODO: return an error if optimizer not found
#
with self.exclusive_optimizer(
optimizer_id=request.OptimizerHandle.Id) as optimizer:
suggested_params = optimizer.suggest(random=request.Random,
context=request.Context)
return OptimizerService_pb2.ConfigurationParameters(
ParametersJsonString=json.dumps(suggested_params.to_dict()))
def to_protobuf(self):
""" Serializes self to a protobuf.
:return:
"""
return OptimizerService_pb2.OptimizationProblem(
ParameterSpace=OptimizerService_pb2.Hypergrid(
HypergridJsonString=json.dumps(self.parameter_space,
cls=HypergridJsonEncoder)),
ObjectiveSpace=OptimizerService_pb2.Hypergrid(
HypergridJsonString=json.dumps(self.objective_space,
cls=HypergridJsonEncoder)),
Objectives=[
OptimizerService_pb2.Objective(Name=objective.name,
Minimize=objective.minimize)
for objective in self.objectives
],
ContextSpace=None if self.context_space is None else
OptimizerService_pb2.Hypergrid(HypergridJsonString=json.dumps(
self.context_space, cls=HypergridJsonEncoder)))
def suggest(self, random=False, context=None): # pylint: disable=unused-argument
if context is not None:
raise NotImplementedError("Context not currently supported on remote optimizers")
suggestion_request = OptimizerService_pb2.SuggestRequest(
OptimizerHandle=self.optimizer_handle_for_optimizer_service,
Random=random,
Context=context
)
suggestion_response = self._optimizer_stub.Suggest(suggestion_request)
suggested_params_dict = json.loads(suggestion_response.ParametersJsonString)
return Point(**suggested_params_dict)
def register(self, feature_values_pandas_frame,
target_values_pandas_frame): # pylint: disable=unused-argument
# TODO: implement RegisterObservations <- plural
#
features_dicts_per_record = feature_values_pandas_frame.to_dict(
orient='records')
objectives_dicts_per_record = target_values_pandas_frame.to_dict(
orient='records')
# TODO: Either implement streaming or arrow flight or batch.
#
for feature_dict, objective_dict in zip(features_dicts_per_record,
objectives_dicts_per_record):
register_request = OptimizerService_pb2.RegisterObservationRequest(
OptimizerHandle=self.optimizer_handle,
Observation=OptimizerService_pb2.Observation(
Features=OptimizerService_pb2.Features(
FeaturesJsonString=json.dumps(feature_dict)),
ObjectiveValues=OptimizerService_pb2.ObjectiveValues(
ObjectiveValuesJsonString=json.dumps(objective_dict))))
self._optimizer_stub.RegisterObservation(register_request)
def encode_dimension(
dimension: Dimension) -> OptimizerService_pb2.Dimension:
if isinstance(dimension, EmptyDimension):
return OptimizerService_pb2.Dimension(
EmptyDimension=OptimizerServiceEncoder.encode_empty_dimension(
dimension))
if isinstance(dimension, ContinuousDimension):
return OptimizerService_pb2.Dimension(
ContinuousDimension=OptimizerServiceEncoder.
encode_continuous_dimension(dimension))
if isinstance(dimension, DiscreteDimension):
return OptimizerService_pb2.Dimension(
DiscreteDimension=OptimizerServiceEncoder.
encode_discrete_dimension(dimension))
if isinstance(dimension, OrdinalDimension):
return OptimizerService_pb2.Dimension(
OrdinalDimension=OptimizerServiceEncoder.
encode_ordinal_dimension(dimension))
if isinstance(dimension, CategoricalDimension):
return OptimizerService_pb2.Dimension(
CategoricalDimension=OptimizerServiceEncoder.
encode_categorical_dimension(dimension))
if isinstance(dimension, CompositeDimension):
return OptimizerService_pb2.Dimension(
CompositeDimension=OptimizerServiceEncoder.
encode_composite_dimension(dimension))
raise TypeError(f"Unsupported dimension type: {type(dimension)}")
def encode_simple_hypergrid(hypergrid: SimpleHypergrid) -> OptimizerService_pb2.SimpleHypergrid:
assert isinstance(hypergrid, SimpleHypergrid)
encoded_subgrids = []
for _, subgrids in hypergrid.joined_subgrids_by_pivot_dimension.items():
for subgrid in subgrids:
encoded_subgrid = OptimizerServiceEncoder.encode_subgrid(subgrid)
encoded_subgrids.append(encoded_subgrid)
return OptimizerService_pb2.SimpleHypergrid(
Name=hypergrid.name,
Dimensions=[OptimizerServiceEncoder.encode_dimension(dimension) for dimension in hypergrid.root_dimensions],
GuestSubgrids=encoded_subgrids
)
def Suggest(self, request, context): # pylint: disable=unused-argument
self.logger.info("Suggesting")
# TODO: return an error if optimizer not found
#
if request.Context.ContextJsonString != "":
raise NotImplementedError("Context not currently supported in remote optimizers")
with self.exclusive_optimizer(optimizer_id=request.OptimizerHandle.Id) as optimizer:
# TODO handle context here
suggested_params = optimizer.suggest(random=request.Random)
return OptimizerService_pb2.ConfigurationParameters(
ParametersJsonString=json.dumps(suggested_params.to_dict())
)
def encode_composite_dimension(dimension: CompositeDimension) -> OptimizerService_pb2.CompositeDimension:
assert isinstance(dimension, CompositeDimension)
encoded_chunks = []
for chunk in dimension.enumerate_chunks():
if dimension.chunks_type is ContinuousDimension:
encoded_chunks.append(OptimizerService_pb2.Dimension(ContinuousDimension=OptimizerServiceEncoder.encode_continuous_dimension(chunk)))
elif dimension.chunks_type is DiscreteDimension:
encoded_chunks.append(OptimizerService_pb2.Dimension(DiscreteDimension=OptimizerServiceEncoder.encode_discrete_dimension(chunk)))
elif dimension.chunks_type is OrdinalDimension:
encoded_chunks.append(OptimizerService_pb2.Dimension(OrdinalDimension=OptimizerServiceEncoder.encode_ordinal_dimension(chunk)))
elif dimension.chunks_type is CategoricalDimension:
encoded_chunks.append(
OptimizerService_pb2.Dimension(CategoricalDimension=OptimizerServiceEncoder.encode_categorical_dimension(chunk))
)
else:
raise TypeError(f"Unsupported chunk type: {dimension.chunks_type.__name__}")
return OptimizerService_pb2.CompositeDimension(
Name=dimension.name,
ChunkType=OptimizerServiceEncoder.dimension_types_to_pb2_types[dimension.chunks_type],
Chunks=encoded_chunks
)
def predict(self, feature_values_pandas_frame, t=None): # pylint: disable=unused-argument
# TODO: make this streaming and/or using arrow.
#
feature_values_dict = feature_values_pandas_frame.to_dict(
orient='list')
prediction_request = OptimizerService_pb2.PredictRequest(
OptimizerHandle=self.optimizer_handle,
Features=OptimizerService_pb2.Features(
FeaturesJsonString=json.dumps(feature_values_dict)))
prediction_response = self._optimizer_stub.Predict(prediction_request)
# To be compliant with the OptimizerInterface, we need to recover a single Prediction object and return it.
#
objective_predictions_pb2 = prediction_response.ObjectivePredictions
assert len(objective_predictions_pb2) == 1
only_prediction_pb2 = objective_predictions_pb2[0]
objective_name = only_prediction_pb2.ObjectiveName
valid_predictions_df = Prediction.dataframe_from_json(
only_prediction_pb2.PredictionDataFrameJsonString)
prediction = Prediction.create_prediction_from_dataframe(
objective_name=objective_name, dataframe=valid_predictions_df)
prediction.add_invalid_rows_at_missing_indices(
desired_index=feature_values_pandas_frame.index)
return prediction
def CreateOptimizer(self,
request: OptimizerService_pb2.CreateOptimizerRequest,
context): # pylint: disable=unused-argument
optimization_problem = OptimizationProblem.from_protobuf(
optimization_problem_pb2=request.OptimizationProblem)
optimizer = BayesianOptimizer(
optimization_problem=optimization_problem,
optimizer_config=BayesianOptimizerConfig.DEFAULT)
optimizer_id = self.get_next_optimizer_id()
# To avoid a race condition we acquire the lock before inserting the lock and the optimizer into their respective
# dictionaries. Otherwise we could end up with a situation where a lock is in the dictionary, but the optimizer
# is not.
optimizer_lock = self._lock_manager.RLock()
with optimizer_lock:
self._optimizer_locks_by_optimizer_id[
optimizer_id] = optimizer_lock
self._optimizers_by_id[optimizer_id] = optimizer
logging.info(f"Created optimizer {optimizer_id}.")
return OptimizerService_pb2.OptimizerHandle(Id=optimizer_id)
def optimizer_handle(self):
return OptimizerService_pb2.OptimizerHandle(Id=self.id)
