def predict(self, parameter_values_pandas_frame, t=None, context_values_pandas_frame=None): # pylint: disable=unused-argument # TODO: make this streaming and/or using arrow. # if context_values_pandas_frame is not None: raise NotImplementedError( "Context not currently supported on remote optimizers") feature_values_dict = parameter_values_pandas_frame.to_dict( orient='list') prediction_request = OptimizerService_pb2.PredictRequest( OptimizerHandle=self.optimizer_handle, Features=OptimizerService_pb2.Features( FeaturesJsonString=json.dumps(feature_values_dict))) prediction_response = self._optimizer_stub.Predict(prediction_request) # To be compliant with the OptimizerBase, we need to recover a single Prediction object and return it. # objective_predictions_pb2 = prediction_response.ObjectivePredictions assert len(objective_predictions_pb2) == 1 only_prediction_pb2 = objective_predictions_pb2[0] objective_name = only_prediction_pb2.ObjectiveName valid_predictions_df = Prediction.dataframe_from_json( only_prediction_pb2.PredictionDataFrameJsonString) prediction = Prediction.create_prediction_from_dataframe( objective_name=objective_name, dataframe=valid_predictions_df) return prediction
def encode_optimization_problem(optimization_problem: OptimizationProblem) -> OptimizerService_pb2.OptimizationProblem: return OptimizerService_pb2.OptimizationProblem( ParameterSpace=OptimizerService_pb2.Hypergrid(HypergridJsonString=json.dumps(optimization_problem.parameter_space, cls=HypergridJsonEncoder)), ObjectiveSpace=OptimizerService_pb2.Hypergrid(HypergridJsonString=json.dumps(optimization_problem.objective_space, cls=HypergridJsonEncoder)), Objectives=[OptimizerService_pb2.Objective(Name=objective.name, Minimize=objective.minimize) for objective in optimization_problem.objectives], ContextSpace=None if optimization_problem.context_space is None else OptimizerService_pb2.Hypergrid(HypergridJsonString=json.dumps(optimization_problem.context_space, cls=HypergridJsonEncoder)) )
def register(self, feature_values_pandas_frame, target_values_pandas_frame): register_request = OptimizerService_pb2.RegisterObservationsRequest( OptimizerHandle=self.optimizer_handle, Observations=OptimizerService_pb2.Observations( Features=OptimizerService_pb2.Features(FeaturesJsonString=feature_values_pandas_frame.to_json(orient='index', double_precision=15)), ObjectiveValues=OptimizerService_pb2.ObjectiveValues( ObjectiveValuesJsonString=target_values_pandas_frame.to_json(orient='index', double_precision=15) ) ) ) self._optimizer_stub.RegisterObservations(register_request)
def encode_primitive_value(value: Union[int, float, bool, str]) -> OptimizerService_pb2.PrimitiveValue: assert isinstance(value, (int, float, bool, str)) if isinstance(value, bool): return OptimizerService_pb2.PrimitiveValue(BoolValue=value) if isinstance(value, int): return OptimizerService_pb2.PrimitiveValue(IntValue=value) if isinstance(value, float): return OptimizerService_pb2.PrimitiveValue(DoubleValue=value) if isinstance(value, str): return OptimizerService_pb2.PrimitiveValue(StringValue=value) raise TypeError(f"{value} is of type: {type(value)} but must be one of (int, float, bool, str)")
def register(self, parameter_values_pandas_frame, target_values_pandas_frame, context_values_pandas_frame=None): if context_values_pandas_frame is not None: raise NotImplementedError("Context not currently supported on remote optimizers") feature_values_pandas_frame = parameter_values_pandas_frame register_request = OptimizerService_pb2.RegisterObservationsRequest( OptimizerHandle=self.optimizer_handle_for_optimizer_service, Observations=OptimizerService_pb2.Observations( Features=OptimizerService_pb2.Features(FeaturesJsonString=feature_values_pandas_frame.to_json(orient='index', double_precision=15)), ObjectiveValues=OptimizerService_pb2.ObjectiveValues( ObjectiveValuesJsonString=target_values_pandas_frame.to_json(orient='index', double_precision=15) ) ) ) self._optimizer_stub.RegisterObservations(register_request) # TODO: we should be using the optimizer_stub for this.
def encode_ordinal_dimension(dimension: OrdinalDimension) -> OptimizerService_pb2.OrdinalDimension: assert isinstance(dimension, OrdinalDimension) return OptimizerService_pb2.OrdinalDimension( Name=dimension.name, Ascending=dimension.ascending, OrderedValues=[OptimizerServiceEncoder.encode_primitive_value(value) for value in dimension.values] )
def CreateOptimizer(self, request: OptimizerService_pb2.CreateOptimizerRequest, context): # pylint: disable=unused-argument self.logger.info("Creating Optimizer") print("CREATING OPTIMIZER") optimization_problem = OptimizationProblem.from_protobuf( optimization_problem_pb2=request.OptimizationProblem) optimizer_config_json = request.OptimizerConfig if optimizer_config_json is not None and len( optimizer_config_json) > 0: optimizer_config = Point.from_json(optimizer_config_json) else: optimizer_config = bayesian_optimizer_config_store.default optimizer = BayesianOptimizer( optimization_problem=optimization_problem, optimizer_config=optimizer_config) optimizer_id = self.get_next_optimizer_id() # To avoid a race condition we acquire the lock before inserting the lock and the optimizer into their respective # dictionaries. Otherwise we could end up with a situation where a lock is in the dictionary, but the optimizer # is not. optimizer_lock = self._lock_manager.RLock() with optimizer_lock: self._optimizer_locks_by_optimizer_id[ optimizer_id] = optimizer_lock self._optimizers_by_id[optimizer_id] = optimizer self._ordered_ids.append(optimizer_id) self.logger.info(f"Created optimizer {optimizer_id}.") return OptimizerService_pb2.OptimizerHandle(Id=optimizer_id)
def encode_discrete_dimension( dimension: DiscreteDimension ) -> OptimizerService_pb2.DiscreteDimension: assert isinstance(dimension, DiscreteDimension) return OptimizerService_pb2.DiscreteDimension(Name=dimension.name, Min=dimension.min, Max=dimension.max)
def encode_optimization_problem( optimization_problem: OptimizationProblem ) -> OptimizerService_pb2.OptimizationProblem: return OptimizerService_pb2.OptimizationProblem( ParameterSpace=OptimizerServiceEncoder.encode_hypergrid( optimization_problem.parameter_space), ObjectiveSpace=OptimizerServiceEncoder.encode_hypergrid( optimization_problem.objective_space), Objectives=[ OptimizerService_pb2.Objective(Name=objective.name, Minimize=objective.minimize) for objective in optimization_problem.objectives ], ContextSpace=None if optimization_problem.context_space is None else OptimizerServiceEncoder.encode_hypergrid( optimization_problem.context_space))
def Predict(self, request, context): # pylint: disable=unused-argument features_dict = json.loads(request.Features.FeaturesJsonString) features_df = pd.DataFrame(features_dict) with self.exclusive_optimizer( optimizer_id=request.OptimizerHandle.Id) as optimizer: prediction = optimizer.predict(features_df) assert isinstance(prediction, Prediction) response = OptimizerService_pb2.PredictResponse(ObjectivePredictions=[ OptimizerService_pb2.SingleObjectivePrediction( ObjectiveName=prediction.objective_name, PredictionDataFrameJsonString=prediction.dataframe_to_json()) ]) return response
def suggest(self, random=False, context=None): # pylint: disable=unused-argument suggestion_request = OptimizerService_pb2.SuggestRequest( OptimizerHandle=self.optimizer_handle, Random=random) suggestion_response = self._optimizer_stub.Suggest(suggestion_request) suggested_params_dict = json.loads( suggestion_response.ParametersJsonString) return Point(**suggested_params_dict)
def encode_empty_dimension( dimension: EmptyDimension) -> OptimizerService_pb2.EmptyDimension: assert isinstance(dimension, EmptyDimension) return OptimizerService_pb2.EmptyDimension( Name=dimension.name, DimensionType=OptimizerServiceEncoder.dimension_types_to_pb2_types[ dimension.type])
def encode_subgrid( subgrid: SimpleHypergrid.JoinedSubgrid ) -> OptimizerService_pb2.GuestSubgrid: assert isinstance(subgrid, SimpleHypergrid.JoinedSubgrid) return OptimizerService_pb2.GuestSubgrid( Subgrid=OptimizerServiceEncoder.encode_hypergrid(subgrid.subgrid), ExternalPivotDimension=OptimizerServiceEncoder.encode_dimension( subgrid.join_dimension))
def encode_continuous_dimension(dimension: ContinuousDimension) -> OptimizerService_pb2.ContinuousDimension: assert isinstance(dimension, ContinuousDimension) return OptimizerService_pb2.ContinuousDimension( Name=dimension.name, Min=dimension.min, Max=dimension.max, IncludeMin=dimension.include_min, IncludeMax=dimension.include_max )
def encode_categorical_dimension( dimension: CategoricalDimension ) -> OptimizerService_pb2.CategoricalDimension: assert isinstance(dimension, CategoricalDimension) return OptimizerService_pb2.CategoricalDimension( Name=dimension.name, Values=[ OptimizerServiceEncoder.encode_primitive_value(value) for value in dimension.values ])
def Suggest(self, request, context): # pylint: disable=unused-argument # TODO: return an error if optimizer not found # with self.exclusive_optimizer( optimizer_id=request.OptimizerHandle.Id) as optimizer: suggested_params = optimizer.suggest(random=request.Random, context=request.Context) return OptimizerService_pb2.ConfigurationParameters( ParametersJsonString=json.dumps(suggested_params.to_dict()))
def to_protobuf(self): """ Serializes self to a protobuf. :return: """ return OptimizerService_pb2.OptimizationProblem( ParameterSpace=OptimizerService_pb2.Hypergrid( HypergridJsonString=json.dumps(self.parameter_space, cls=HypergridJsonEncoder)), ObjectiveSpace=OptimizerService_pb2.Hypergrid( HypergridJsonString=json.dumps(self.objective_space, cls=HypergridJsonEncoder)), Objectives=[ OptimizerService_pb2.Objective(Name=objective.name, Minimize=objective.minimize) for objective in self.objectives ], ContextSpace=None if self.context_space is None else OptimizerService_pb2.Hypergrid(HypergridJsonString=json.dumps( self.context_space, cls=HypergridJsonEncoder)))
def suggest(self, random=False, context=None): # pylint: disable=unused-argument if context is not None: raise NotImplementedError("Context not currently supported on remote optimizers") suggestion_request = OptimizerService_pb2.SuggestRequest( OptimizerHandle=self.optimizer_handle_for_optimizer_service, Random=random, Context=context ) suggestion_response = self._optimizer_stub.Suggest(suggestion_request) suggested_params_dict = json.loads(suggestion_response.ParametersJsonString) return Point(**suggested_params_dict)
def register(self, feature_values_pandas_frame, target_values_pandas_frame): # pylint: disable=unused-argument # TODO: implement RegisterObservations <- plural # features_dicts_per_record = feature_values_pandas_frame.to_dict( orient='records') objectives_dicts_per_record = target_values_pandas_frame.to_dict( orient='records') # TODO: Either implement streaming or arrow flight or batch. # for feature_dict, objective_dict in zip(features_dicts_per_record, objectives_dicts_per_record): register_request = OptimizerService_pb2.RegisterObservationRequest( OptimizerHandle=self.optimizer_handle, Observation=OptimizerService_pb2.Observation( Features=OptimizerService_pb2.Features( FeaturesJsonString=json.dumps(feature_dict)), ObjectiveValues=OptimizerService_pb2.ObjectiveValues( ObjectiveValuesJsonString=json.dumps(objective_dict)))) self._optimizer_stub.RegisterObservation(register_request)
def encode_dimension( dimension: Dimension) -> OptimizerService_pb2.Dimension: if isinstance(dimension, EmptyDimension): return OptimizerService_pb2.Dimension( EmptyDimension=OptimizerServiceEncoder.encode_empty_dimension( dimension)) if isinstance(dimension, ContinuousDimension): return OptimizerService_pb2.Dimension( ContinuousDimension=OptimizerServiceEncoder. encode_continuous_dimension(dimension)) if isinstance(dimension, DiscreteDimension): return OptimizerService_pb2.Dimension( DiscreteDimension=OptimizerServiceEncoder. encode_discrete_dimension(dimension)) if isinstance(dimension, OrdinalDimension): return OptimizerService_pb2.Dimension( OrdinalDimension=OptimizerServiceEncoder. encode_ordinal_dimension(dimension)) if isinstance(dimension, CategoricalDimension): return OptimizerService_pb2.Dimension( CategoricalDimension=OptimizerServiceEncoder. encode_categorical_dimension(dimension)) if isinstance(dimension, CompositeDimension): return OptimizerService_pb2.Dimension( CompositeDimension=OptimizerServiceEncoder. encode_composite_dimension(dimension)) raise TypeError(f"Unsupported dimension type: {type(dimension)}")
def encode_simple_hypergrid(hypergrid: SimpleHypergrid) -> OptimizerService_pb2.SimpleHypergrid: assert isinstance(hypergrid, SimpleHypergrid) encoded_subgrids = [] for _, subgrids in hypergrid.joined_subgrids_by_pivot_dimension.items(): for subgrid in subgrids: encoded_subgrid = OptimizerServiceEncoder.encode_subgrid(subgrid) encoded_subgrids.append(encoded_subgrid) return OptimizerService_pb2.SimpleHypergrid( Name=hypergrid.name, Dimensions=[OptimizerServiceEncoder.encode_dimension(dimension) for dimension in hypergrid.root_dimensions], GuestSubgrids=encoded_subgrids )
def Suggest(self, request, context): # pylint: disable=unused-argument self.logger.info("Suggesting") # TODO: return an error if optimizer not found # if request.Context.ContextJsonString != "": raise NotImplementedError("Context not currently supported in remote optimizers") with self.exclusive_optimizer(optimizer_id=request.OptimizerHandle.Id) as optimizer: # TODO handle context here suggested_params = optimizer.suggest(random=request.Random) return OptimizerService_pb2.ConfigurationParameters( ParametersJsonString=json.dumps(suggested_params.to_dict()) )
def encode_composite_dimension(dimension: CompositeDimension) -> OptimizerService_pb2.CompositeDimension: assert isinstance(dimension, CompositeDimension) encoded_chunks = [] for chunk in dimension.enumerate_chunks(): if dimension.chunks_type is ContinuousDimension: encoded_chunks.append(OptimizerService_pb2.Dimension(ContinuousDimension=OptimizerServiceEncoder.encode_continuous_dimension(chunk))) elif dimension.chunks_type is DiscreteDimension: encoded_chunks.append(OptimizerService_pb2.Dimension(DiscreteDimension=OptimizerServiceEncoder.encode_discrete_dimension(chunk))) elif dimension.chunks_type is OrdinalDimension: encoded_chunks.append(OptimizerService_pb2.Dimension(OrdinalDimension=OptimizerServiceEncoder.encode_ordinal_dimension(chunk))) elif dimension.chunks_type is CategoricalDimension: encoded_chunks.append( OptimizerService_pb2.Dimension(CategoricalDimension=OptimizerServiceEncoder.encode_categorical_dimension(chunk)) ) else: raise TypeError(f"Unsupported chunk type: {dimension.chunks_type.__name__}") return OptimizerService_pb2.CompositeDimension( Name=dimension.name, ChunkType=OptimizerServiceEncoder.dimension_types_to_pb2_types[dimension.chunks_type], Chunks=encoded_chunks )
def predict(self, feature_values_pandas_frame, t=None): # pylint: disable=unused-argument # TODO: make this streaming and/or using arrow. # feature_values_dict = feature_values_pandas_frame.to_dict( orient='list') prediction_request = OptimizerService_pb2.PredictRequest( OptimizerHandle=self.optimizer_handle, Features=OptimizerService_pb2.Features( FeaturesJsonString=json.dumps(feature_values_dict))) prediction_response = self._optimizer_stub.Predict(prediction_request) # To be compliant with the OptimizerInterface, we need to recover a single Prediction object and return it. # objective_predictions_pb2 = prediction_response.ObjectivePredictions assert len(objective_predictions_pb2) == 1 only_prediction_pb2 = objective_predictions_pb2[0] objective_name = only_prediction_pb2.ObjectiveName valid_predictions_df = Prediction.dataframe_from_json( only_prediction_pb2.PredictionDataFrameJsonString) prediction = Prediction.create_prediction_from_dataframe( objective_name=objective_name, dataframe=valid_predictions_df) prediction.add_invalid_rows_at_missing_indices( desired_index=feature_values_pandas_frame.index) return prediction
def CreateOptimizer(self, request: OptimizerService_pb2.CreateOptimizerRequest, context): # pylint: disable=unused-argument optimization_problem = OptimizationProblem.from_protobuf( optimization_problem_pb2=request.OptimizationProblem) optimizer = BayesianOptimizer( optimization_problem=optimization_problem, optimizer_config=BayesianOptimizerConfig.DEFAULT) optimizer_id = self.get_next_optimizer_id() # To avoid a race condition we acquire the lock before inserting the lock and the optimizer into their respective # dictionaries. Otherwise we could end up with a situation where a lock is in the dictionary, but the optimizer # is not. optimizer_lock = self._lock_manager.RLock() with optimizer_lock: self._optimizer_locks_by_optimizer_id[ optimizer_id] = optimizer_lock self._optimizers_by_id[optimizer_id] = optimizer logging.info(f"Created optimizer {optimizer_id}.") return OptimizerService_pb2.OptimizerHandle(Id=optimizer_id)
def optimizer_handle(self): return OptimizerService_pb2.OptimizerHandle(Id=self.id)