def simple_experiment_from_json( object_json: Dict[str, Any]) -> SimpleExperiment: """Load AE SimpleExperiment from JSON.""" experiment_info = _get_experiment_info(object_json) description_json = object_json.pop("description") is_test_json = object_json.pop("is_test") optimization_config = object_from_json( object_json.pop("optimization_config")) # not relevant to simple experiment del object_json["tracking_metrics"] del object_json["runner"] kwargs = {k: object_from_json(v) for k, v in object_json.items()} kwargs["evaluation_function"] = unimplemented_evaluation_function kwargs["objective_name"] = optimization_config.objective.metric.name kwargs["minimize"] = optimization_config.objective.minimize kwargs["outcome_constraints"] = optimization_config.outcome_constraints experiment = SimpleExperiment(**kwargs) experiment.description = object_from_json(description_json) experiment.is_test = object_from_json(is_test_json) _load_experiment_info(exp=experiment, exp_info=experiment_info) return experiment
def simple_experiment_from_json(object_json: Dict[str, Any]) -> SimpleExperiment: """Load AE SimpleExperiment from JSON.""" time_created_json = object_json.pop("time_created") trials_json = object_json.pop("trials") experiment_type_json = object_json.pop("experiment_type") data_by_trial_json = object_json.pop("data_by_trial") description_json = object_json.pop("description") is_test_json = object_json.pop("is_test") optimization_config = object_from_json(object_json.pop("optimization_config")) # not relevant to simple experiment del object_json["tracking_metrics"] del object_json["runner"] kwargs = {k: object_from_json(v) for k, v in object_json.items()} kwargs["evaluation_function"] = unimplemented_evaluation_function kwargs["objective_name"] = optimization_config.objective.metric.name kwargs["minimize"] = optimization_config.objective.minimize kwargs["outcome_constraints"] = optimization_config.outcome_constraints experiment = SimpleExperiment(**kwargs) experiment.description = object_from_json(description_json) experiment.is_test = object_from_json(is_test_json) experiment._time_created = object_from_json(time_created_json) experiment._trials = trials_from_json(experiment, trials_json) for trial in experiment._trials.values(): for arm in trial.arms: experiment._arms_by_signature[arm.signature] = arm if experiment.status_quo is not None: sq_sig = experiment.status_quo.signature experiment._arms_by_signature[sq_sig] = experiment.status_quo experiment._experiment_type = object_from_json(experiment_type_json) experiment._data_by_trial = data_from_json(data_by_trial_json) return experiment
def _init_experiment_from_sqa(self, experiment_sqa: SQAExperiment) -> Experiment: """First step of conversion within experiment_from_sqa.""" opt_config, tracking_metrics = self.opt_config_and_tracking_metrics_from_sqa( metrics_sqa=experiment_sqa.metrics ) search_space = self.search_space_from_sqa( parameters_sqa=experiment_sqa.parameters, parameter_constraints_sqa=experiment_sqa.parameter_constraints, ) if search_space is None: raise SQADecodeError( # pragma: no cover "Experiment SearchSpace cannot be None." ) status_quo = ( Arm( parameters=experiment_sqa.status_quo_parameters, name=experiment_sqa.status_quo_name, ) if experiment_sqa.status_quo_parameters is not None else None ) if len(experiment_sqa.runners) == 0: runner = None elif len(experiment_sqa.runners) == 1: runner = self.runner_from_sqa(experiment_sqa.runners[0]) else: raise ValueError( # pragma: no cover "Multiple runners on experiment " "only supported for MultiTypeExperiment." ) subclass = (experiment_sqa.properties or {}).get("subclass") if subclass == "SimpleExperiment": if opt_config is None: raise SQADecodeError( # pragma: no cover "SimpleExperiment must have an optimization config." ) experiment = SimpleExperiment( name=experiment_sqa.name, search_space=search_space, objective_name=opt_config.objective.metric.name, minimize=opt_config.objective.minimize, outcome_constraints=opt_config.outcome_constraints, status_quo=status_quo, ) experiment.description = experiment_sqa.description experiment.is_test = experiment_sqa.is_test else: experiment = Experiment( name=experiment_sqa.name, description=experiment_sqa.description, search_space=search_space, optimization_config=opt_config, tracking_metrics=tracking_metrics, runner=runner, status_quo=status_quo, is_test=experiment_sqa.is_test, ) return experiment
def get_simple_experiment() -> SimpleExperiment: experiment = SimpleExperiment(name="test_branin", search_space=get_branin_search_space(), objective_name="sum") experiment.description = "foobar" return experiment
def get_simple_experiment() -> SimpleExperiment: experiment = SimpleExperiment( name="test_branin", search_space=get_branin_search_space(), status_quo=Arm(parameters={"x1": 0.0, "x2": 0.0}), objective_name="sum", ) experiment.description = "foobar" return experiment
def _init_experiment_from_sqa(self, experiment_sqa: SQAExperiment) -> Experiment: """First step of conversion within experiment_from_sqa.""" opt_config, tracking_metrics = self.opt_config_and_tracking_metrics_from_sqa( metrics_sqa=experiment_sqa.metrics) search_space = self.search_space_from_sqa( parameters_sqa=experiment_sqa.parameters, parameter_constraints_sqa=experiment_sqa.parameter_constraints, ) if search_space is None: raise SQADecodeError( # pragma: no cover "Experiment SearchSpace cannot be None.") status_quo = ( Arm( # pyre-fixme[6]: Expected `Dict[str, Optional[Union[bool, float, # int, str]]]` for 1st param but got `Optional[Dict[str, # Optional[Union[bool, float, int, str]]]]`. parameters=experiment_sqa.status_quo_parameters, name=experiment_sqa.status_quo_name, ) if experiment_sqa.status_quo_parameters is not None else None) if len(experiment_sqa.runners) == 0: runner = None elif len(experiment_sqa.runners) == 1: runner = self.runner_from_sqa(experiment_sqa.runners[0]) else: raise ValueError( # pragma: no cover "Multiple runners on experiment " "only supported for MultiTypeExperiment.") # `experiment_sqa.properties` is `sqlalchemy.ext.mutable.MutableDict` # so need to convert it to regular dict. properties = dict(experiment_sqa.properties or {}) # Remove 'subclass' from experiment's properties, since its only # used for decoding to the correct experiment subclass in storage. subclass = properties.pop(Keys.SUBCLASS, None) default_data_type = experiment_sqa.default_data_type if subclass == "SimpleExperiment": if opt_config is None: raise SQADecodeError( # pragma: no cover "SimpleExperiment must have an optimization config.") experiment = SimpleExperiment( name=experiment_sqa.name, search_space=search_space, objective_name=opt_config.objective.metric.name, minimize=opt_config.objective.minimize, outcome_constraints=opt_config.outcome_constraints, status_quo=status_quo, properties=properties, default_data_type=default_data_type, ) experiment.description = experiment_sqa.description experiment.is_test = experiment_sqa.is_test else: experiment = Experiment( name=experiment_sqa.name, description=experiment_sqa.description, search_space=search_space, optimization_config=opt_config, tracking_metrics=tracking_metrics, runner=runner, status_quo=status_quo, is_test=experiment_sqa.is_test, properties=properties, default_data_type=default_data_type, ) return experiment
def experiment_from_sqa(self, experiment_sqa: SQAExperiment) -> Experiment: """Convert SQLAlchemy Experiment to Ax Experiment.""" opt_config, tracking_metrics = self.opt_config_and_tracking_metrics_from_sqa( metrics_sqa=experiment_sqa.metrics) search_space = self.search_space_from_sqa( parameters_sqa=experiment_sqa.parameters, parameter_constraints_sqa=experiment_sqa.parameter_constraints, ) if search_space is None: raise SQADecodeError( # pragma: no cover "Experiment SearchSpace cannot be None.") runner = (self.runner_from_sqa(experiment_sqa.runner) if experiment_sqa.runner else None) status_quo = (Arm( parameters=experiment_sqa.status_quo_parameters, name=experiment_sqa.status_quo_name, ) if experiment_sqa.status_quo_parameters is not None else None) if (experiment_sqa.properties is not None and experiment_sqa.properties.get("subclass") == "SimpleExperiment"): if opt_config is None: raise SQADecodeError( # pragma: no cover "SimpleExperiment must have an optimization config.") experiment = SimpleExperiment( name=experiment_sqa.name, search_space=search_space, objective_name=opt_config.objective.metric.name, minimize=opt_config.objective.minimize, outcome_constraints=opt_config.outcome_constraints, status_quo=status_quo, ) experiment.description = experiment_sqa.description experiment.is_test = experiment_sqa.is_test else: experiment = Experiment( name=experiment_sqa.name, description=experiment_sqa.description, search_space=search_space, optimization_config=opt_config, tracking_metrics=tracking_metrics, runner=runner, status_quo=status_quo, is_test=experiment_sqa.is_test, ) trials = [ self.trial_from_sqa(trial_sqa=trial, experiment=experiment) for trial in experiment_sqa.trials ] data_by_trial = defaultdict(dict) for data_sqa in experiment_sqa.data: trial_index = data_sqa.trial_index timestamp = data_sqa.time_created data_by_trial[trial_index][timestamp] = self.data_from_sqa( data_sqa=data_sqa) data_by_trial = { trial_index: OrderedDict(sorted(data_by_timestamp.items())) for trial_index, data_by_timestamp in data_by_trial.items() } experiment._trials = {trial.index: trial for trial in trials} for trial in trials: for arm in trial.arms: experiment._arms_by_signature[arm.signature] = arm if experiment.status_quo is not None: sq_sig = experiment.status_quo.signature experiment._arms_by_signature[sq_sig] = experiment.status_quo experiment._time_created = experiment_sqa.time_created experiment._experiment_type = self.get_enum_name( value=experiment_sqa.experiment_type, enum=self.config.experiment_type_enum) experiment._data_by_trial = dict(data_by_trial) return experiment