def _init_experiment_from_sqa(self, experiment_sqa: SQAExperiment) -> Experiment:
"""First step of conversion within experiment_from_sqa."""
opt_config, tracking_metrics = self.opt_config_and_tracking_metrics_from_sqa(
metrics_sqa=experiment_sqa.metrics
)
search_space = self.search_space_from_sqa(
parameters_sqa=experiment_sqa.parameters,
parameter_constraints_sqa=experiment_sqa.parameter_constraints,
)
if search_space is None:
raise SQADecodeError( # pragma: no cover
"Experiment SearchSpace cannot be None."
)
status_quo = (
Arm(
parameters=experiment_sqa.status_quo_parameters,
name=experiment_sqa.status_quo_name,
)
if experiment_sqa.status_quo_parameters is not None
else None
)
if len(experiment_sqa.runners) == 0:
runner = None
elif len(experiment_sqa.runners) == 1:
runner = self.runner_from_sqa(experiment_sqa.runners[0])
else:
raise ValueError( # pragma: no cover
"Multiple runners on experiment "
"only supported for MultiTypeExperiment."
)
subclass = (experiment_sqa.properties or {}).get("subclass")
if subclass == "SimpleExperiment":
if opt_config is None:
raise SQADecodeError( # pragma: no cover
"SimpleExperiment must have an optimization config."
)
experiment = SimpleExperiment(
name=experiment_sqa.name,
search_space=search_space,
objective_name=opt_config.objective.metric.name,
minimize=opt_config.objective.minimize,
outcome_constraints=opt_config.outcome_constraints,
status_quo=status_quo,
)
experiment.description = experiment_sqa.description
experiment.is_test = experiment_sqa.is_test
else:
experiment = Experiment(
name=experiment_sqa.name,
description=experiment_sqa.description,
search_space=search_space,
optimization_config=opt_config,
tracking_metrics=tracking_metrics,
runner=runner,
status_quo=status_quo,
is_test=experiment_sqa.is_test,
)
return experiment
def parameter_constraint_from_sqa(
self,
parameter_constraint_sqa: SQAParameterConstraint,
parameters: List[Parameter],
) -> ParameterConstraint:
"""Convert SQLAlchemy ParameterConstraint to Ax ParameterConstraint."""
parameter_map = {p.name: p for p in parameters}
if parameter_constraint_sqa.type == ParameterConstraintType.ORDER:
lower_name = None
upper_name = None
for k, v in parameter_constraint_sqa.constraint_dict.items():
if v == 1:
lower_name = k
elif v == -1:
upper_name = k
if not lower_name or not upper_name:
raise SQADecodeError(
"Cannot decode SQAParameterConstraint because `lower_name` or "
"`upper_name` was not found."
)
lower_parameter = parameter_map[lower_name]
upper_parameter = parameter_map[upper_name]
constraint = OrderConstraint(
lower_parameter=lower_parameter, upper_parameter=upper_parameter
)
elif parameter_constraint_sqa.type == ParameterConstraintType.SUM:
# This operation is potentially very inefficient.
# It is O(#constrained_parameters * #total_parameters)
parameter_names = list(parameter_constraint_sqa.constraint_dict.keys())
constraint_parameters = [
next(
search_space_param
for search_space_param in parameters
if search_space_param.name == c_p_name
)
for c_p_name in parameter_names
]
a_values = list(parameter_constraint_sqa.constraint_dict.values())
if len(a_values) == 0:
raise SQADecodeError(
"Cannot decode SQAParameterConstraint because `constraint_dict` "
"is empty."
)
a = a_values[0]
is_upper_bound = a == 1
bound = parameter_constraint_sqa.bound * a
constraint = SumConstraint(
parameters=constraint_parameters,
is_upper_bound=is_upper_bound,
bound=bound,
)
else:
constraint = ParameterConstraint(
constraint_dict=dict(parameter_constraint_sqa.constraint_dict),
bound=parameter_constraint_sqa.bound,
)
constraint.db_id = parameter_constraint_sqa.id
return constraint
def parameter_from_sqa(self, parameter_sqa: SQAParameter) -> Parameter:
"""Convert SQLAlchemy Parameter to Ax Parameter."""
if parameter_sqa.domain_type == DomainType.RANGE:
if parameter_sqa.lower is None or parameter_sqa.upper is None:
raise SQADecodeError( # pragma: no cover
"`lower` and `upper` must be set for RangeParameter.")
parameter = RangeParameter(
name=parameter_sqa.name,
parameter_type=parameter_sqa.parameter_type,
# pyre-fixme[6]: Expected `float` for 3rd param but got
# `Optional[float]`.
lower=parameter_sqa.lower,
# pyre-fixme[6]: Expected `float` for 4th param but got
# `Optional[float]`.
upper=parameter_sqa.upper,
log_scale=parameter_sqa.log_scale or False,
digits=parameter_sqa.digits,
is_fidelity=parameter_sqa.is_fidelity or False,
target_value=parameter_sqa.target_value,
)
elif parameter_sqa.domain_type == DomainType.CHOICE:
if parameter_sqa.choice_values is None:
raise SQADecodeError( # pragma: no cover
"`values` must be set for ChoiceParameter.")
parameter = ChoiceParameter(
name=parameter_sqa.name,
parameter_type=parameter_sqa.parameter_type,
# pyre-fixme[6]: Expected `List[Optional[Union[bool, float, int,
# str]]]` for 3rd param but got `Optional[List[Optional[Union[bool,
# float, int, str]]]]`.
values=parameter_sqa.choice_values,
is_fidelity=parameter_sqa.is_fidelity or False,
target_value=parameter_sqa.target_value,
# pyre-fixme[6]: Expected `bool` for 6th param but got `Optional[bool]`.
is_ordered=parameter_sqa.is_ordered,
# pyre-fixme[6]: Expected `bool` for 7th param but got `Optional[bool]`.
is_task=parameter_sqa.is_task,
)
elif parameter_sqa.domain_type == DomainType.FIXED:
# Don't throw an error if parameter_sqa.fixed_value is None;
# that might be the actual value!
parameter = FixedParameter(
name=parameter_sqa.name,
parameter_type=parameter_sqa.parameter_type,
value=parameter_sqa.fixed_value,
is_fidelity=parameter_sqa.is_fidelity or False,
target_value=parameter_sqa.target_value,
)
else:
raise SQADecodeError(
f"Cannot decode SQAParameter because {parameter_sqa.domain_type} "
"is an invalid domain type.")
parameter.db_id = parameter_sqa.id
return parameter
def metric_from_sqa(
self, metric_sqa: SQAMetric
) -> Union[Metric, Objective, OutcomeConstraint]:
"""Convert SQLAlchemy Metric to Ax Metric, Objective, or OutcomeConstraint."""
metric_class = REVERSE_METRIC_REGISTRY.get(metric_sqa.metric_type)
if metric_class is None:
raise SQADecodeError(
f"Cannot decode SQAMetric because {metric_sqa.metric_type} "
f"is an invalid type."
)
args = self.get_init_args_from_properties(
# pyre-fixme[6]: Expected `SQABase` for ...es` but got `SQAMetric`.
object_sqa=metric_sqa,
class_=metric_class,
)
metric = metric_class(**args)
if metric_sqa.intent == MetricIntent.TRACKING:
return metric
elif metric_sqa.intent == MetricIntent.OBJECTIVE:
if metric_sqa.minimize is None:
raise SQADecodeError( # pragma: no cover
"Cannot decode SQAMetric to Objective because minimize is None."
)
# pyre-fixme[6]: Expected `bool` for 2nd param but got `Optional[bool]`.
return Objective(metric=metric, minimize=metric_sqa.minimize)
elif metric_sqa.intent == MetricIntent.OUTCOME_CONSTRAINT:
if (
metric_sqa.bound is None
or metric_sqa.op is None
or metric_sqa.relative is None
):
raise SQADecodeError( # pragma: no cover
"Cannot decode SQAMetric to OutcomeConstraint because "
"bound, op, or relative is None."
)
return OutcomeConstraint(
metric=metric,
# pyre-fixme[6]: Expected `float` for 2nd param but got
# `Optional[float]`.
bound=metric_sqa.bound,
op=metric_sqa.op,
relative=metric_sqa.relative,
)
else:
raise SQADecodeError(
f"Cannot decode SQAMetric because {metric_sqa.intent} "
f"is an invalid intent."
)
def get_init_args_from_properties(
self, object_sqa: SQABase, class_: Base
) -> Dict[str, Any]:
"""Given a SQAAlchemy instance with a properties blob, extract the
arguments required for its class's initializer.
"""
args = dict(getattr(object_sqa, "properties", None) or {})
signature = inspect.signature(class_.__init__)
exclude_args = ["self", "args", "kwargs"]
for arg, info in signature.parameters.items():
if arg in exclude_args or arg in args:
continue
value = getattr(object_sqa, arg, None)
if value is None:
# Only necessary to raise an exception if there is no default
# value for this argument
if info.default is inspect.Parameter.empty:
raise SQADecodeError(
f"Cannot decode because required argument {arg} is missing."
)
else:
# Constructor will use default value
continue # pragma: no cover
args[arg] = value
return args
def trial_from_sqa(self, trial_sqa: SQATrial,
experiment: Experiment) -> BaseTrial:
"""Convert SQLAlchemy Trial to Ax Trial."""
if trial_sqa.is_batch:
trial = BatchTrial(experiment=experiment,
optimize_for_power=trial_sqa.optimize_for_power)
generator_run_structs = [
GeneratorRunStruct(
generator_run=self.generator_run_from_sqa(
generator_run_sqa=generator_run_sqa),
weight=generator_run_sqa.weight or 1.0,
) for generator_run_sqa in trial_sqa.generator_runs
]
if trial_sqa.status_quo_name is not None:
new_generator_run_structs = []
for struct in generator_run_structs:
if (struct.generator_run.generator_run_type ==
GeneratorRunType.STATUS_QUO.name):
status_quo_weight = struct.generator_run.weights[0]
trial._status_quo = struct.generator_run.arms[0]
trial._status_quo_weight_override = status_quo_weight
else:
new_generator_run_structs.append(struct)
generator_run_structs = new_generator_run_structs
trial._generator_run_structs = generator_run_structs
trial._abandoned_arms_metadata = {
abandoned_arm_sqa.name: self.abandoned_arm_from_sqa(
abandoned_arm_sqa=abandoned_arm_sqa)
for abandoned_arm_sqa in trial_sqa.abandoned_arms
}
else:
trial = Trial(experiment=experiment)
if trial_sqa.generator_runs:
if len(trial_sqa.generator_runs) != 1:
raise SQADecodeError( # pragma: no cover
"Cannot decode SQATrial to Trial because trial is not batched "
"but has more than one generator run.")
trial._generator_run = self.generator_run_from_sqa(
generator_run_sqa=trial_sqa.generator_runs[0])
trial._index = trial_sqa.index
trial._trial_type = trial_sqa.trial_type
# Swap `DISPATCHED` for `RUNNING`, since `DISPATCHED` is deprecated and nearly
# equivalent to `RUNNING`.
trial._status = (trial_sqa.status
if trial_sqa.status != TrialStatus.DISPATCHED else
TrialStatus.RUNNING)
trial._time_created = trial_sqa.time_created
trial._time_completed = trial_sqa.time_completed
trial._time_staged = trial_sqa.time_staged
trial._time_run_started = trial_sqa.time_run_started
trial._abandoned_reason = trial_sqa.abandoned_reason
# pyre-fixme[9]: _run_metadata has type `Dict[str, Any]`; used as
# `Optional[Dict[str, Any]]`.
trial._run_metadata = (dict(trial_sqa.run_metadata)
if trial_sqa.run_metadata is not None else None)
trial._num_arms_created = trial_sqa.num_arms_created
trial._runner = (self.runner_from_sqa(trial_sqa.runner)
if trial_sqa.runner else None)
return trial
def generation_strategy_from_sqa(
self,
gs_sqa: SQAGenerationStrategy,
experiment: Optional[Experiment] = None,
reduced_state: bool = False,
) -> GenerationStrategy:
"""Convert SQALchemy generation strategy to Ax `GenerationStrategy`."""
steps = object_from_json(
gs_sqa.steps,
decoder_registry=self.config.json_decoder_registry,
class_decoder_registry=self.config.json_class_decoder_registry,
)
gs = GenerationStrategy(name=gs_sqa.name, steps=steps)
gs._curr = gs._steps[gs_sqa.curr_index]
immutable_ss_and_oc = (experiment.immutable_search_space_and_opt_config
if experiment is not None else False)
if reduced_state and gs_sqa.generator_runs:
# Only fully load the last of the generator runs, load the rest with
# reduced state.
gs._generator_runs = [
self.generator_run_from_sqa(
generator_run_sqa=gr,
reduced_state=True,
immutable_search_space_and_opt_config=immutable_ss_and_oc,
) for gr in gs_sqa.generator_runs[:-1]
]
gs._generator_runs.append(
self.generator_run_from_sqa(
generator_run_sqa=gs_sqa.generator_runs[-1],
reduced_state=False,
immutable_search_space_and_opt_config=immutable_ss_and_oc,
))
else:
gs._generator_runs = [
self.generator_run_from_sqa(
generator_run_sqa=gr,
reduced_state=False,
immutable_search_space_and_opt_config=immutable_ss_and_oc,
) for gr in gs_sqa.generator_runs
]
if len(gs._generator_runs) > 0:
# Generation strategy had an initialized model.
if experiment is None:
raise SQADecodeError(
"Cannot decode a generation strategy with a non-zero number of "
"generator runs without an experiment.")
gs._experiment = experiment
# If model in the current step was not directly from the `Models` enum,
# pass its type to `restore_model_from_generator_run`, which will then
# attempt to use this type to recreate the model.
if type(gs._curr.model) != Models:
models_enum = type(gs._curr.model)
assert issubclass(models_enum, ModelRegistryBase)
# pyre-ignore[6]: `models_enum` typing hackiness
gs._restore_model_from_generator_run(models_enum=models_enum)
else:
gs._restore_model_from_generator_run()
gs.db_id = gs_sqa.id
return gs
def generator_run_from_sqa(
self, generator_run_sqa: SQAGeneratorRun) -> GeneratorRun:
"""Convert SQLAlchemy GeneratorRun to Ax GeneratorRun."""
arms = []
weights = []
opt_config = None
search_space = None
for arm_sqa in generator_run_sqa.arms:
arms.append(self.arm_from_sqa(arm_sqa=arm_sqa))
weights.append(arm_sqa.weight)
opt_config, tracking_metrics = self.opt_config_and_tracking_metrics_from_sqa(
metrics_sqa=generator_run_sqa.metrics)
if len(tracking_metrics) > 0:
raise SQADecodeError( # pragma: no cover
"GeneratorRun should not have tracking metrics.")
search_space = self.search_space_from_sqa(
parameters_sqa=generator_run_sqa.parameters,
parameter_constraints_sqa=generator_run_sqa.parameter_constraints,
)
best_arm_predictions = None
model_predictions = None
if (generator_run_sqa.best_arm_parameters is not None
and generator_run_sqa.best_arm_predictions is not None):
best_arm = Arm(
name=generator_run_sqa.best_arm_name,
parameters=generator_run_sqa.best_arm_parameters,
)
best_arm_predictions = (
best_arm,
tuple(generator_run_sqa.best_arm_predictions),
)
model_predictions = (tuple(generator_run_sqa.model_predictions)
if generator_run_sqa.model_predictions is not None
else None)
generator_run = GeneratorRun(
arms=arms,
weights=weights,
optimization_config=opt_config,
search_space=search_space,
fit_time=generator_run_sqa.fit_time,
gen_time=generator_run_sqa.gen_time,
best_arm_predictions=best_arm_predictions,
model_predictions=model_predictions,
)
generator_run._time_created = generator_run_sqa.time_created
generator_run._generator_run_type = self.get_enum_name(
value=generator_run_sqa.generator_run_type,
enum=self.config.generator_run_type_enum,
)
generator_run._index = generator_run_sqa.index
return generator_run
def _init_mt_experiment_from_sqa(
self, experiment_sqa: SQAExperiment) -> MultiTypeExperiment:
"""First step of conversion within experiment_from_sqa."""
opt_config, tracking_metrics = self.opt_config_and_tracking_metrics_from_sqa(
metrics_sqa=experiment_sqa.metrics)
search_space = self.search_space_from_sqa(
parameters_sqa=experiment_sqa.parameters,
parameter_constraints_sqa=experiment_sqa.parameter_constraints,
)
if search_space is None:
raise SQADecodeError( # pragma: no cover
"Experiment SearchSpace cannot be None.")
status_quo = (
Arm(
# pyre-fixme[6]: Expected `Dict[str, Optional[Union[bool, float,
# int, str]]]` for 1st param but got `Optional[Dict[str,
# Optional[Union[bool, float, int, str]]]]`.
parameters=experiment_sqa.status_quo_parameters,
name=experiment_sqa.status_quo_name,
) if experiment_sqa.status_quo_parameters is not None else None)
trial_type_to_runner = {
not_none(sqa_runner.trial_type): self.runner_from_sqa(sqa_runner)
for sqa_runner in experiment_sqa.runners
}
default_trial_type = not_none(experiment_sqa.default_trial_type)
properties = experiment_sqa.properties
if properties:
# Remove 'subclass' from experiment's properties, since its only
# used for decoding to the correct experiment subclass in storage.
properties.pop(Keys.SUBCLASS, None)
default_data_type = experiment_sqa.default_data_type
experiment = MultiTypeExperiment(
name=experiment_sqa.name,
description=experiment_sqa.description,
search_space=search_space,
default_trial_type=default_trial_type,
default_runner=trial_type_to_runner[default_trial_type],
optimization_config=opt_config,
status_quo=status_quo,
properties=properties,
default_data_type=default_data_type,
)
experiment._trial_type_to_runner = trial_type_to_runner
sqa_metric_dict = {
metric.name: metric
for metric in experiment_sqa.metrics
}
for tracking_metric in tracking_metrics:
sqa_metric = sqa_metric_dict[tracking_metric.name]
experiment.add_tracking_metric(
tracking_metric,
trial_type=not_none(sqa_metric.trial_type),
canonical_name=sqa_metric.canonical_name,
)
return experiment
def runner_from_sqa(self, runner_sqa: SQARunner) -> Runner:
"""Convert SQLAlchemy Runner to Ax Runner."""
runner_class = REVERSE_RUNNER_REGISTRY.get(runner_sqa.runner_type)
if runner_class is None:
raise SQADecodeError(
f"Cannot decode SQARunner because {runner_sqa.runner_type} "
f"is an invalid type.")
args = runner_class.deserialize_init_args(
args=runner_sqa.properties or {})
# pyre-fixme[45]: Cannot instantiate abstract class `Runner`.
return runner_class(**args)
def metric_from_sqa_util(self, metric_sqa: SQAMetric) -> Metric:
"""Convert SQLAlchemy Metric to Ax Metric"""
metric_class = REVERSE_METRIC_REGISTRY.get(metric_sqa.metric_type)
if metric_class is None:
raise SQADecodeError(
f"Cannot decode SQAMetric because {metric_sqa.metric_type} "
f"is an invalid type.")
args = metric_sqa.properties or {}
args["name"] = metric_sqa.name
args["lower_is_better"] = metric_sqa.lower_is_better
args = metric_class.deserialize_init_args(args=args)
metric = metric_class(**args)
return metric
def get_enum_name(self, value: Optional[int],
enum: Optional[Enum]) -> Optional[str]:
"""Given an enum value (int) and an enum (of ints), return the
corresponding enum name. If the value is not present in the enum,
throw an error.
"""
if value is None or enum is None:
return None
try:
return enum(value).name # pyre-ignore T29651755
except ValueError:
raise SQADecodeError(f"Value {value} is invalid for enum {enum}.")
def runner_from_sqa(self, runner_sqa: SQARunner) -> Runner:
"""Convert SQLAlchemy Runner to Ax Runner."""
runner_class = REVERSE_RUNNER_REGISTRY.get(runner_sqa.runner_type)
if runner_class is None:
raise SQADecodeError(
f"Cannot decode SQARunner because {runner_sqa.runner_type} "
f"is an invalid type.")
args = self.get_init_args_from_properties(
# pyre-fixme[6]: Expected `SQABase` for ...es` but got `SQARunner`.
object_sqa=runner_sqa,
class_=runner_class,
)
return runner_class(**args)
def parameter_from_sqa(self, parameter_sqa: SQAParameter) -> Parameter:
"""Convert SQLAlchemy Parameter to Ax Parameter."""
if parameter_sqa.domain_type == DomainType.RANGE:
if parameter_sqa.lower is None or parameter_sqa.upper is None:
raise SQADecodeError( # pragma: no cover
"`lower` and `upper` must be set for RangeParameter."
)
return RangeParameter(
name=parameter_sqa.name,
parameter_type=parameter_sqa.parameter_type,
lower=parameter_sqa.lower,
upper=parameter_sqa.upper,
log_scale=parameter_sqa.log_scale or False,
digits=parameter_sqa.digits,
)
elif parameter_sqa.domain_type == DomainType.CHOICE:
if parameter_sqa.choice_values is None:
raise SQADecodeError( # pragma: no cover
"`values` must be set for ChoiceParameter."
)
return ChoiceParameter(
name=parameter_sqa.name,
parameter_type=parameter_sqa.parameter_type,
values=parameter_sqa.choice_values,
)
elif parameter_sqa.domain_type == DomainType.FIXED:
# Don't throw an error if parameter_sqa.fixed_value is None;
# that might be the actual value!
return FixedParameter(
name=parameter_sqa.name,
parameter_type=parameter_sqa.parameter_type,
value=parameter_sqa.fixed_value,
)
else:
raise SQADecodeError(
f"Cannot decode SQAParameter because {parameter_sqa.domain_type} "
"is an invalid domain type."
)
def metric_from_sqa_util(self, metric_sqa: SQAMetric) -> Metric:
"""Convert SQLAlchemy Metric to Ax Metric"""
metric_class = REVERSE_METRIC_REGISTRY.get(metric_sqa.metric_type)
if metric_class is None:
raise SQADecodeError(
f"Cannot decode SQAMetric because {metric_sqa.metric_type} "
f"is an invalid type.")
args = self.get_init_args_from_properties(
# pyre-fixme[6]: Expected `SQABase` for ...es` but got `SQAMetric`.
object_sqa=metric_sqa,
class_=metric_class,
)
metric = metric_class(**args)
return metric
def _init_mt_experiment_from_sqa(
self, experiment_sqa: SQAExperiment
) -> MultiTypeExperiment:
"""First step of conversion within experiment_from_sqa."""
opt_config, tracking_metrics = self.opt_config_and_tracking_metrics_from_sqa(
metrics_sqa=experiment_sqa.metrics
)
search_space = self.search_space_from_sqa(
parameters_sqa=experiment_sqa.parameters,
parameter_constraints_sqa=experiment_sqa.parameter_constraints,
)
if search_space is None:
raise SQADecodeError( # pragma: no cover
"Experiment SearchSpace cannot be None."
)
status_quo = (
Arm(
parameters=experiment_sqa.status_quo_parameters,
name=experiment_sqa.status_quo_name,
)
if experiment_sqa.status_quo_parameters is not None
else None
)
trial_type_to_runner = {
not_none(sqa_runner.trial_type): self.runner_from_sqa(sqa_runner)
for sqa_runner in experiment_sqa.runners
}
default_trial_type = not_none(experiment_sqa.default_trial_type)
experiment = MultiTypeExperiment(
name=experiment_sqa.name,
search_space=search_space,
default_trial_type=default_trial_type,
default_runner=trial_type_to_runner[default_trial_type],
optimization_config=opt_config,
status_quo=status_quo,
)
experiment._trial_type_to_runner = trial_type_to_runner
sqa_metric_dict = {metric.name: metric for metric in experiment_sqa.metrics}
for tracking_metric in tracking_metrics:
sqa_metric = sqa_metric_dict[tracking_metric.name]
experiment.add_tracking_metric(
tracking_metric,
trial_type=not_none(sqa_metric.trial_type),
canonical_name=sqa_metric.canonical_name,
)
return experiment
def metric_from_sqa_util(self, metric_sqa: SQAMetric) -> Metric:
"""Convert SQLAlchemy Metric to Ax Metric"""
metric_class = REVERSE_METRIC_REGISTRY.get(metric_sqa.metric_type)
if metric_class is None:
raise SQADecodeError(
f"Cannot decode SQAMetric because {metric_sqa.metric_type} "
f"is an invalid type.")
args = dict(
object_from_json(
metric_sqa.properties,
decoder_registry=self.config.json_decoder_registry,
class_decoder_registry=self.config.json_class_decoder_registry,
) or {})
args["name"] = metric_sqa.name
args["lower_is_better"] = metric_sqa.lower_is_better
args = metric_class.deserialize_init_args(args=args)
metric = metric_class(**args)
metric.db_id = metric_sqa.id
return metric
def _init_experiment_from_sqa(self,
experiment_sqa: SQAExperiment) -> Experiment:
"""First step of conversion within experiment_from_sqa."""
opt_config, tracking_metrics = self.opt_config_and_tracking_metrics_from_sqa(
metrics_sqa=experiment_sqa.metrics)
search_space = self.search_space_from_sqa(
parameters_sqa=experiment_sqa.parameters,
parameter_constraints_sqa=experiment_sqa.parameter_constraints,
)
if search_space is None:
raise SQADecodeError( # pragma: no cover
"Experiment SearchSpace cannot be None.")
status_quo = (Arm(
parameters=experiment_sqa.status_quo_parameters,
name=experiment_sqa.status_quo_name,
) if experiment_sqa.status_quo_parameters is not None else None)
if len(experiment_sqa.runners) == 0:
runner = None
elif len(experiment_sqa.runners) == 1:
runner = self.runner_from_sqa(experiment_sqa.runners[0])
else:
raise ValueError( # pragma: no cover
"Multiple runners on experiment "
"only supported for MultiTypeExperiment.")
# `experiment_sqa.properties` is `sqlalchemy.ext.mutable.MutableDict`
# so need to convert it to regular dict.
properties = dict(experiment_sqa.properties or {})
default_data_type = experiment_sqa.default_data_type
return Experiment(
name=experiment_sqa.name,
description=experiment_sqa.description,
search_space=search_space,
optimization_config=opt_config,
tracking_metrics=tracking_metrics,
runner=runner,
status_quo=status_quo,
is_test=experiment_sqa.is_test,
properties=properties,
default_data_type=default_data_type,
)
def trial_from_sqa(self,
trial_sqa: SQATrial,
experiment: Experiment,
reduced_state: bool = False) -> BaseTrial:
"""Convert SQLAlchemy Trial to Ax Trial.
Args:
trial_sqa: `SQATrial` to decode.
reduced_state: Whether to load trial's generator run(s) with a slightly
reduced state (without model state, search space, and optimization config).
"""
if trial_sqa.is_batch:
trial = BatchTrial(
experiment=experiment,
optimize_for_power=trial_sqa.optimize_for_power,
ttl_seconds=trial_sqa.ttl_seconds,
index=trial_sqa.index,
)
generator_run_structs = [
GeneratorRunStruct(
generator_run=self.generator_run_from_sqa(
generator_run_sqa=generator_run_sqa,
reduced_state=reduced_state,
),
weight=generator_run_sqa.weight or 1.0,
) for generator_run_sqa in trial_sqa.generator_runs
]
if trial_sqa.status_quo_name is not None:
new_generator_run_structs = []
for struct in generator_run_structs:
if (struct.generator_run.generator_run_type ==
GeneratorRunType.STATUS_QUO.name):
status_quo_weight = struct.generator_run.weights[0]
trial._status_quo = struct.generator_run.arms[0]
trial._status_quo_weight_override = status_quo_weight
else:
new_generator_run_structs.append(struct)
generator_run_structs = new_generator_run_structs
trial._generator_run_structs = generator_run_structs
if not reduced_state:
trial._abandoned_arms_metadata = {
abandoned_arm_sqa.name: self.abandoned_arm_from_sqa(
abandoned_arm_sqa=abandoned_arm_sqa)
for abandoned_arm_sqa in trial_sqa.abandoned_arms
}
trial._refresh_arms_by_name() # Trigger cache build
else:
trial = Trial(
experiment=experiment,
ttl_seconds=trial_sqa.ttl_seconds,
index=trial_sqa.index,
)
if trial_sqa.generator_runs:
if len(trial_sqa.generator_runs) != 1:
raise SQADecodeError( # pragma: no cover
"Cannot decode SQATrial to Trial because trial is not batched "
"but has more than one generator run.")
trial._generator_run = self.generator_run_from_sqa(
generator_run_sqa=trial_sqa.generator_runs[0],
reduced_state=reduced_state,
)
trial._trial_type = trial_sqa.trial_type
# Swap `DISPATCHED` for `RUNNING`, since `DISPATCHED` is deprecated and nearly
# equivalent to `RUNNING`.
trial._status = (trial_sqa.status
if trial_sqa.status != TrialStatus.DISPATCHED else
TrialStatus.RUNNING)
trial._time_created = trial_sqa.time_created
trial._time_completed = trial_sqa.time_completed
trial._time_staged = trial_sqa.time_staged
trial._time_run_started = trial_sqa.time_run_started
trial._abandoned_reason = trial_sqa.abandoned_reason
# pyre-fixme[9]: _run_metadata has type `Dict[str, Any]`; used as
# `Optional[Dict[str, Any]]`.
# pyre-fixme[8]: Attribute has type `Dict[str, typing.Any]`; used as
# `Optional[typing.Dict[Variable[_KT], Variable[_VT]]]`.
trial._run_metadata = (
# pyre-fixme[6]: Expected `Mapping[Variable[_KT], Variable[_VT]]` for
# 1st param but got `Optional[Dict[str, typing.Any]]`.
dict(trial_sqa.run_metadata)
if trial_sqa.run_metadata is not None else None)
trial._num_arms_created = trial_sqa.num_arms_created
trial._runner = (self.runner_from_sqa(trial_sqa.runner)
if trial_sqa.runner else None)
trial._generation_step_index = trial_sqa.generation_step_index
trial._properties = trial_sqa.properties or {}
trial.db_id = trial_sqa.id
return trial
def generator_run_from_sqa(self,
generator_run_sqa: SQAGeneratorRun,
reduced_state: bool = False) -> GeneratorRun:
"""Convert SQLAlchemy GeneratorRun to Ax GeneratorRun.
Args:
generator_run_sqa: `SQAGeneratorRun` to decode.
reduced_state: Whether to load generator runs with a slightly reduced state
(without model state, search space, and optimization config).
"""
arms = []
weights = []
opt_config = None
search_space = None
for arm_sqa in generator_run_sqa.arms:
arms.append(self.arm_from_sqa(arm_sqa=arm_sqa))
weights.append(arm_sqa.weight)
if not reduced_state:
(
opt_config,
tracking_metrics,
) = self.opt_config_and_tracking_metrics_from_sqa(
metrics_sqa=generator_run_sqa.metrics)
if len(tracking_metrics) > 0:
raise SQADecodeError( # pragma: no cover
"GeneratorRun should not have tracking metrics.")
search_space = self.search_space_from_sqa(
parameters_sqa=generator_run_sqa.parameters,
parameter_constraints_sqa=generator_run_sqa.
parameter_constraints,
)
best_arm_predictions = None
model_predictions = None
if (generator_run_sqa.best_arm_parameters is not None
and generator_run_sqa.best_arm_predictions is not None):
best_arm = Arm(
name=generator_run_sqa.best_arm_name,
parameters=not_none(generator_run_sqa.best_arm_parameters),
)
best_arm_predictions = (
best_arm,
tuple(not_none(generator_run_sqa.best_arm_predictions)),
)
model_predictions = (
tuple(not_none(generator_run_sqa.model_predictions))
if generator_run_sqa.model_predictions is not None else None)
generator_run = GeneratorRun(
arms=arms,
weights=weights,
optimization_config=opt_config,
search_space=search_space,
fit_time=generator_run_sqa.fit_time,
gen_time=generator_run_sqa.gen_time,
best_arm_predictions=best_arm_predictions, # pyre-ignore[6]
model_predictions=model_predictions,
model_key=generator_run_sqa.model_key,
model_kwargs=None if reduced_state else object_from_json(
generator_run_sqa.model_kwargs),
bridge_kwargs=None if reduced_state else object_from_json(
generator_run_sqa.bridge_kwargs),
gen_metadata=None if reduced_state else object_from_json(
generator_run_sqa.gen_metadata),
model_state_after_gen=None if reduced_state else object_from_json(
generator_run_sqa.model_state_after_gen),
generation_step_index=generator_run_sqa.generation_step_index,
candidate_metadata_by_arm_signature=object_from_json(
generator_run_sqa.candidate_metadata_by_arm_signature),
)
generator_run._time_created = generator_run_sqa.time_created
generator_run._generator_run_type = self.get_enum_name(
value=generator_run_sqa.generator_run_type,
enum=self.config.generator_run_type_enum,
)
generator_run._index = generator_run_sqa.index
generator_run.db_id = generator_run_sqa.id
return generator_run
def metric_from_sqa(
self, metric_sqa: SQAMetric
) -> Union[Metric, Objective, OutcomeConstraint]:
"""Convert SQLAlchemy Metric to Ax Metric, Objective, or OutcomeConstraint."""
metric = self.metric_from_sqa_util(metric_sqa)
if metric_sqa.intent == MetricIntent.TRACKING:
return metric
elif metric_sqa.intent == MetricIntent.OBJECTIVE:
if metric_sqa.minimize is None:
raise SQADecodeError( # pragma: no cover
"Cannot decode SQAMetric to Objective because minimize is None."
)
if metric_sqa.scalarized_objective_weight is not None:
raise SQADecodeError( # pragma: no cover
"The metric corresponding to regular objective does not \
have weight attribute")
return Objective(metric=metric, minimize=metric_sqa.minimize)
elif (metric_sqa.intent == MetricIntent.MULTI_OBJECTIVE
): # metric_sqa is a parent whose children are individual
# metrics in MultiObjective
if metric_sqa.minimize is None:
raise SQADecodeError( # pragma: no cover
"Cannot decode SQAMetric to MultiObjective \
because minimize is None.")
metrics_sqa_children = metric_sqa.scalarized_objective_children_metrics
if metrics_sqa_children is None:
raise SQADecodeError( # pragma: no cover
"Cannot decode SQAMetric to MultiObjective \
because the parent metric has no children metrics.")
# Extracting metric and weight for each child
metrics = [
self.metric_from_sqa_util(child)
for child in metrics_sqa_children
]
return MultiObjective(
metrics=list(metrics),
# pyre-fixme[6]: Expected `bool` for 2nd param but got `Optional[bool]`.
minimize=metric_sqa.minimize,
)
elif (metric_sqa.intent == MetricIntent.SCALARIZED_OBJECTIVE
): # metric_sqa is a parent whose children are individual
# metrics in Scalarized Objective
if metric_sqa.minimize is None:
raise SQADecodeError( # pragma: no cover
"Cannot decode SQAMetric to Scalarized Objective \
because minimize is None.")
metrics_sqa_children = metric_sqa.scalarized_objective_children_metrics
if metrics_sqa_children is None:
raise SQADecodeError( # pragma: no cover
"Cannot decode SQAMetric to Scalarized Objective \
because the parent metric has no children metrics.")
# Extracting metric and weight for each child
metrics, weights = zip(*[(
self.metric_from_sqa_util(child),
child.scalarized_objective_weight,
) for child in metrics_sqa_children])
return ScalarizedObjective(
metrics=list(metrics),
weights=list(weights),
# pyre-fixme[6]: Expected `bool` for 3nd param but got `Optional[bool]`.
minimize=metric_sqa.minimize,
)
elif metric_sqa.intent == MetricIntent.OUTCOME_CONSTRAINT:
if (metric_sqa.bound is None or metric_sqa.op is None
or metric_sqa.relative is None):
raise SQADecodeError( # pragma: no cover
"Cannot decode SQAMetric to OutcomeConstraint because "
"bound, op, or relative is None.")
return OutcomeConstraint(
metric=metric,
# pyre-fixme[6]: Expected `float` for 2nd param but got
# `Optional[float]`.
bound=metric_sqa.bound,
op=metric_sqa.op,
relative=metric_sqa.relative,
)
elif metric_sqa.intent == MetricIntent.SCALARIZED_OUTCOME_CONSTRAINT:
if (metric_sqa.bound is None or metric_sqa.op is None
or metric_sqa.relative is None):
raise SQADecodeError( # pragma: no cover
"Cannot decode SQAMetric to Scalarized OutcomeConstraint because "
"bound, op, or relative is None.")
metrics_sqa_children = (
metric_sqa.scalarized_outcome_constraint_children_metrics)
if metrics_sqa_children is None:
raise SQADecodeError( # pragma: no cover
"Cannot decode SQAMetric to Scalarized OutcomeConstraint \
because the parent metric has no children metrics.")
# Extracting metric and weight for each child
metrics, weights = zip(*[(
self.metric_from_sqa_util(child),
child.scalarized_outcome_constraint_weight,
) for child in metrics_sqa_children])
return ScalarizedOutcomeConstraint(
metrics=list(metrics),
weights=list(weights),
# pyre-fixme[6]: Expected `float` for 2nd param but got
# `Optional[float]`.
bound=metric_sqa.bound,
op=metric_sqa.op,
relative=metric_sqa.relative,
)
elif metric_sqa.intent == MetricIntent.OBJECTIVE_THRESHOLD:
if metric_sqa.bound is None or metric_sqa.relative is None:
raise SQADecodeError( # pragma: no cover
"Cannot decode SQAMetric to ObjectiveThreshold because "
"bound, op, or relative is None.")
return ObjectiveThreshold(
metric=metric,
# pyre-fixme[6]: Expected `float` for 2nd param but got
# `Optional[float]`.
bound=metric_sqa.bound,
relative=metric_sqa.relative,
op=metric_sqa.op,
)
else:
raise SQADecodeError(
f"Cannot decode SQAMetric because {metric_sqa.intent} "
f"is an invalid intent.")
def experiment_from_sqa(self, experiment_sqa: SQAExperiment) -> Experiment:
"""Convert SQLAlchemy Experiment to Ax Experiment."""
opt_config, tracking_metrics = self.opt_config_and_tracking_metrics_from_sqa(
metrics_sqa=experiment_sqa.metrics)
search_space = self.search_space_from_sqa(
parameters_sqa=experiment_sqa.parameters,
parameter_constraints_sqa=experiment_sqa.parameter_constraints,
)
if search_space is None:
raise SQADecodeError( # pragma: no cover
"Experiment SearchSpace cannot be None.")
runner = (self.runner_from_sqa(experiment_sqa.runner)
if experiment_sqa.runner else None)
status_quo = (Arm(
parameters=experiment_sqa.status_quo_parameters,
name=experiment_sqa.status_quo_name,
) if experiment_sqa.status_quo_parameters is not None else None)
if (experiment_sqa.properties is not None
and experiment_sqa.properties.get("subclass")
== "SimpleExperiment"):
if opt_config is None:
raise SQADecodeError( # pragma: no cover
"SimpleExperiment must have an optimization config.")
experiment = SimpleExperiment(
name=experiment_sqa.name,
search_space=search_space,
objective_name=opt_config.objective.metric.name,
minimize=opt_config.objective.minimize,
outcome_constraints=opt_config.outcome_constraints,
status_quo=status_quo,
)
experiment.description = experiment_sqa.description
experiment.is_test = experiment_sqa.is_test
else:
experiment = Experiment(
name=experiment_sqa.name,
description=experiment_sqa.description,
search_space=search_space,
optimization_config=opt_config,
tracking_metrics=tracking_metrics,
runner=runner,
status_quo=status_quo,
is_test=experiment_sqa.is_test,
)
trials = [
self.trial_from_sqa(trial_sqa=trial, experiment=experiment)
for trial in experiment_sqa.trials
]
data_by_trial = defaultdict(dict)
for data_sqa in experiment_sqa.data:
trial_index = data_sqa.trial_index
timestamp = data_sqa.time_created
data_by_trial[trial_index][timestamp] = self.data_from_sqa(
data_sqa=data_sqa)
data_by_trial = {
trial_index: OrderedDict(sorted(data_by_timestamp.items()))
for trial_index, data_by_timestamp in data_by_trial.items()
}
experiment._trials = {trial.index: trial for trial in trials}
for trial in trials:
for arm in trial.arms:
experiment._arms_by_signature[arm.signature] = arm
if experiment.status_quo is not None:
sq_sig = experiment.status_quo.signature
experiment._arms_by_signature[sq_sig] = experiment.status_quo
experiment._time_created = experiment_sqa.time_created
experiment._experiment_type = self.get_enum_name(
value=experiment_sqa.experiment_type,
enum=self.config.experiment_type_enum)
experiment._data_by_trial = dict(data_by_trial)
return experiment
def generator_run_from_sqa(
self, generator_run_sqa: SQAGeneratorRun
) -> GeneratorRun:
"""Convert SQLAlchemy GeneratorRun to Ax GeneratorRun."""
arms = []
weights = []
opt_config = None
search_space = None
for arm_sqa in generator_run_sqa.arms:
arms.append(self.arm_from_sqa(arm_sqa=arm_sqa))
weights.append(arm_sqa.weight)
opt_config, tracking_metrics = self.opt_config_and_tracking_metrics_from_sqa(
metrics_sqa=generator_run_sqa.metrics
)
if len(tracking_metrics) > 0:
raise SQADecodeError( # pragma: no cover
"GeneratorRun should not have tracking metrics."
)
search_space = self.search_space_from_sqa(
parameters_sqa=generator_run_sqa.parameters,
parameter_constraints_sqa=generator_run_sqa.parameter_constraints,
)
best_arm_predictions = None
model_predictions = None
if (
generator_run_sqa.best_arm_parameters is not None
and generator_run_sqa.best_arm_predictions is not None
):
best_arm = Arm(
name=generator_run_sqa.best_arm_name,
# pyre-fixme[6]: Expected `Dict[str, Optional[Union[bool, float,
# int, str]]]` for 2nd param but got `Optional[Dict[str,
# Optional[Union[bool, float, int, str]]]]`.
parameters=generator_run_sqa.best_arm_parameters,
)
best_arm_predictions = (
best_arm,
# pyre-fixme[6]: Expected `Iterable[_T_co]` for 1st param but got
# `Optional[Tuple[Dict[str, float], Optional[Dict[str, Dict[str,
# float]]]]]`.
tuple(generator_run_sqa.best_arm_predictions),
)
model_predictions = (
# pyre-fixme[6]: Expected `Iterable[_T_co]` for 1st param but got
# `Optional[Tuple[Dict[str, List[float]], Dict[str, Dict[str,
# List[float]]]]]`.
tuple(generator_run_sqa.model_predictions)
if generator_run_sqa.model_predictions is not None
else None
)
generator_run = GeneratorRun(
arms=arms,
weights=weights,
optimization_config=opt_config,
search_space=search_space,
fit_time=generator_run_sqa.fit_time,
gen_time=generator_run_sqa.gen_time,
# pyre-fixme[6]: Expected `Optional[Tuple[Arm, Optional[Tuple[Dict[str,
# float], Optional[Dict[str, Dict[str, float]]]]]]]` for 7th param but got
# `Optional[Tuple[Arm, Tuple[Any, ...]]]`.
best_arm_predictions=best_arm_predictions,
model_predictions=model_predictions,
model_key=generator_run_sqa.model_key,
model_kwargs=object_from_json(generator_run_sqa.model_kwargs),
bridge_kwargs=object_from_json(generator_run_sqa.bridge_kwargs),
gen_metadata=object_from_json(generator_run_sqa.gen_metadata),
model_state_after_gen=object_from_json(
generator_run_sqa.model_state_after_gen
),
generation_step_index=generator_run_sqa.generation_step_index,
candidate_metadata_by_arm_signature=object_from_json(
generator_run_sqa.candidate_metadata_by_arm_signature
),
)
generator_run._time_created = generator_run_sqa.time_created
generator_run._generator_run_type = self.get_enum_name(
value=generator_run_sqa.generator_run_type,
enum=self.config.generator_run_type_enum,
)
generator_run._index = generator_run_sqa.index
return generator_run
def copy_db_ids(source: Any, target: Any, path: Optional[List[str]] = None) -> None:
"""Takes as input two objects, `source` and `target`, that should be identical,
except that `source` has _db_ids set and `target` doesn't. Recursively copies the
_db_ids from `source` to `target`.
Raise a SQADecodeError when the assumption of equality on `source` and `target`
is violated, since this method is meant to be used when returning a new
user-facing object after saving.
"""
if not path:
path = []
error_message_prefix = (
f"Error encountered while traversing source {path + [str(source)]} and "
f"target {path + [str(target)]}: "
)
if len(path) > 15:
# This shouldn't happen, but is a precaution against accidentally
# introducing infinite loops
raise SQADecodeError(error_message_prefix + "Encountered path of length > 10.")
if type(source) != type(target):
if not issubclass(type(target), type(source)):
raise SQADecodeError(
error_message_prefix + "Encountered two objects of different "
f"types: {type(source)} and {type(target)}."
)
if isinstance(source, Base):
for attr, val in source.__dict__.items():
if attr.endswith("_db_id"):
# we're at a "leaf" node; copy the db_id and return
setattr(target, attr, val)
continue
# Skip over:
# * doubly private attributes
# * _experiment (to prevent infinite loops)
# * most generator run and generation strategy metadata
# (since no Base objects are nested in there,
# and we don't have guarantees about the structure of some
# of that data, so the recursion could fail somewhere)
if attr.startswith("__") or attr in {
"_best_arm_predictions",
"_bridge_kwargs",
"_candidate_metadata_by_arm_signature",
"_curr",
"_experiment",
"_gen_metadata",
"_model_kwargs",
"_model_predictions",
"_model_state_after_gen",
"_model",
"_seen_trial_indices_by_status",
"_steps",
"analysis_scheduler",
}:
continue
# Arms are referenced twice on an Experiment object; once in
# experiment.arms_by_name/signature and once in
# trial.arms_by_name/signature. When copying db_ids, we should
# ignore the former, since it will "collapse" arms of the same
# name/signature that appear in more than one trial.
if isinstance(source, Experiment) and attr in {
"_arms_by_name",
"_arms_by_signature",
}:
continue
copy_db_ids(val, getattr(target, attr), path + [attr])
elif isinstance(source, (list, set)):
source = list(source)
target = list(target)
if len(source) != len(target):
raise SQADecodeError(
error_message_prefix + "Encountered lists of different lengths."
)
if len(source) == 0:
return
if isinstance(source[0], Base) and not isinstance(source[0], SortableBase):
raise SQADecodeError(
error_message_prefix + f"Cannot sort instances of {type(source[0])}; "
"sorting is only defined on instances of SortableBase."
)
try:
source = sorted(source)
target = sorted(target)
except TypeError as e:
raise SQADecodeError(
error_message_prefix + f"TypeError encountered during sorting: {e}"
)
for index, x in enumerate(source):
copy_db_ids(x, target[index], path + [str(index)])
elif isinstance(source, dict):
for k, v in source.items():
if k not in target:
raise SQADecodeError(
error_message_prefix + "Encountered key only present "
f"in source dictionary: {k}."
)
copy_db_ids(v, target[k], path + [k])
else:
return
def copy_db_ids(source: Any,
target: Any,
path: Optional[List[str]] = None) -> None:
"""Takes as input two objects, `source` and `target`, that should be identical,
except that `source` has _db_ids set and `target` doesn't. Recursively copies the
_db_ids from `source` to `target`.
Raise a SQADecodeError when the assumption of equality on `source` and `target`
is violated, since this method is meant to be used when returning a new
user-facing object after saving.
"""
if not path:
path = []
error_message_prefix = (
f"Error encountered while traversing source {path + [str(source)]} and "
f"target {path + [str(target)]}: ")
if len(path) > 10:
# this shouldn't happen, but is a precaution against accidentally
# introducing infinite loops
return
if type(source) != type(target):
raise SQADecodeError(error_message_prefix +
"Encountered two objects of different "
f"types: {type(source)} and {type(target)}.")
if isinstance(source, Base):
for attr, val in source.__dict__.items():
if attr.endswith("_db_id"):
# we're at a "leaf" node; copy the db_id and return
setattr(target, attr, val)
continue
# skip over _experiment to prevent infinite loops,
# and ignore doubly private attributes
if attr == "_experiment" or attr.startswith("__"):
continue
copy_db_ids(val, getattr(target, attr), path + [attr])
elif isinstance(source, (list, set)):
source = list(source)
target = list(target)
if len(source) != len(target):
raise SQADecodeError(error_message_prefix +
"Encountered lists of different lengths.")
# Safe to skip over lists of types (e.g. transforms)
if len(source) == 0 or isinstance(source[0], type):
return
if isinstance(source[0],
Base) and not isinstance(source[0], SortableBase):
raise SQADecodeError(
error_message_prefix +
f"Cannot sort instances of {type(source[0])}; "
"sorting is only defined on instances of SortableBase.")
source = sorted(source)
target = sorted(target)
for index, x in enumerate(source):
copy_db_ids(x, target[index], path + [str(index)])
elif isinstance(source, dict):
for k, v in source.items():
if k not in target:
raise SQADecodeError(error_message_prefix +
"Encountered key only present "
f"in source dictionary: {k}.")
copy_db_ids(v, target[k], path + [k])
else:
return
def generator_run_from_sqa(
self,
generator_run_sqa: SQAGeneratorRun,
reduced_state: bool,
immutable_search_space_and_opt_config: bool,
) -> GeneratorRun:
"""Convert SQLAlchemy GeneratorRun to Ax GeneratorRun.
Args:
generator_run_sqa: `SQAGeneratorRun` to decode.
reduced_state: Whether to load generator runs with a slightly reduced state
(without model state, search space, and optimization config).
immutable_search_space_and_opt_config: Whether to load generator runs
without search space and optimization config. Unlike `reduced_state`,
we do still load model state.
"""
arms = []
weights = []
opt_config = None
search_space = None
for arm_sqa in generator_run_sqa.arms:
arms.append(self.arm_from_sqa(arm_sqa=arm_sqa))
weights.append(arm_sqa.weight)
if not reduced_state and not immutable_search_space_and_opt_config:
(
opt_config,
tracking_metrics,
) = self.opt_config_and_tracking_metrics_from_sqa(
metrics_sqa=generator_run_sqa.metrics)
if len(tracking_metrics) > 0:
raise SQADecodeError( # pragma: no cover
"GeneratorRun should not have tracking metrics.")
search_space = self.search_space_from_sqa(
parameters_sqa=generator_run_sqa.parameters,
parameter_constraints_sqa=generator_run_sqa.
parameter_constraints,
)
best_arm_predictions = None
model_predictions = None
if (generator_run_sqa.best_arm_parameters is not None
and generator_run_sqa.best_arm_predictions is not None):
best_arm = Arm(
name=generator_run_sqa.best_arm_name,
parameters=not_none(generator_run_sqa.best_arm_parameters),
)
best_arm_predictions = (
best_arm,
tuple(not_none(generator_run_sqa.best_arm_predictions)),
)
model_predictions = (
tuple(not_none(generator_run_sqa.model_predictions))
if generator_run_sqa.model_predictions is not None else None)
generator_run = GeneratorRun(
arms=arms,
weights=weights,
optimization_config=opt_config,
search_space=search_space,
fit_time=generator_run_sqa.fit_time,
gen_time=generator_run_sqa.gen_time,
best_arm_predictions=best_arm_predictions, # pyre-ignore[6]
# pyre-fixme[6]: Expected `Optional[Tuple[typing.Dict[str, List[float]],
# typing.Dict[str, typing.Dict[str, List[float]]]]]` for 8th param but got
# `Optional[typing.Tuple[Union[typing.Dict[str, List[float]],
# typing.Dict[str, typing.Dict[str, List[float]]]], ...]]`.
model_predictions=model_predictions,
model_key=generator_run_sqa.model_key,
model_kwargs=None if reduced_state else object_from_json(
generator_run_sqa.model_kwargs,
decoder_registry=self.config.json_decoder_registry,
class_decoder_registry=self.config.json_class_decoder_registry,
),
bridge_kwargs=None if reduced_state else object_from_json(
generator_run_sqa.bridge_kwargs,
decoder_registry=self.config.json_decoder_registry,
class_decoder_registry=self.config.json_class_decoder_registry,
),
gen_metadata=None if reduced_state else object_from_json(
generator_run_sqa.gen_metadata,
decoder_registry=self.config.json_decoder_registry,
class_decoder_registry=self.config.json_class_decoder_registry,
),
model_state_after_gen=None if reduced_state else object_from_json(
generator_run_sqa.model_state_after_gen,
decoder_registry=self.config.json_decoder_registry,
class_decoder_registry=self.config.json_class_decoder_registry,
),
generation_step_index=generator_run_sqa.generation_step_index,
candidate_metadata_by_arm_signature=object_from_json(
generator_run_sqa.candidate_metadata_by_arm_signature,
decoder_registry=self.config.json_decoder_registry,
class_decoder_registry=self.config.json_class_decoder_registry,
),
)
generator_run._time_created = generator_run_sqa.time_created
generator_run._generator_run_type = self.get_enum_name(
value=generator_run_sqa.generator_run_type,
enum=self.config.generator_run_type_enum,
)
generator_run._index = generator_run_sqa.index
generator_run.db_id = generator_run_sqa.id
return generator_run
def _init_experiment_from_sqa(self,
experiment_sqa: SQAExperiment) -> Experiment:
"""First step of conversion within experiment_from_sqa."""
opt_config, tracking_metrics = self.opt_config_and_tracking_metrics_from_sqa(
metrics_sqa=experiment_sqa.metrics)
search_space = self.search_space_from_sqa(
parameters_sqa=experiment_sqa.parameters,
parameter_constraints_sqa=experiment_sqa.parameter_constraints,
)
if search_space is None:
raise SQADecodeError( # pragma: no cover
"Experiment SearchSpace cannot be None.")
status_quo = (
Arm(
# pyre-fixme[6]: Expected `Dict[str, Optional[Union[bool, float,
# int, str]]]` for 1st param but got `Optional[Dict[str,
# Optional[Union[bool, float, int, str]]]]`.
parameters=experiment_sqa.status_quo_parameters,
name=experiment_sqa.status_quo_name,
) if experiment_sqa.status_quo_parameters is not None else None)
if len(experiment_sqa.runners) == 0:
runner = None
elif len(experiment_sqa.runners) == 1:
runner = self.runner_from_sqa(experiment_sqa.runners[0])
else:
raise ValueError( # pragma: no cover
"Multiple runners on experiment "
"only supported for MultiTypeExperiment.")
# `experiment_sqa.properties` is `sqlalchemy.ext.mutable.MutableDict`
# so need to convert it to regular dict.
properties = dict(experiment_sqa.properties or {})
# Remove 'subclass' from experiment's properties, since its only
# used for decoding to the correct experiment subclass in storage.
subclass = properties.pop(Keys.SUBCLASS, None)
default_data_type = experiment_sqa.default_data_type
if subclass == "SimpleExperiment":
if opt_config is None:
raise SQADecodeError( # pragma: no cover
"SimpleExperiment must have an optimization config.")
experiment = SimpleExperiment(
name=experiment_sqa.name,
search_space=search_space,
objective_name=opt_config.objective.metric.name,
minimize=opt_config.objective.minimize,
outcome_constraints=opt_config.outcome_constraints,
status_quo=status_quo,
properties=properties,
default_data_type=default_data_type,
)
experiment.description = experiment_sqa.description
experiment.is_test = experiment_sqa.is_test
else:
experiment = Experiment(
name=experiment_sqa.name,
description=experiment_sqa.description,
search_space=search_space,
optimization_config=opt_config,
tracking_metrics=tracking_metrics,
runner=runner,
status_quo=status_quo,
is_test=experiment_sqa.is_test,
properties=properties,
default_data_type=default_data_type,
)
return experiment
def metric_from_sqa(
self, metric_sqa: SQAMetric
) -> Union[Metric, Objective, OutcomeConstraint]:
"""Convert SQLAlchemy Metric to Ax Metric, Objective, or OutcomeConstraint."""
metric = self.metric_from_sqa_util(metric_sqa)
if metric_sqa.intent == MetricIntent.TRACKING:
return metric
elif metric_sqa.intent == MetricIntent.OBJECTIVE:
if metric_sqa.minimize is None:
raise SQADecodeError( # pragma: no cover
"Cannot decode SQAMetric to Objective because minimize is None."
)
if metric_sqa.scalarized_objective_weight is not None:
raise SQADecodeError( # pragma: no cover
"The metric corresponding to regular objective does not \
have weight attribute")
return Objective(metric=metric, minimize=metric_sqa.minimize)
elif (metric_sqa.intent == MetricIntent.MULTI_OBJECTIVE
): # metric_sqa is a parent whose children are individual
# metrics in MultiObjective
try:
metrics_sqa_children = metric_sqa.scalarized_objective_children_metrics
except DetachedInstanceError:
metrics_sqa_children = _get_scalarized_objective_children_metrics(
metric_id=metric_sqa.id, decoder=self)
if metrics_sqa_children is None:
raise SQADecodeError( # pragma: no cover
"Cannot decode SQAMetric to MultiObjective \
because the parent metric has no children metrics.")
# Extracting metric and weight for each child
objectives = [
Objective(
metric=self.metric_from_sqa_util(metric_sqa),
minimize=metric_sqa.minimize,
) for metric_sqa in metrics_sqa_children
]
multi_objective = MultiObjective(objectives=objectives)
multi_objective.db_id = metric_sqa.id
return multi_objective
elif (metric_sqa.intent == MetricIntent.SCALARIZED_OBJECTIVE
): # metric_sqa is a parent whose children are individual
# metrics in Scalarized Objective
if metric_sqa.minimize is None:
raise SQADecodeError( # pragma: no cover
"Cannot decode SQAMetric to Scalarized Objective \
because minimize is None.")
try:
metrics_sqa_children = metric_sqa.scalarized_objective_children_metrics
except DetachedInstanceError:
metrics_sqa_children = _get_scalarized_objective_children_metrics(
metric_id=metric_sqa.id, decoder=self)
if metrics_sqa_children is None:
raise SQADecodeError( # pragma: no cover
"Cannot decode SQAMetric to Scalarized Objective \
because the parent metric has no children metrics.")
# Extracting metric and weight for each child
metrics, weights = zip(*[(
self.metric_from_sqa_util(child),
child.scalarized_objective_weight,
) for child in metrics_sqa_children])
scalarized_objective = ScalarizedObjective(
metrics=list(metrics),
weights=list(weights),
minimize=not_none(metric_sqa.minimize),
)
scalarized_objective.db_id = metric_sqa.id
return scalarized_objective
elif metric_sqa.intent == MetricIntent.OUTCOME_CONSTRAINT:
if (metric_sqa.bound is None or metric_sqa.op is None
or metric_sqa.relative is None):
raise SQADecodeError( # pragma: no cover
"Cannot decode SQAMetric to OutcomeConstraint because "
"bound, op, or relative is None.")
return OutcomeConstraint(
metric=metric,
bound=metric_sqa.bound,
op=metric_sqa.op,
relative=metric_sqa.relative,
)
elif metric_sqa.intent == MetricIntent.SCALARIZED_OUTCOME_CONSTRAINT:
if (metric_sqa.bound is None or metric_sqa.op is None
or metric_sqa.relative is None):
raise SQADecodeError( # pragma: no cover
"Cannot decode SQAMetric to Scalarized OutcomeConstraint because "
"bound, op, or relative is None.")
try:
metrics_sqa_children = (
metric_sqa.scalarized_outcome_constraint_children_metrics)
except DetachedInstanceError:
metrics_sqa_children = (
_get_scalarized_outcome_constraint_children_metrics(
metric_id=metric_sqa.id, decoder=self))
if metrics_sqa_children is None:
raise SQADecodeError( # pragma: no cover
"Cannot decode SQAMetric to Scalarized OutcomeConstraint \
because the parent metric has no children metrics.")
# Extracting metric and weight for each child
metrics, weights = zip(*[(
self.metric_from_sqa_util(child),
child.scalarized_outcome_constraint_weight,
) for child in metrics_sqa_children])
scalarized_outcome_constraint = ScalarizedOutcomeConstraint(
metrics=list(metrics),
weights=list(weights),
bound=not_none(metric_sqa.bound),
op=not_none(metric_sqa.op),
relative=not_none(metric_sqa.relative),
)
scalarized_outcome_constraint.db_id = metric_sqa.id
return scalarized_outcome_constraint
elif metric_sqa.intent == MetricIntent.OBJECTIVE_THRESHOLD:
if metric_sqa.bound is None or metric_sqa.relative is None:
raise SQADecodeError( # pragma: no cover
"Cannot decode SQAMetric to ObjectiveThreshold because "
"bound, op, or relative is None.")
ot = ObjectiveThreshold(
metric=metric,
bound=metric_sqa.bound,
relative=metric_sqa.relative,
op=metric_sqa.op,
)
# ObjectiveThreshold constructor clones the passed-in metric, which means
# the db id gets lost and so we need to reset it
ot.metric._db_id = metric.db_id
return ot
else:
raise SQADecodeError(
f"Cannot decode SQAMetric because {metric_sqa.intent} "
f"is an invalid intent.")
