def get_multi_type_experiment(
add_trial_type: bool = True, add_trials: bool = False
) -> MultiTypeExperiment:
oc = OptimizationConfig(Objective(BraninMetric("m1", ["x1", "x2"])))
experiment = MultiTypeExperiment(
name="test_exp",
search_space=get_branin_search_space(),
default_trial_type="type1",
default_runner=SyntheticRunner(dummy_metadata="dummy1"),
optimization_config=oc,
)
experiment.add_trial_type(
trial_type="type2", runner=SyntheticRunner(dummy_metadata="dummy2")
)
# Switch the order of variables so metric gives different results
experiment.add_tracking_metric(
BraninMetric("m2", ["x2", "x1"]), trial_type="type2", canonical_name="m1"
)
if add_trials and add_trial_type:
generator = get_sobol(experiment.search_space)
gr = generator.gen(10)
t1 = experiment.new_batch_trial(generator_run=gr, trial_type="type1")
t2 = experiment.new_batch_trial(generator_run=gr, trial_type="type2")
t1.set_status_quo_with_weight(status_quo=t1.arms[0], weight=0.5)
t2.set_status_quo_with_weight(status_quo=t2.arms[0], weight=0.5)
t1.run()
t2.run()
return experiment
def testRunner(self):
# Verify BatchTrial without runner will fail
with self.assertRaises(ValueError):
self.batch.run()
# Verify mark running without runner will fail
with self.assertRaises(ValueError):
self.batch.mark_running()
self.batch.runner = SyntheticRunner()
self.batch.run()
self.assertEqual(self.batch.deployed_name, "test_0")
self.assertNotEqual(len(self.batch.run_metadata.keys()), 0)
self.assertEqual(self.batch.status, TrialStatus.RUNNING)
# Verify setting runner on experiment but not BatchTrial
# Also mock staging_required to be false
staging_mock = PropertyMock()
with patch.object(SyntheticRunner, "staging_required", staging_mock):
mock_runner = SyntheticRunner()
staging_mock.return_value = True
self.experiment.runner = mock_runner
b2 = self.experiment.new_batch_trial()
b2.run()
self.assertEqual(b2.deployed_name, "test_1")
self.assertEqual(b2.status, TrialStatus.STAGED)
def testExperimentRunner(self):
original_runner = SyntheticRunner()
self.experiment.runner = original_runner
batch = self.experiment.new_batch_trial()
batch.run()
self.assertEqual(batch.runner, original_runner)
# Simulate a failed run/deployment, in which the runner is attached
# but the actual run fails, and so the trial remains CANDIDATE.
candidate_batch = self.experiment.new_batch_trial()
candidate_batch.run()
candidate_batch._status = TrialStatus.CANDIDATE
self.assertEqual(self.experiment.trials_expecting_data, [batch])
tbs = self.experiment.trials_by_status # All statuses should be present
self.assertEqual(len(tbs), len(TrialStatus))
self.assertEqual(tbs[TrialStatus.RUNNING], [batch])
self.assertEqual(tbs[TrialStatus.CANDIDATE], [candidate_batch])
tibs = self.experiment.trial_indices_by_status
self.assertEqual(len(tibs), len(TrialStatus))
self.assertEqual(tibs[TrialStatus.RUNNING], {0})
self.assertEqual(tibs[TrialStatus.CANDIDATE], {1})
identifier = {"new_runner": True}
new_runner = SyntheticRunner(dummy_metadata=identifier)
self.experiment.reset_runners(new_runner)
# Don't update trials that have been run.
self.assertEqual(batch.runner, original_runner)
# Update default runner
self.assertEqual(self.experiment.runner, new_runner)
# Update candidate trial runners.
self.assertEqual(self.experiment.trials[1].runner, new_runner)
def test_immutable_search_space_and_opt_config(self):
mutable_exp = self._setupBraninExperiment(n=5)
self.assertFalse(mutable_exp.immutable_search_space_and_opt_config)
immutable_exp = Experiment(
name="test4",
search_space=get_branin_search_space(),
tracking_metrics=[
BraninMetric(name="b", param_names=["x1", "x2"])
],
optimization_config=get_branin_optimization_config(),
runner=SyntheticRunner(),
properties={Keys.IMMUTABLE_SEARCH_SPACE_AND_OPT_CONF: True},
)
self.assertTrue(immutable_exp.immutable_search_space_and_opt_config)
with self.assertRaises(UnsupportedError):
immutable_exp.optimization_config = get_branin_optimization_config(
)
immutable_exp.new_batch_trial()
with self.assertRaises(UnsupportedError):
immutable_exp.search_space = get_branin_search_space()
# Check that passing the property as just a string is processed
# correctly.
immutable_exp_2 = Experiment(
name="test4",
search_space=get_branin_search_space(),
tracking_metrics=[
BraninMetric(name="b", param_names=["x1", "x2"])
],
runner=SyntheticRunner(),
properties={Keys.IMMUTABLE_SEARCH_SPACE_AND_OPT_CONF.value: True},
)
self.assertTrue(immutable_exp_2.immutable_search_space_and_opt_config)
def get_multi_type_experiment_with_multi_objective(
add_trials: bool = False,
) -> MultiTypeExperiment:
oc = get_branin_multi_objective_optimization_config()
experiment = MultiTypeExperiment(
name="test_exp",
search_space=get_branin_search_space(),
default_trial_type="type1",
default_runner=SyntheticRunner(dummy_metadata="dummy1"),
optimization_config=oc,
)
experiment.add_trial_type(
trial_type="type2", runner=SyntheticRunner(dummy_metadata="dummy2")
)
if add_trials:
generator = get_sobol(experiment.search_space)
gr = generator.gen(10)
t1 = experiment.new_batch_trial(generator_run=gr, trial_type="type1")
t2 = experiment.new_batch_trial(generator_run=gr, trial_type="type2")
t1.set_status_quo_with_weight(status_quo=t1.arms[0], weight=0.5)
t2.set_status_quo_with_weight(status_quo=t2.arms[0], weight=0.5)
t1.run()
t2.run()
return experiment
def get_branin_experiment_with_multi_objective(
has_optimization_config: bool = True,
has_objective_thresholds: bool = False,
with_batch: bool = False,
with_status_quo: bool = False,
with_fidelity_parameter: bool = False,
num_objectives: int = 2,
) -> Experiment:
exp = Experiment(
name="branin_test_experiment",
search_space=get_branin_search_space(
with_fidelity_parameter=with_fidelity_parameter),
optimization_config=get_branin_multi_objective_optimization_config(
has_objective_thresholds=has_objective_thresholds,
num_objectives=num_objectives,
) if has_optimization_config else None,
runner=SyntheticRunner(),
is_test=True,
)
if with_status_quo:
# Experiment chooses the name "status_quo" by default
exp.status_quo = Arm(parameters={"x1": 0.0, "x2": 0.0})
if with_batch:
sobol_generator = get_sobol(search_space=exp.search_space,
seed=TEST_SOBOL_SEED)
sobol_run = sobol_generator.gen(n=5)
exp.new_batch_trial(
optimize_for_power=with_status_quo).add_generator_run(sobol_run)
return exp
def get_factorial_experiment(
has_optimization_config: bool = True,
with_batch: bool = False,
with_status_quo: bool = False,
) -> Experiment:
exp = Experiment(
name="factorial_test_experiment",
search_space=get_factorial_search_space(),
optimization_config=OptimizationConfig(
objective=Objective(metric=get_factorial_metric())
)
if has_optimization_config
else None,
runner=SyntheticRunner(),
is_test=True,
tracking_metrics=[get_factorial_metric("secondary_metric")],
)
if with_status_quo:
exp.status_quo = Arm(
parameters={
"factor1": "level11",
"factor2": "level21",
"factor3": "level31",
}
)
if with_batch:
factorial_generator = get_factorial(search_space=exp.search_space)
factorial_run = factorial_generator.gen(n=-1)
exp.new_batch_trial(optimize_for_power=with_status_quo).add_generator_run(
factorial_run
)
return exp
def testEarlyStoppedBatchTrial(self):
self.batch.runner = SyntheticRunner()
self.batch.run()
self.batch.mark_early_stopped()
self.assertEqual(self.batch.status, TrialStatus.EARLY_STOPPED)
self.assertIsNotNone(self.batch.time_completed)
def testFailedBatchTrial(self):
self.batch.runner = SyntheticRunner()
self.batch.run()
self.batch.mark_failed()
self.assertEqual(self.batch.status, TrialStatus.FAILED)
self.assertIsNotNone(self.batch.time_completed)
def get_trial() -> Trial:
experiment = get_experiment()
trial = experiment.new_trial()
arm = get_arms_from_dict(get_arm_weights1())[0]
trial.add_arm(arm)
trial.runner = SyntheticRunner()
return trial
def get_trial() -> Trial:
experiment = get_experiment()
trial = experiment.new_trial()
arm = get_arms()[0]
trial.add_arm(arm)
trial.runner = SyntheticRunner()
return trial
def get_branin_experiment(
has_optimization_config: bool = True,
with_batch: bool = False,
with_status_quo: bool = False,
with_fidelity_parameter: bool = False,
with_choice_parameter: bool = False,
search_space: Optional[SearchSpace] = None,
) -> Experiment:
search_space = search_space or get_branin_search_space(
with_fidelity_parameter=with_fidelity_parameter,
with_choice_parameter=with_choice_parameter,
)
exp = Experiment(
name="branin_test_experiment",
search_space=search_space,
optimization_config=get_branin_optimization_config()
if has_optimization_config
else None,
runner=SyntheticRunner(),
is_test=True,
)
if with_status_quo:
exp.status_quo = Arm(parameters={"x1": 0.0, "x2": 0.0})
if with_batch:
sobol_generator = get_sobol(search_space=exp.search_space)
sobol_run = sobol_generator.gen(n=15)
exp.new_batch_trial(optimize_for_power=with_status_quo).add_generator_run(
sobol_run
)
return exp
def get_trial() -> Trial:
experiment = get_experiment()
trial = experiment.new_trial(ttl_seconds=72)
arm = get_arms_from_dict(get_arm_weights1())[0]
trial.add_arm(arm)
trial.runner = SyntheticRunner()
trial._generation_step_index = 0
return trial
def get_trial() -> Trial:
experiment = get_experiment()
trial = experiment.new_trial(ttl_seconds=72)
arm = get_arms_from_dict(get_arm_weights1())[0]
trial.add_arm(arm)
trial.runner = SyntheticRunner()
# pyre-fixme[8]: Attribute has type `None`; used as `int`.
trial._generation_step_index = 0
return trial
def testBasicSetter(self):
self.batch.runner = SyntheticRunner()
self.assertIsNotNone(self.batch.runner)
self.batch.trial_type = None
self.assertIsNone(self.batch.trial_type)
# Default experiment only supports None as trial_type
with self.assertRaises(ValueError):
self.batch.trial_type = ""
def get_batch_trial() -> BatchTrial:
experiment = get_experiment()
batch = experiment.new_batch_trial()
arms = get_arms()
weights = get_weights()
batch.add_arms_and_weights(arms=arms, weights=weights, multiplier=0.75)
batch.mark_arm_abandoned(batch.arms[0].name, "abandoned reason")
batch.runner = SyntheticRunner()
batch.set_status_quo_with_weight(status_quo=arms[0], weight=0.5)
return batch
def testExperimentWithoutName(self):
exp = Experiment(
search_space=get_branin_search_space(),
tracking_metrics=[BraninMetric(name="b", param_names=["x1", "x2"])],
runner=SyntheticRunner(),
)
self.assertEqual("Experiment(None)", str(exp))
batch = exp.new_batch_trial()
batch.add_arms_and_weights(arms=get_branin_arms(n=5, seed=0))
batch.run()
self.assertEqual(batch.run_metadata, {"name": "0"})
def testBatchLifecycle(self):
staging_mock = PropertyMock()
with patch.object(SyntheticRunner, "staging_required", staging_mock):
mock_runner = SyntheticRunner()
staging_mock.return_value = True
self.batch.runner = mock_runner
self.batch.run()
self.assertEqual(self.batch.status, TrialStatus.STAGED)
self.assertIsNotNone(self.batch.time_staged)
self.assertTrue(self.batch.status.is_deployed)
self.assertFalse(self.batch.status.expecting_data)
# Cannot change arms or runner once run
with self.assertRaises(ValueError):
self.batch.add_arms_and_weights(arms=self.arms,
weights=self.weights)
with self.assertRaises(ValueError):
self.batch.runner = None
# Cannot run batch that was already run
with self.assertRaises(ValueError):
self.batch.run()
self.batch.mark_running()
self.assertEqual(self.batch.status, TrialStatus.RUNNING)
self.assertIsNotNone(self.batch.time_run_started)
self.assertTrue(self.batch.status.expecting_data)
self.batch.complete()
# Cannot complete that which is already completed
with self.assertRaises(ValueError):
self.batch.complete()
# Verify trial is completed
self.assertEqual(self.batch.status, TrialStatus.COMPLETED)
self.assertIsNotNone(self.batch.time_completed)
self.assertTrue(self.batch.status.is_terminal)
# Cannot change status after BatchTrial is completed
with self.assertRaises(ValueError):
self.batch.mark_staged()
with self.assertRaises(ValueError):
self.batch.mark_completed()
with self.assertRaises(ValueError):
self.batch.mark_running()
with self.assertRaises(ValueError):
self.batch.mark_abandoned()
with self.assertRaises(ValueError):
self.batch.mark_failed()
def testBadBehavior(self):
# Add trial type that already exists
with self.assertRaises(ValueError):
self.experiment.add_trial_type("type1", SyntheticRunner())
# Update runner for non-existent trial type
with self.assertRaises(ValueError):
self.experiment.update_runner("type3", SyntheticRunner())
# Add metric for trial_type that doesn't exist
with self.assertRaises(ValueError):
self.experiment.add_tracking_metric(
BraninMetric("m2", ["x1", "x2"]), "type3")
# Try to remove metric that doesn't exist
with self.assertRaises(ValueError):
self.experiment.remove_tracking_metric("m3")
# Try to change optimization metric to non-primary trial type
with self.assertRaises(ValueError):
self.experiment.update_tracking_metric(
BraninMetric("m1", ["x1", "x2"]), "type2")
# Update metric definition for trial_type that doesn't exist
with self.assertRaises(ValueError):
self.experiment.update_tracking_metric(
BraninMetric("m2", ["x1", "x2"]), "type3")
# Try to get runner for trial_type that's not supported
batch = self.experiment.new_batch_trial()
batch._trial_type = "type3" # Force override trial type
with self.assertRaises(ValueError):
self.experiment.runner_for_trial(batch)
# Try making trial with unsupported trial type
with self.assertRaises(ValueError):
self.experiment.new_batch_trial(trial_type="type3")
# Try resetting runners.
with self.assertRaises(NotImplementedError):
self.experiment.reset_runners(SyntheticRunner())
def get_batch_trial(abandon_arm: bool = True) -> BatchTrial:
experiment = get_experiment()
batch = experiment.new_batch_trial()
arms = get_arms_from_dict(get_arm_weights1())
weights = get_weights_from_dict(get_arm_weights1())
batch.add_arms_and_weights(arms=arms, weights=weights, multiplier=0.75)
if abandon_arm:
batch.mark_arm_abandoned(batch.arms[0].name, "abandoned reason")
batch.runner = SyntheticRunner()
batch.set_status_quo_with_weight(status_quo=arms[0], weight=0.5)
batch._generation_step_index = 0
return batch
def testExperimentRunners(self):
original_runner = SyntheticRunner()
self.experiment.runner = original_runner
batch = self.experiment.new_batch_trial()
batch.run()
self.assertEqual(batch.runner, original_runner)
# Simulate a failed run/deployment, in which the runner is attached
# but the actual run fails, and so the trial remains CANDIDATE.
candidate_batch = self.experiment.new_batch_trial()
candidate_batch.run()
candidate_batch._status = TrialStatus.CANDIDATE
identifier = {"new_runner": True}
new_runner = SyntheticRunner(dummy_metadata=identifier)
self.experiment.reset_runners(new_runner)
# Don't update trials that have been run.
self.assertEqual(batch.runner, original_runner)
# Update default runner
self.assertEqual(self.experiment.runner, new_runner)
# Update candidate trial runners.
self.assertEqual(self.experiment.trials[1].runner, new_runner)
def __init__(self,
problem: BenchmarkProblem,
total_iterations: int = 20,
batch_size: int = 1) -> None:
super().__init__(
name=problem.name,
search_space=problem.search_space,
runner=SyntheticRunner(),
optimization_config=problem.optimization_config,
)
self.problem = problem
self.total_iterations = total_iterations
self.batch_size = batch_size
def _benchmark_replication_Dev_API(
problem: BenchmarkProblem,
method: GenerationStrategy,
num_trials: int,
experiment_name: str,
batch_size: int = 1,
raise_all_exceptions: bool = False,
benchmark_trial: FunctionType = benchmark_trial,
verbose_logging: bool = True,
# Number of trials that need to fail for a replication to be considered failed.
failed_trials_tolerated: int = 5,
async_benchmark_options: Optional[AsyncBenchmarkOptions] = None,
) -> Tuple[Experiment, List[Exception]]:
"""Run a benchmark replication via the Developer API because the problem was
set up with Ax classes (likely to allow for additional complexity like
adding constraints or non-range parameters).
"""
if async_benchmark_options is not None:
raise NonRetryableBenchmarkingError(
"`async_benchmark_options` not supported when using the Dev API."
)
exceptions = []
experiment = Experiment(
name=experiment_name,
search_space=problem.search_space,
optimization_config=problem.optimization_config,
runner=SyntheticRunner(),
)
for trial_index in range(num_trials):
try:
gr = method.gen(experiment=experiment, n=batch_size)
if batch_size == 1:
trial = experiment.new_trial(generator_run=gr)
else:
assert batch_size > 1
trial = experiment.new_batch_trial(generator_run=gr)
trial.run()
# TODO[T94059549]: Rm 3 lines below when attaching data in fetch is fixed.
data = benchmark_trial(experiment=experiment, trial_index=trial_index)
if not data.df.empty:
experiment.attach_data(data=data)
except Exception as err: # TODO[T53975770]: test
if raise_all_exceptions:
raise
exceptions.append(err)
if len(exceptions) > failed_trials_tolerated:
raise RuntimeError( # TODO[T53975770]: test
f"More than {failed_trials_tolerated} failed for {experiment_name}."
)
return experiment, exceptions
def test_fetch_as_class(self):
class MyMetric(Metric):
@property
def fetch_multi_group_by_metric(self) -> Type[Metric]:
return Metric
m = MyMetric(name="test_metric")
exp = Experiment(
name="test",
search_space=get_branin_search_space(),
tracking_metrics=[m],
runner=SyntheticRunner(),
)
self.assertEqual(exp._metrics_by_class(), {Metric: [m]})
def get_experiment_with_multi_objective() -> Experiment:
optimization_config = get_multi_objective_optimization_config()
exp = Experiment(
name="test_experiment_multi_objective",
search_space=get_branin_search_space(),
optimization_config=optimization_config,
description="test experiment with multi objective",
runner=SyntheticRunner(),
tracking_metrics=[Metric(name="tracking")],
is_test=True,
)
return exp
def _setupBraninExperiment(self, n: int) -> Experiment:
exp = Experiment(
name="test3",
search_space=get_branin_search_space(),
tracking_metrics=[BraninMetric(name="b", param_names=["x1", "x2"])],
runner=SyntheticRunner(),
)
batch = exp.new_batch_trial()
batch.add_arms_and_weights(arms=get_branin_arms(n=n, seed=0))
batch.run()
batch_2 = exp.new_batch_trial()
batch_2.add_arms_and_weights(arms=get_branin_arms(n=3 * n, seed=1))
batch_2.run()
return exp
def get_branin_experiment(
has_optimization_config: bool = True,
with_batch: bool = False,
with_trial: bool = False,
with_status_quo: bool = False,
with_fidelity_parameter: bool = False,
with_choice_parameter: bool = False,
with_str_choice_param: bool = False,
search_space: Optional[SearchSpace] = None,
minimize: bool = False,
named: bool = True,
with_completed_trial: bool = False,
) -> Experiment:
search_space = search_space or get_branin_search_space(
with_fidelity_parameter=with_fidelity_parameter,
with_choice_parameter=with_choice_parameter,
with_str_choice_param=with_str_choice_param,
)
exp = Experiment(
name="branin_test_experiment" if named else None,
search_space=search_space,
optimization_config=get_branin_optimization_config(
minimize=minimize) if has_optimization_config else None,
runner=SyntheticRunner(),
is_test=True,
)
if with_status_quo:
exp.status_quo = Arm(parameters={"x1": 0.0, "x2": 0.0})
if with_batch:
sobol_generator = get_sobol(search_space=exp.search_space)
sobol_run = sobol_generator.gen(n=15)
exp.new_batch_trial(
optimize_for_power=with_status_quo).add_generator_run(sobol_run)
if with_trial or with_completed_trial:
sobol_generator = get_sobol(search_space=exp.search_space)
sobol_run = sobol_generator.gen(n=1)
trial = exp.new_trial(generator_run=sobol_run)
if with_completed_trial:
trial.mark_running(no_runner_required=True)
exp.attach_data(
get_branin_data(trials=[trial])) # Add data for one trial
trial.mark_completed()
return exp
def testMTExperimentFlow(self):
self.assertTrue(self.experiment.supports_trial_type("type1"))
self.assertTrue(self.experiment.supports_trial_type("type2"))
self.assertFalse(self.experiment.supports_trial_type(None))
n = 10
arms = get_branin_arms(n=n, seed=0)
b1 = self.experiment.new_batch_trial()
b1.add_arms_and_weights(arms=arms)
self.assertEqual(b1.trial_type, "type1")
b1.run()
self.assertEqual(b1.run_metadata["dummy_metadata"], "dummy1")
self.experiment.update_runner("type2",
SyntheticRunner(dummy_metadata="dummy3"))
b2 = self.experiment.new_batch_trial()
b2.trial_type = "type2"
b2.add_arms_and_weights(arms=arms)
self.assertEqual(b2.trial_type, "type2")
b2.run()
self.assertEqual(b2.run_metadata["dummy_metadata"], "dummy3")
df = self.experiment.fetch_data().df
for _, row in df.iterrows():
# Make sure proper metric present for each batch only
self.assertEqual(row["metric_name"],
"m1" if row["trial_index"] == 0 else "m2")
arm_0_slice = df.loc[df["arm_name"] == "0_0"]
self.assertNotEqual(
float(arm_0_slice[df["trial_index"] == 0]["mean"]),
float(arm_0_slice[df["trial_index"] == 1]["mean"]),
)
self.assertEqual(len(df), 2 * n)
# Set 2 metrics to be equal
self.experiment.update_tracking_metric(BraninMetric(
"m2", ["x1", "x2"]),
trial_type="type2")
df = self.experiment.fetch_data().df
arm_0_slice = df.loc[df["arm_name"] == "0_0"]
self.assertAlmostEqual(
float(arm_0_slice[df["trial_index"] == 0]["mean"]),
float(arm_0_slice[df["trial_index"] == 1]["mean"]),
places=10,
)
def get_experiment_with_multi_objective() -> Experiment:
objective = get_multi_objective()
outcome_constraints = [get_outcome_constraint()]
optimization_config = OptimizationConfig(
objective=objective, outcome_constraints=outcome_constraints)
exp = Experiment(
name="test_experiment_multi_objective",
search_space=get_branin_search_space(),
optimization_config=optimization_config,
description="test experiment with multi objective",
runner=SyntheticRunner(),
tracking_metrics=[Metric(name="tracking")],
is_test=True,
)
return exp
def testBasic(self) -> None:
self.assertTrue(self.experiment.is_simple_experiment)
trial = self.experiment.new_trial()
with self.assertRaises(NotImplementedError):
trial.runner = SyntheticRunner()
with self.assertRaises(NotImplementedError):
self.experiment.add_tracking_metric(Metric(name="test"))
with self.assertRaises(NotImplementedError):
self.experiment.update_tracking_metric(Metric(name="test"))
self.assertTrue(self.experiment.eval_trial(trial).df.empty)
batch = self.experiment.new_batch_trial()
batch.add_arm(Arm(parameters={"x1": 5, "x2": 10}))
self.assertEqual(self.experiment.eval_trial(batch).df["mean"][0], 15)
self.experiment.new_batch_trial().add_arm(Arm(parameters={"x1": 15, "x2": 25}))
self.assertAlmostEqual(self.experiment.eval().df["mean"][1], 40)
self.assertEqual(batch.fetch_data().df["mean"][0], 15)
self.assertAlmostEqual(self.experiment.fetch_data().df["mean"][1], 40)
