def testMetricSetters(self): # Establish current metrics size self.assertEqual( len(get_optimization_config().metrics) + 1, len(self.experiment.metrics)) # Add optimization config with 1 different metric opt_config = get_optimization_config() opt_config.outcome_constraints[0].metric = Metric(name="m3") self.experiment.optimization_config = opt_config # Verify total metrics size is the same. self.assertEqual( len(get_optimization_config().metrics) + 1, len(self.experiment.metrics)) # Test adding new tracking metric self.experiment.add_tracking_metric(Metric(name="m4")) self.assertEqual( len(get_optimization_config().metrics) + 2, len(self.experiment.metrics)) # Test adding new tracking metrics self.experiment.add_tracking_metrics([Metric(name="z1")]) self.assertEqual( len(get_optimization_config().metrics) + 3, len(self.experiment.metrics)) # Verify update_tracking_metric updates the metric definition self.assertIsNone(self.experiment.metrics["m4"].lower_is_better) self.experiment.update_tracking_metric( Metric(name="m4", lower_is_better=True)) self.assertTrue(self.experiment.metrics["m4"].lower_is_better) # Verify unable to add existing metric with self.assertRaises(ValueError): self.experiment.add_tracking_metric(Metric(name="m4")) # Verify unable to add existing metric with self.assertRaises(ValueError): self.experiment.add_tracking_metrics( [Metric(name="z1"), Metric(name="m4")]) # Verify unable to add metric in optimization config with self.assertRaises(ValueError): self.experiment.add_tracking_metric(Metric(name="m1")) # Verify unable to add metric in optimization config with self.assertRaises(ValueError): self.experiment.add_tracking_metrics( [Metric(name="z2"), Metric(name="m1")]) # Cannot update metric not already on experiment with self.assertRaises(ValueError): self.experiment.update_tracking_metric(Metric(name="m5")) # Cannot remove metric not already on experiment with self.assertRaises(ValueError): self.experiment.remove_tracking_metric(metric_name="m5")
def test_split_by_arm(self): gm = {"hello": "world"} generator_run = GeneratorRun( arms=self.arms, weights=self.weights, optimization_config=get_optimization_config(), search_space=get_search_space(), gen_metadata=gm, ) generator_runs = generator_run.split_by_arm() self.assertEqual(len(generator_runs), len(self.arms)) for a, w, gr in zip(self.arms, self.weights, generator_runs): with self.subTest(a=a, w=w, gr=gr): # Make sure correct arms and weights appear in split # generator runs. self.assertEqual(gr.arms, [a]) self.assertEqual(gr.weights, [w]) self.assertEqual( gr._generator_run_type, generator_run._generator_run_type ) self.assertEqual(gr._model_key, generator_run._model_key) self.assertEqual( gr._generation_step_index, generator_run._generation_step_index ) self.assertIsNone(gr._optimization_config) self.assertIsNone(gr._search_space) self.assertIsNone(gr._gen_metadata)
def testExperimentObjectiveUpdates(self): experiment = get_experiment_with_batch_trial() save_experiment(experiment) self.assertEqual(get_session().query(SQAMetric).count(), len(experiment.metrics)) # update objective # (should perform update in place) optimization_config = get_optimization_config() objective = get_objective() objective.minimize = True optimization_config.objective = objective experiment.optimization_config = optimization_config save_experiment(experiment) self.assertEqual(get_session().query(SQAMetric).count(), len(experiment.metrics)) # replace objective # (old one should become tracking metric) optimization_config.objective = Objective(metric=Metric( name="objective")) experiment.optimization_config = optimization_config save_experiment(experiment) self.assertEqual(get_session().query(SQAMetric).count(), len(experiment.metrics)) loaded_experiment = load_experiment(experiment.name) self.assertEqual(experiment, loaded_experiment)
def test_basic(self): """Run through the benchmarking loop.""" results = full_benchmark_run( problem_groups={ self.CATEGORY_NAME: [ SimpleBenchmarkProblem(branin, noise_sd=0.4), BenchmarkProblem( name="Branin", search_space=get_branin_search_space(), optimization_config=get_branin_optimization_config(), ), BenchmarkProblem( search_space=get_branin_search_space(), optimization_config=get_optimization_config(), ), ] }, method_groups={ self.CATEGORY_NAME: [ GenerationStrategy(steps=[ GenerationStep(model=Models.SOBOL, num_trials=-1) ]) ] }, num_replications=3, num_trials=5, # Just to have it be more telling if something is broken raise_all_exceptions=True, batch_size=[[1], [3], [1]], ) self.assertEqual(len(results["Branin"]["Sobol"]), 3)
def testGenMetadata(self): gm = {"hello": "world"} generator_run = GeneratorRun( arms=self.arms, weights=self.weights, optimization_config=get_optimization_config(), search_space=get_search_space(), gen_metadata=gm, ) self.assertEqual(generator_run.gen_metadata, gm)
def testTrackingMetricsMerge(self): # Tracking and optimization metrics should get merged # m1 is on optimization_config while m3 is not exp = Experiment( name="test2", search_space=get_search_space(), optimization_config=get_optimization_config(), tracking_metrics=[Metric(name="m1"), Metric(name="m3")], ) self.assertEqual(len(exp.optimization_config.metrics) + 1, len(exp.metrics))
def testEq(self): self.assertEqual(self.experiment, self.experiment) experiment2 = Experiment( name="test2", search_space=get_search_space(), optimization_config=get_optimization_config(), status_quo=get_arm(), description="test description", ) self.assertNotEqual(self.experiment, experiment2)
def testBestArm(self): generator_run = GeneratorRun( arms=self.arms, weights=self.weights, optimization_config=get_optimization_config(), search_space=get_search_space(), best_arm_predictions=(self.arms[0], ({"a": 1.0}, {"a": {"a": 2.0}})), ) self.assertEqual( generator_run.best_arm_predictions, (self.arms[0], ({"a": 1.0}, {"a": {"a": 2.0}})), )
def testModelPredictions(self): self.assertEqual(self.unweighted_run.model_predictions, get_model_predictions()) self.assertEqual( self.unweighted_run.model_predictions_by_arm, get_model_predictions_per_arm(), ) run_no_model_predictions = GeneratorRun( arms=self.arms, weights=self.weights, optimization_config=get_optimization_config(), search_space=get_search_space(), ) self.assertIsNone(run_no_model_predictions.model_predictions) self.assertIsNone(run_no_model_predictions.model_predictions_by_arm)
def testExperimentOutcomeConstraintUpdates(self): experiment = get_experiment_with_batch_trial() save_experiment(experiment) self.assertEqual( get_session().query(SQAMetric).count(), len(experiment.metrics) ) # update outcome constraint # (should perform update in place) optimization_config = get_optimization_config() outcome_constraint = get_outcome_constraint() outcome_constraint.bound = -1.0 optimization_config.outcome_constraints = [outcome_constraint] experiment.optimization_config = optimization_config save_experiment(experiment) self.assertEqual( get_session().query(SQAMetric).count(), len(experiment.metrics) ) # add outcome constraint outcome_constraint2 = OutcomeConstraint( metric=Metric(name="outcome"), op=ComparisonOp.GEQ, bound=-0.5 ) optimization_config.outcome_constraints = [ outcome_constraint, outcome_constraint2, ] experiment.optimization_config = optimization_config save_experiment(experiment) self.assertEqual( get_session().query(SQAMetric).count(), len(experiment.metrics) ) # remove outcome constraint # (old one should become tracking metric) optimization_config.outcome_constraints = [outcome_constraint] experiment.optimization_config = optimization_config save_experiment(experiment) self.assertEqual( get_session().query(SQAMetric).count(), len(experiment.metrics) ) loaded_experiment = load_experiment(experiment.name) self.assertEqual(experiment, loaded_experiment)
def setUp(self): self.model_predictions = get_model_predictions() self.optimization_config = get_optimization_config() self.search_space = get_search_space() self.arms = get_arms() self.weights = [2, 1, 1] self.unweighted_run = GeneratorRun( arms=self.arms, optimization_config=self.optimization_config, search_space=self.search_space, model_predictions=self.model_predictions, fit_time=4.0, gen_time=10.0, ) self.weighted_run = GeneratorRun( arms=self.arms, weights=self.weights, optimization_config=self.optimization_config, search_space=self.search_space, model_predictions=self.model_predictions, )
def testBasicProperties(self): self.assertEqual(self.experiment.status_quo, get_status_quo()) self.assertEqual(self.experiment.search_space, get_search_space()) self.assertEqual(self.experiment.optimization_config, get_optimization_config()) self.assertEqual(self.experiment.is_test, True)