def _update_trials(experiment: Experiment, trials: List[BaseTrial],
encoder: Encoder) -> None:
"""Update trials and attach data."""
trial_sqa_class = encoder.config.class_to_sqa_class[Trial]
trial_indices = [trial.index for trial in trials]
obj_to_sqa = []
with session_scope() as session:
experiment_id = _get_experiment_id(experiment=experiment,
encoder=encoder,
session=session)
existing_trials = (
session.query(trial_sqa_class).filter_by(
experiment_id=experiment_id).filter(
trial_sqa_class.index.in_(trial_indices)) # pyre-ignore
.all())
trial_index_to_existing_trial = {
trial.index: trial
for trial in existing_trials
}
updated_sqa_trials, new_sqa_data = [], []
for trial in trials:
existing_trial = trial_index_to_existing_trial.get(trial.index)
if existing_trial is None:
raise ValueError(
f"Trial {trial.index} is not attached to the experiment.")
new_sqa_trial, _obj_to_sqa = encoder.trial_to_sqa(trial)
obj_to_sqa.extend(_obj_to_sqa)
existing_trial.update(new_sqa_trial)
updated_sqa_trials.append(existing_trial)
data, ts = experiment.lookup_data_for_trial(trial_index=trial.index)
if ts != -1:
sqa_data = encoder.data_to_sqa(data=data,
trial_index=trial.index,
timestamp=ts)
obj_to_sqa.append((data, sqa_data))
sqa_data.experiment_id = experiment_id
new_sqa_data.append(sqa_data)
with session_scope() as session:
session.add_all(updated_sqa_trials)
session.add_all(new_sqa_data)
session.flush()
_set_db_ids(obj_to_sqa=obj_to_sqa)
def _save_new_trial(experiment: Experiment, trial: BaseTrial, encoder: Encoder) -> None:
"""Add new trial to the experiment, using given Encoder instance."""
exp_sqa_class = encoder.config.class_to_sqa_class[Experiment]
with session_scope() as session:
existing_sqa_experiment = (
session.query(exp_sqa_class).filter_by(name=experiment.name).one_or_none()
)
if existing_sqa_experiment is None:
raise ValueError("Must save experiment before adding a new trial.")
existing_trial_indices = {trial.index for trial in existing_sqa_experiment.trials}
if trial.index in existing_trial_indices:
raise ValueError(f"Trial {trial.index} already attached to experiment.")
new_sqa_trial = encoder.trial_to_sqa(trial)
with session_scope() as session:
existing_sqa_experiment.trials.append(new_sqa_trial)
session.add(existing_sqa_experiment)
def _update_trial(experiment: Experiment, trial: BaseTrial,
encoder: Encoder) -> None:
"""Update trial and attach data, using given Encoder instance."""
exp_sqa_class = encoder.config.class_to_sqa_class[Experiment]
with session_scope() as session:
existing_sqa_experiment = (session.query(exp_sqa_class).filter_by(
name=experiment.name).one_or_none())
if existing_sqa_experiment is None:
raise ValueError("Must save experiment before updating a trial.")
existing_trial_indices = {
trial.index
for trial in existing_sqa_experiment.trials
}
if trial.index not in existing_trial_indices:
raise ValueError(
f"Trial {trial.index} is not attached to the experiment.")
# There should only be one existing trial with the same index
existing_sqa_trials = [
sqa_trial for sqa_trial in existing_sqa_experiment.trials
if sqa_trial.index == trial.index
]
assert len(existing_sqa_trials) == 1
existing_sqa_trial = existing_sqa_trials[0]
new_sqa_trial = encoder.trial_to_sqa(trial)
existing_sqa_trial.update(new_sqa_trial)
with session_scope() as session:
session.add(existing_sqa_trial)
data, ts = experiment.lookup_data_for_trial(trial_index=trial.index)
if ts != -1:
sqa_data = encoder.data_to_sqa(data=data,
trial_index=trial.index,
timestamp=ts)
with session_scope() as session:
existing_sqa_experiment.data.append(sqa_data)
session.add(existing_sqa_experiment)
def save_new_trials(experiment: Experiment, trials: List[BaseTrial],
encoder: Encoder) -> None:
"""Add new trials to the experiment."""
trial_sqa_class = encoder.config.class_to_sqa_class[Trial]
with session_scope() as session:
experiment_id = _get_experiment_id(experiment=experiment,
encoder=encoder,
session=session)
existing_trial_indices = (
session.query(trial_sqa_class.index) # pyre-ignore
.filter_by(experiment_id=experiment_id).all())
existing_trial_indices = {x[0] for x in existing_trial_indices}
for trial in trials:
if trial.index in existing_trial_indices:
raise ValueError(
f"Trial {trial.index} already attached to experiment.")
new_sqa_trial = encoder.trial_to_sqa(trial)
new_sqa_trial.experiment_id = experiment_id
session.add(new_sqa_trial)
existing_trial_indices.add(trial.index)
def update_trials(experiment: Experiment, trials: List[BaseTrial],
encoder: Encoder) -> None:
"""Update trials and attach data."""
trial_sqa_class = encoder.config.class_to_sqa_class[Trial]
with session_scope() as session:
experiment_id = _get_experiment_id(experiment=experiment,
encoder=encoder,
session=session)
trial_indices = [trial.index for trial in trials]
existing_trials = (
session.query(trial_sqa_class).filter_by(
experiment_id=experiment_id).filter(
trial_sqa_class.index.in_(trial_indices)) # pyre-ignore
.all())
trial_index_to_existing_trial = {
trial.index: trial
for trial in existing_trials
}
for trial in trials:
existing_trial = trial_index_to_existing_trial.get(trial.index)
if existing_trial is None:
raise ValueError(
f"Trial {trial.index} is not attached to the experiment.")
new_sqa_trial = encoder.trial_to_sqa(trial)
existing_trial.update(new_sqa_trial)
session.add(existing_trial)
data, ts = experiment.lookup_data_for_trial(
trial_index=trial.index)
if ts != -1:
sqa_data = encoder.data_to_sqa(data=data,
trial_index=trial.index,
timestamp=ts)
sqa_data.experiment_id = experiment_id
session.add(sqa_data)
def _save_new_trials(experiment: Experiment, trials: List[BaseTrial],
encoder: Encoder) -> None:
"""Add new trials to the experiment."""
trial_sqa_class = encoder.config.class_to_sqa_class[Trial]
obj_to_sqa = []
with session_scope() as session:
experiment_id = _get_experiment_id(experiment=experiment,
encoder=encoder,
session=session)
existing_trial_indices = (
session.query(trial_sqa_class.index) # pyre-ignore
.filter_by(experiment_id=experiment_id).all())
existing_trial_indices = {x[0] for x in existing_trial_indices}
new_trial_idcs = set()
trials_sqa = []
for trial in trials:
if trial.index in existing_trial_indices:
raise ValueError(
f"Trial {trial.index} already attached to experiment.")
if trial.index in new_trial_idcs:
raise ValueError(
f"Trial {trial.index} appears in `trials` more than once.")
new_sqa_trial, _obj_to_sqa = encoder.trial_to_sqa(trial)
obj_to_sqa.extend(_obj_to_sqa)
new_sqa_trial.experiment_id = experiment_id
trials_sqa.append(new_sqa_trial)
new_trial_idcs.add(trial.index)
with session_scope() as session:
session.add_all(trials_sqa)
session.flush()
_set_db_ids(obj_to_sqa=obj_to_sqa)
class SQAStoreTest(TestCase):
def setUp(self):
init_test_engine_and_session_factory(force_init=True)
self.config = SQAConfig()
self.encoder = Encoder(config=self.config)
self.decoder = Decoder(config=self.config)
self.experiment = get_experiment_with_batch_trial()
self.dummy_parameters = [
get_range_parameter(), # w
get_range_parameter2(), # x
]
def testCreationOfTestDB(self):
init_test_engine_and_session_factory(tier_or_path=":memory:",
force_init=True)
engine = get_engine()
self.assertIsNotNone(engine)
def testDBConnectionWithoutForceInit(self):
init_test_engine_and_session_factory(tier_or_path=":memory:")
def testConnectionToDBWithURL(self):
init_engine_and_session_factory(url="sqlite://", force_init=True)
def testConnectionToDBWithCreator(self):
def MockDBAPI():
connection = Mock()
def connect(*args, **kwargs):
return connection
return MagicMock(connect=Mock(side_effect=connect))
mocked_dbapi = MockDBAPI()
init_engine_and_session_factory(
creator=lambda: mocked_dbapi.connect(),
force_init=True,
module=mocked_dbapi,
echo=True,
pool_size=2,
_initialize=False,
)
with session_scope() as session:
engine = session.bind
engine.connect()
self.assertEqual(mocked_dbapi.connect.call_count, 1)
self.assertTrue(engine.echo)
self.assertEqual(engine.pool.size(), 2)
def testEquals(self):
trial_sqa = self.encoder.trial_to_sqa(get_batch_trial())
self.assertTrue(trial_sqa.equals(trial_sqa))
def testListEquals(self):
self.assertTrue(SQABase.list_equals([1, 2, 3], [1, 2, 3]))
self.assertFalse(SQABase.list_equals([4], ["foo"]))
self.assertFalse(SQABase.list_equals([4], []))
with self.assertRaises(ValueError):
SQABase.list_equals([[4]], [[4]])
def testListUpdate(self):
self.assertEqual(SQABase.list_update([1, 2, 3], [1, 2, 3]), [1, 2, 3])
self.assertEqual(SQABase.list_update([4], [5]), [5])
self.assertEqual(SQABase.list_update([4], ["foo"]), ["foo"])
with self.assertRaises(ValueError):
SQABase.list_update([[4]], [[4]])
def testValidateUpdate(self):
parameter = get_choice_parameter()
parameter_sqa = self.encoder.parameter_to_sqa(parameter)
parameter2 = get_choice_parameter()
parameter2._name = 5
parameter_sqa_2 = self.encoder.parameter_to_sqa(parameter2)
with self.assertRaises(ImmutabilityError):
parameter_sqa.update(parameter_sqa_2)
def testGeneratorRunTypeValidation(self):
experiment = get_experiment_with_batch_trial()
generator_run = experiment.trials[0].generator_run_structs[
0].generator_run
generator_run._generator_run_type = "foobar"
with self.assertRaises(SQAEncodeError):
self.encoder.generator_run_to_sqa(generator_run)
generator_run._generator_run_type = "STATUS_QUO"
generator_run_sqa = self.encoder.generator_run_to_sqa(generator_run)
generator_run_sqa.generator_run_type = 2
with self.assertRaises(SQADecodeError):
self.decoder.generator_run_from_sqa(generator_run_sqa)
generator_run_sqa.generator_run_type = 0
self.decoder.generator_run_from_sqa(generator_run_sqa)
def testExperimentSaveAndLoad(self):
for exp in [
self.experiment,
get_experiment_with_multi_objective(),
get_experiment_with_scalarized_objective(),
]:
exp = self.experiment
save_experiment(exp)
loaded_experiment = load_experiment(exp.name)
self.assertEqual(loaded_experiment, exp)
def testMTExperimentSaveAndLoad(self):
experiment = get_multi_type_experiment(add_trials=True)
save_experiment(experiment)
loaded_experiment = load_experiment(experiment.name)
self.assertEqual(loaded_experiment.default_trial_type, "type1")
self.assertEqual(len(loaded_experiment._trial_type_to_runner), 2)
self.assertEqual(loaded_experiment.metric_to_trial_type["m1"], "type1")
self.assertEqual(loaded_experiment.metric_to_trial_type["m2"], "type2")
self.assertEqual(loaded_experiment._metric_to_canonical_name["m2"],
"m1")
self.assertEqual(len(loaded_experiment.trials), 2)
def testExperimentNewTrial(self):
save_experiment(self.experiment)
trial = self.experiment.new_batch_trial()
save_new_trial(experiment=self.experiment, trial=trial)
loaded_experiment = load_experiment(self.experiment.name)
self.assertEqual(len(loaded_experiment.trials), 2)
self.assertEqual(trial, loaded_experiment.trials[1])
trial = self.experiment.new_batch_trial(
generator_run=get_generator_run())
save_new_trial(experiment=self.experiment, trial=trial)
loaded_experiment = load_experiment(self.experiment.name)
self.assertEqual(len(loaded_experiment.trials), 3)
self.assertEqual(trial, loaded_experiment.trials[2])
def testExperimentNewTrialValidation(self):
trial = self.experiment.new_batch_trial()
with self.assertRaises(ValueError):
# must save experiment first
save_new_trial(experiment=self.experiment, trial=trial)
save_experiment(self.experiment)
with self.assertRaises(ValueError):
# can't save new trial twice
save_new_trial(experiment=self.experiment, trial=trial)
def testExperimentUpdateTrial(self):
save_experiment(self.experiment)
trial = self.experiment.trials[0]
trial.mark_staged()
update_trial(experiment=self.experiment, trial=trial)
loaded_experiment = load_experiment(self.experiment.name)
self.assertEqual(trial, loaded_experiment.trials[0])
trial._run_metadata = {"foo": "bar"}
update_trial(experiment=self.experiment, trial=trial)
loaded_experiment = load_experiment(self.experiment.name)
self.assertEqual(trial, loaded_experiment.trials[0])
self.experiment.attach_data(get_data(trial_index=trial.index))
update_trial(experiment=self.experiment, trial=trial)
loaded_experiment = load_experiment(self.experiment.name)
self.assertEqual(self.experiment, loaded_experiment)
trial = self.experiment.new_batch_trial(
generator_run=get_generator_run())
save_new_trial(experiment=self.experiment, trial=trial)
self.experiment.attach_data(get_data(trial_index=trial.index))
update_trial(experiment=self.experiment, trial=trial)
loaded_experiment = load_experiment(self.experiment.name)
self.assertEqual(self.experiment, loaded_experiment)
def testExperimentUpdateTrialValidation(self):
trial = self.experiment.trials[0]
with self.assertRaises(ValueError):
# must save experiment first
update_trial(experiment=self.experiment, trial=trial)
save_experiment(self.experiment)
trial._index = 1
with self.assertRaises(ValueError):
# has bad index
update_trial(experiment=self.experiment, trial=trial)
def testSaveValidation(self):
with self.assertRaises(ValueError):
save_experiment(self.experiment.trials[0])
experiment = get_experiment_with_batch_trial()
experiment.name = None
with self.assertRaises(ValueError):
save_experiment(experiment)
def testEncodeDecode(self):
for class_, fake_func, unbound_encode_func, unbound_decode_func in TEST_CASES:
# Can't load trials from SQL, because a trial needs an experiment
# in order to be initialized
if class_ == "BatchTrial" or class_ == "Trial":
continue
original_object = fake_func()
encode_func = unbound_encode_func.__get__(self.encoder)
decode_func = unbound_decode_func.__get__(self.decoder)
sqa_object = encode_func(original_object)
if (class_ == "OrderConstraint" or class_ == "ParameterConstraint"
or class_ == "SumConstraint"):
converted_object = decode_func(sqa_object,
self.dummy_parameters)
else:
converted_object = decode_func(sqa_object)
if class_ == "SimpleExperiment":
# Evaluation functions will be different, so need to do
# this so equality test passes
with self.assertRaises(Exception):
converted_object.evaluation_function()
original_object.evaluation_function = None
converted_object.evaluation_function = None
self.assertEqual(
original_object,
converted_object,
msg=f"Error encoding/decoding {class_}.",
)
def testEncoders(self):
for class_, fake_func, unbound_encode_func, _ in TEST_CASES:
original_object = fake_func()
# We can skip metrics and runners; the encoders will automatically
# handle the addition of new fields to these classes
if isinstance(original_object, Metric) or isinstance(
original_object, Runner):
continue
encode_func = unbound_encode_func.__get__(self.encoder)
sqa_object = encode_func(original_object)
if isinstance(original_object,
AbandonedArm): # handle NamedTuple differently
object_keys = original_object._asdict().keys()
else:
object_keys = original_object.__dict__.keys()
object_keys = {remove_prefix(key, "_") for key in object_keys}
sqa_keys = {
remove_prefix(key, "_")
for key in sqa_object.attributes
if key not in ["id", "_sa_instance_state"]
and not is_foreign_key_field(key)
}
# Account for fields that appear in the Python object but not the SQA
# the SQA but not the Python, and for fields that appear in both places
# but with different names
if class_ in ENCODE_DECODE_FIELD_MAPS:
map = ENCODE_DECODE_FIELD_MAPS[class_]
for field in map.python_only:
sqa_keys.add(field)
for field in map.encoded_only:
object_keys.add(field)
for python, encoded in map.python_to_encoded.items():
sqa_keys.remove(encoded)
sqa_keys.add(python)
self.assertEqual(
object_keys,
sqa_keys,
msg=
f"Mismatch between Python and SQA representation in {class_}.",
)
def testExperimentUpdates(self):
experiment = get_experiment_with_batch_trial()
save_experiment(experiment)
self.assertEqual(get_session().query(SQAExperiment).count(), 1)
# update experiment
# (should perform update in place)
experiment.description = "foobar"
save_experiment(experiment)
self.assertEqual(get_session().query(SQAExperiment).count(), 1)
experiment.status_quo = Arm(parameters={
"w": 0.0,
"x": 1,
"y": "y",
"z": True
})
save_experiment(experiment)
self.assertEqual(get_session().query(SQAExperiment).count(), 1)
loaded_experiment = load_experiment(experiment.name)
self.assertEqual(experiment, loaded_experiment)
def testExperimentParameterUpdates(self):
experiment = get_experiment_with_batch_trial()
save_experiment(experiment)
self.assertEqual(
get_session().query(SQAParameter).count(),
len(experiment.search_space.parameters),
)
# update a parameter
# (should perform update in place)
search_space = get_search_space()
parameter = get_choice_parameter()
parameter.add_values(["foobar"])
search_space.update_parameter(parameter)
experiment.search_space = search_space
save_experiment(experiment)
self.assertEqual(
get_session().query(SQAParameter).count(),
len(experiment.search_space.parameters),
)
# add a parameter
parameter = RangeParameter(name="x1",
parameter_type=ParameterType.FLOAT,
lower=-5,
upper=10)
search_space.add_parameter(parameter)
experiment.search_space = search_space
save_experiment(experiment)
self.assertEqual(
get_session().query(SQAParameter).count(),
len(experiment.search_space.parameters),
)
# remove a parameter
# (old one should be deleted)
del search_space._parameters["x1"]
experiment.search_space = search_space
save_experiment(experiment)
self.assertEqual(
get_session().query(SQAParameter).count(),
len(experiment.search_space.parameters),
)
loaded_experiment = load_experiment(experiment.name)
self.assertEqual(experiment, loaded_experiment)
def testExperimentParameterConstraintUpdates(self):
experiment = get_experiment_with_batch_trial()
save_experiment(experiment)
self.assertEqual(
get_session().query(SQAParameterConstraint).count(), # 3
len(experiment.search_space.parameter_constraints), # 3
)
# add a parameter constraint
search_space = experiment.search_space
existing_constraint = experiment.search_space.parameter_constraints[0]
new_constraint = get_sum_constraint2()
search_space.add_parameter_constraints([new_constraint])
experiment.search_space = search_space
save_experiment(experiment)
self.assertEqual(
get_session().query(SQAParameterConstraint).count(),
len(experiment.search_space.parameter_constraints),
)
# update a parameter constraint
# (since we don't have UIDs for these, we throw out the old one
# and create a new one)
new_constraint.bound = 5.0
search_space.set_parameter_constraints(
[existing_constraint, new_constraint])
experiment.search_space = search_space
save_experiment(experiment)
self.assertEqual(
get_session().query(SQAParameterConstraint).count(),
len(experiment.search_space.parameter_constraints),
)
# remove a parameter constraint
# (old one should be deleted)
search_space.set_parameter_constraints([new_constraint])
experiment.search_space = search_space
save_experiment(experiment)
self.assertEqual(
get_session().query(SQAParameterConstraint).count(),
len(experiment.search_space.parameter_constraints),
)
loaded_experiment = load_experiment(experiment.name)
self.assertEqual(experiment, loaded_experiment)
def testExperimentObjectiveUpdates(self):
experiment = get_experiment_with_batch_trial()
save_experiment(experiment)
self.assertEqual(get_session().query(SQAMetric).count(),
len(experiment.metrics))
# update objective
# (should perform update in place)
optimization_config = get_optimization_config()
objective = get_objective()
objective.minimize = True
optimization_config.objective = objective
experiment.optimization_config = optimization_config
save_experiment(experiment)
self.assertEqual(get_session().query(SQAMetric).count(),
len(experiment.metrics))
# replace objective
# (old one should become tracking metric)
optimization_config.objective = Objective(metric=Metric(
name="objective"))
experiment.optimization_config = optimization_config
save_experiment(experiment)
self.assertEqual(get_session().query(SQAMetric).count(),
len(experiment.metrics))
loaded_experiment = load_experiment(experiment.name)
self.assertEqual(experiment, loaded_experiment)
def testExperimentOutcomeConstraintUpdates(self):
experiment = get_experiment_with_batch_trial()
save_experiment(experiment)
self.assertEqual(get_session().query(SQAMetric).count(),
len(experiment.metrics))
# update outcome constraint
# (should perform update in place)
optimization_config = get_optimization_config()
outcome_constraint = get_outcome_constraint()
outcome_constraint.bound = -1.0
optimization_config.outcome_constraints = [outcome_constraint]
experiment.optimization_config = optimization_config
save_experiment(experiment)
self.assertEqual(get_session().query(SQAMetric).count(),
len(experiment.metrics))
# add outcome constraint
outcome_constraint2 = OutcomeConstraint(metric=Metric(name="outcome"),
op=ComparisonOp.GEQ,
bound=-0.5)
optimization_config.outcome_constraints = [
outcome_constraint,
outcome_constraint2,
]
experiment.optimization_config = optimization_config
save_experiment(experiment)
self.assertEqual(get_session().query(SQAMetric).count(),
len(experiment.metrics))
# remove outcome constraint
# (old one should become tracking metric)
optimization_config.outcome_constraints = [outcome_constraint]
experiment.optimization_config = optimization_config
save_experiment(experiment)
self.assertEqual(get_session().query(SQAMetric).count(),
len(experiment.metrics))
loaded_experiment = load_experiment(experiment.name)
self.assertEqual(experiment, loaded_experiment)
def testFailedLoad(self):
with self.assertRaises(ValueError):
load_experiment("nonexistent_experiment")
def testExperimentTrackingMetricUpdates(self):
experiment = get_experiment_with_batch_trial()
save_experiment(experiment)
self.assertEqual(get_session().query(SQAMetric).count(),
len(experiment.metrics))
# update tracking metric
# (should perform update in place)
metric = Metric(name="tracking", lower_is_better=True)
experiment.update_tracking_metric(metric)
save_experiment(experiment)
self.assertEqual(get_session().query(SQAMetric).count(),
len(experiment.metrics))
# add tracking metric
metric = Metric(name="tracking2")
experiment.add_tracking_metric(metric)
save_experiment(experiment)
self.assertEqual(get_session().query(SQAMetric).count(),
len(experiment.metrics))
# remove tracking metric
# (old one should be deleted)
experiment.remove_tracking_metric("tracking2")
save_experiment(experiment)
self.assertEqual(get_session().query(SQAMetric).count(),
len(experiment.metrics))
loaded_experiment = load_experiment(experiment.name)
self.assertEqual(experiment, loaded_experiment)
def testExperimentRunnerUpdates(self):
experiment = get_experiment_with_batch_trial()
save_experiment(experiment)
# one runner on the batch
self.assertEqual(get_session().query(SQARunner).count(), 1)
# add runner to experiment
runner = get_synthetic_runner()
experiment.runner = runner
save_experiment(experiment)
self.assertEqual(get_session().query(SQARunner).count(), 2)
# update runner
# (should perform update in place)
runner = get_synthetic_runner()
runner.dummy_metadata = {"foo": "bar"}
experiment.runner = runner
save_experiment(experiment)
self.assertEqual(get_session().query(SQARunner).count(), 2)
# remove runner
# (old one should be deleted)
experiment.runner = None
save_experiment(experiment)
self.assertEqual(get_session().query(SQARunner).count(), 1)
loaded_experiment = load_experiment(experiment.name)
self.assertEqual(experiment, loaded_experiment)
def testExperimentTrialUpdates(self):
experiment = get_experiment_with_batch_trial()
save_experiment(experiment)
self.assertEqual(get_session().query(SQATrial).count(), 1)
self.assertEqual(get_session().query(SQARunner).count(), 1)
# add trial
trial = experiment.new_batch_trial()
runner = get_synthetic_runner()
trial.runner = runner
save_experiment(experiment)
self.assertEqual(get_session().query(SQATrial).count(), 2)
self.assertEqual(get_session().query(SQARunner).count(), 2)
# update trial's runner
runner.dummy_metadata = "dummy metadata"
trial.runner = runner
save_experiment(experiment)
self.assertEqual(get_session().query(SQATrial).count(), 2)
self.assertEqual(get_session().query(SQARunner).count(), 2)
trial.run()
save_experiment(experiment)
self.assertEqual(get_session().query(SQATrial).count(), 2)
loaded_experiment = load_experiment(experiment.name)
self.assertEqual(experiment, loaded_experiment)
def testExperimentAbandonedArmUpdates(self):
experiment = get_experiment_with_batch_trial()
save_experiment(experiment)
# one arm is already abandoned
self.assertEqual(get_session().query(SQAAbandonedArm).count(), 1)
trial = experiment.trials[0]
trial.mark_arm_abandoned(trial.arms[1].name)
save_experiment(experiment)
self.assertEqual(get_session().query(SQAAbandonedArm).count(), 2)
loaded_experiment = load_experiment(experiment.name)
self.assertEqual(experiment, loaded_experiment)
def testExperimentGeneratorRunUpdates(self):
experiment = get_experiment_with_batch_trial()
save_experiment(experiment)
# one main generator run, one for the status quo
self.assertEqual(get_session().query(SQAGeneratorRun).count(), 2)
# add a arm
# this will create one wrapper generator run
# this will also replace the status quo generator run,
# since the weight of the status quo will have changed
trial = experiment.trials[0]
trial.add_arm(get_arm())
save_experiment(experiment)
self.assertEqual(get_session().query(SQAGeneratorRun).count(), 3)
generator_run = get_generator_run()
trial.add_generator_run(generator_run=generator_run, multiplier=0.5)
save_experiment(experiment)
self.assertEqual(get_session().query(SQAGeneratorRun).count(), 4)
loaded_experiment = load_experiment(experiment.name)
self.assertEqual(experiment, loaded_experiment)
def testParameterValidation(self):
sqa_parameter = SQAParameter(
domain_type=DomainType.RANGE,
parameter_type=ParameterType.FLOAT,
name="foobar",
)
with self.assertRaises(ValueError):
with session_scope() as session:
session.add(sqa_parameter)
sqa_parameter.experiment_id = 0
with session_scope() as session:
session.add(sqa_parameter)
with self.assertRaises(ValueError):
sqa_parameter.generator_run_id = 0
with session_scope() as session:
session.add(sqa_parameter)
sqa_parameter = SQAParameter(
domain_type=DomainType.RANGE,
parameter_type=ParameterType.FLOAT,
name="foobar",
generator_run_id=0,
)
with session_scope() as session:
session.add(sqa_parameter)
with self.assertRaises(ValueError):
sqa_parameter.experiment_id = 0
with session_scope() as session:
session.add(sqa_parameter)
def testParameterDecodeFailure(self):
parameter = get_fixed_parameter()
sqa_parameter = self.encoder.parameter_to_sqa(parameter)
sqa_parameter.domain_type = 5
with self.assertRaises(SQADecodeError):
self.decoder.parameter_from_sqa(sqa_parameter)
def testParameterUpdateFailure(self):
parameter = get_range_parameter()
sqa_parameter = self.encoder.parameter_to_sqa(parameter)
parameter._name = "new"
sqa_parameter_2 = self.encoder.parameter_to_sqa(parameter)
with self.assertRaises(ImmutabilityError):
sqa_parameter.update(sqa_parameter_2)
def testParameterConstraintValidation(self):
sqa_parameter_constraint = SQAParameterConstraint(
bound=0, constraint_dict={}, type=ParameterConstraintType.LINEAR)
with self.assertRaises(ValueError):
with session_scope() as session:
session.add(sqa_parameter_constraint)
sqa_parameter_constraint.experiment_id = 0
with session_scope() as session:
session.add(sqa_parameter_constraint)
with self.assertRaises(ValueError):
sqa_parameter_constraint.generator_run_id = 0
with session_scope() as session:
session.add(sqa_parameter_constraint)
sqa_parameter_constraint = SQAParameterConstraint(
bound=0,
constraint_dict={},
type=ParameterConstraintType.LINEAR,
generator_run_id=0,
)
with session_scope() as session:
session.add(sqa_parameter_constraint)
with self.assertRaises(ValueError):
sqa_parameter_constraint.experiment_id = 0
with session_scope() as session:
session.add(sqa_parameter_constraint)
def testDecodeOrderParameterConstraintFailure(self):
sqa_parameter = SQAParameterConstraint(
type=ParameterConstraintType.ORDER, constraint_dict={}, bound=0)
with self.assertRaises(SQADecodeError):
self.decoder.parameter_constraint_from_sqa(sqa_parameter,
self.dummy_parameters)
def testDecodeSumParameterConstraintFailure(self):
sqa_parameter = SQAParameterConstraint(
type=ParameterConstraintType.SUM, constraint_dict={}, bound=0)
with self.assertRaises(SQADecodeError):
self.decoder.parameter_constraint_from_sqa(sqa_parameter,
self.dummy_parameters)
def testMetricValidation(self):
sqa_metric = SQAMetric(
name="foobar",
intent=MetricIntent.OBJECTIVE,
metric_type=METRIC_REGISTRY[BraninMetric],
)
with self.assertRaises(ValueError):
with session_scope() as session:
session.add(sqa_metric)
sqa_metric.experiment_id = 0
with session_scope() as session:
session.add(sqa_metric)
with self.assertRaises(ValueError):
sqa_metric.generator_run_id = 0
with session_scope() as session:
session.add(sqa_metric)
sqa_metric = SQAMetric(
name="foobar",
intent=MetricIntent.OBJECTIVE,
metric_type=METRIC_REGISTRY[BraninMetric],
generator_run_id=0,
)
with session_scope() as session:
session.add(sqa_metric)
with self.assertRaises(ValueError):
sqa_metric.experiment_id = 0
with session_scope() as session:
session.add(sqa_metric)
def testMetricEncodeFailure(self):
metric = get_branin_metric()
del metric.__dict__["param_names"]
with self.assertRaises(AttributeError):
self.encoder.metric_to_sqa(metric)
def testMetricDecodeFailure(self):
metric = get_branin_metric()
sqa_metric = self.encoder.metric_to_sqa(metric)
sqa_metric.metric_type = "foobar"
with self.assertRaises(SQADecodeError):
self.decoder.metric_from_sqa(sqa_metric)
sqa_metric.metric_type = METRIC_REGISTRY[BraninMetric]
sqa_metric.intent = "foobar"
with self.assertRaises(SQADecodeError):
self.decoder.metric_from_sqa(sqa_metric)
sqa_metric.intent = MetricIntent.TRACKING
sqa_metric.properties = {}
with self.assertRaises(ValueError):
self.decoder.metric_from_sqa(sqa_metric)
def testRunnerDecodeFailure(self):
runner = get_synthetic_runner()
sqa_runner = self.encoder.runner_to_sqa(runner)
sqa_runner.runner_type = "foobar"
with self.assertRaises(SQADecodeError):
self.decoder.runner_from_sqa(sqa_runner)
def testRunnerValidation(self):
sqa_runner = SQARunner(runner_type=RUNNER_REGISTRY[SyntheticRunner])
with self.assertRaises(ValueError):
with session_scope() as session:
session.add(sqa_runner)
sqa_runner.experiment_id = 0
with session_scope() as session:
session.add(sqa_runner)
with self.assertRaises(ValueError):
sqa_runner.trial_id = 0
with session_scope() as session:
session.add(sqa_runner)
sqa_runner = SQARunner(runner_type=RUNNER_REGISTRY[SyntheticRunner],
trial_id=0)
with session_scope() as session:
session.add(sqa_runner)
with self.assertRaises(ValueError):
sqa_runner.experiment_id = 0
with session_scope() as session:
session.add(sqa_runner)
def testTimestampUpdate(self):
self.experiment.trials[0]._time_staged = datetime.now()
save_experiment(self.experiment)
# second save should not fail
save_experiment(self.experiment)
def testGetProperties(self):
# Extract default value.
properties = serialize_init_args(Metric(name="foo"))
self.assertEqual(properties, {"name": "foo", "lower_is_better": None})
# Extract passed value.
properties = serialize_init_args(
Metric(name="foo", lower_is_better=True))
self.assertEqual(properties, {"name": "foo", "lower_is_better": True})
def testRegistryAdditions(self):
class MyRunner(Runner):
def run():
pass
def staging_required():
return False
class MyMetric(Metric):
pass
register_metric(MyMetric)
register_runner(MyRunner)
experiment = get_experiment_with_batch_trial()
experiment.runner = MyRunner()
experiment.add_tracking_metric(MyMetric(name="my_metric"))
save_experiment(experiment)
loaded_experiment = load_experiment(experiment.name)
self.assertEqual(loaded_experiment, experiment)
def testEncodeDecodeGenerationStrategy(self):
# Cannot load generation strategy before it has been saved
with self.assertRaises(ValueError):
load_generation_strategy_by_id(gs_id=0)
# Check that we can encode and decode the generation strategy *before*
# it has generated some trials and been updated with some data.
generation_strategy = get_generation_strategy()
# Check that we can save a generation strategy without an experiment
# attached.
save_generation_strategy(generation_strategy=generation_strategy)
# Also try restoring this generation strategy by its ID in the DB.
new_generation_strategy = load_generation_strategy_by_id(
gs_id=generation_strategy._db_id)
self.assertEqual(generation_strategy, new_generation_strategy)
self.assertIsNone(generation_strategy._experiment)
# Cannot load generation strategy before it has been saved
experiment = get_branin_experiment()
save_experiment(experiment)
with self.assertRaises(ValueError):
load_generation_strategy_by_experiment_name(
experiment_name=experiment.name)
# Check that we can encode and decode the generation strategy *after*
# it has generated some trials and been updated with some data.
generation_strategy = new_generation_strategy
experiment.new_trial(generation_strategy.gen(experiment=experiment))
experiment.new_trial(
generation_strategy.gen(experiment, data=get_branin_data()))
save_generation_strategy(generation_strategy=generation_strategy)
save_experiment(experiment)
# Try restoring the generation strategy using the experiment its
# attached to.
new_generation_strategy = load_generation_strategy_by_experiment_name(
experiment_name=experiment.name)
# `_seen_trial_indices_by_status` attribute of a GS is not saved in DB,
# so it will be None in the restored version of the GS.
# Hackily removing it from the original GS to check equality.
generation_strategy._seen_trial_indices_by_status = None
self.assertEqual(generation_strategy, new_generation_strategy)
self.assertIsInstance(new_generation_strategy._steps[0].model, Models)
self.assertIsInstance(new_generation_strategy.model, ModelBridge)
self.assertEqual(len(new_generation_strategy._generator_runs), 2)
self.assertEqual(new_generation_strategy._experiment._name,
experiment._name)
def testUpdateGenerationStrategy(self):
generation_strategy = get_generation_strategy()
save_generation_strategy(generation_strategy=generation_strategy)
experiment = get_branin_experiment()
generation_strategy = get_generation_strategy()
save_experiment(experiment)
# add generator run, save, reload
experiment.new_trial(generator_run=generation_strategy.gen(experiment))
save_generation_strategy(generation_strategy=generation_strategy)
loaded_generation_strategy = load_generation_strategy_by_experiment_name(
experiment_name=experiment.name)
# `_seen_trial_indices_by_status` attribute of a GS is not saved in DB,
# so it will be None in the restored version of the GS.
# Hackily removing it from the original GS to check equality.
generation_strategy._seen_trial_indices_by_status = None
self.assertEqual(generation_strategy, loaded_generation_strategy)
# add another generator run, save, reload
experiment.new_trial(generator_run=generation_strategy.gen(
experiment, data=get_branin_data()))
save_generation_strategy(generation_strategy=generation_strategy)
save_experiment(experiment)
loaded_generation_strategy = load_generation_strategy_by_experiment_name(
experiment_name=experiment.name)
# `_seen_trial_indices_by_status` attribute of a GS is not saved in DB,
# so it will be None in the restored version of the GS.
# Hackily removing it from the original GS to check equality.
generation_strategy._seen_trial_indices_by_status = None
self.assertEqual(generation_strategy, loaded_generation_strategy)
# make sure that we can update the experiment too
experiment.description = "foobar"
save_experiment(experiment)
loaded_generation_strategy = load_generation_strategy_by_experiment_name(
experiment_name=experiment.name)
self.assertEqual(generation_strategy, loaded_generation_strategy)
self.assertEqual(generation_strategy._experiment.description,
experiment.description)
self.assertEqual(
generation_strategy._experiment.description,
loaded_generation_strategy._experiment.description,
)
def testGeneratorRunGenMetadata(self):
gen_metadata = {"hello": "world"}
gr = GeneratorRun(arms=[], gen_metadata=gen_metadata)
generator_run_sqa = self.encoder.generator_run_to_sqa(gr)
decoded_gr = self.decoder.generator_run_from_sqa(generator_run_sqa)
self.assertEqual(decoded_gr.gen_metadata, gen_metadata)
def testUpdateGenerationStrategyIncrementally(self):
experiment = get_branin_experiment()
generation_strategy = choose_generation_strategy(
experiment.search_space)
save_experiment(experiment=experiment)
save_generation_strategy(generation_strategy=generation_strategy)
# add generator runs, save, reload
generator_runs = []
for i in range(7):
data = get_branin_data() if i > 0 else None
gr = generation_strategy.gen(experiment, data=data)
generator_runs.append(gr)
trial = experiment.new_trial(generator_run=gr).mark_running(
no_runner_required=True)
trial.mark_completed()
save_experiment(experiment=experiment)
update_generation_strategy(generation_strategy=generation_strategy,
generator_runs=generator_runs)
loaded_generation_strategy = load_generation_strategy_by_experiment_name(
experiment_name=experiment.name)
self.assertEqual(generation_strategy._curr.index,
loaded_generation_strategy._curr.index, 1)
self.assertEqual(len(loaded_generation_strategy._generator_runs), 7)
def _save_or_update_trials(experiment: Experiment, trials: List[BaseTrial],
encoder: Encoder) -> None:
"""Add new trials to the experiment, or update if they already exist."""
experiment_id = experiment._db_id
if experiment_id is None:
raise ValueError("Must save experiment first.")
data_sqa_class = encoder.config.class_to_sqa_class[Data]
trial_sqa_class = encoder.config.class_to_sqa_class[Trial]
obj_to_sqa = []
with session_scope() as session:
# Fetch the ids of all trials already saved to the experiment
existing_trial_ids = (
session.query(trial_sqa_class.id) # pyre-ignore
.filter_by(experiment_id=experiment_id).all())
existing_trial_ids = {x[0] for x in existing_trial_ids}
update_trial_ids = set()
update_trial_indices = set()
for trial in trials:
if trial._db_id not in existing_trial_ids:
continue
update_trial_ids.add(trial._db_id)
update_trial_indices.add(trial.index)
# We specifically fetch the *whole* trial (and corresponding data)
# for old trials that we need to update.
# We could fetch the whole trial for all trials attached to the experiment,
# and therefore combine this call with the one above, but that might be
# unnecessarily costly if we're not updating many or any trials.
with session_scope() as session:
existing_trials = (session.query(trial_sqa_class).filter(
trial_sqa_class.id.in_(update_trial_ids)).all())
with session_scope() as session:
existing_data = (
session.query(data_sqa_class).filter_by(
experiment_id=experiment_id).filter(
data_sqa_class.trial_index.in_(
update_trial_indices)) # pyre-ignore
.all())
trial_id_to_existing_trial = {trial.id: trial for trial in existing_trials}
data_id_to_existing_data = {data.id: data for data in existing_data}
sqa_trials, sqa_datas = [], []
for trial in trials:
sqa_trial, _obj_to_sqa = encoder.trial_to_sqa(trial)
obj_to_sqa.extend(_obj_to_sqa)
existing_trial = trial_id_to_existing_trial.get(trial._db_id)
if existing_trial is None:
sqa_trial.experiment_id = experiment_id
sqa_trials.append(sqa_trial)
else:
existing_trial.update(sqa_trial)
sqa_trials.append(existing_trial)
datas = experiment.data_by_trial.get(trial.index, {})
for ts, data in datas.items():
sqa_data = encoder.data_to_sqa(data=data,
trial_index=trial.index,
timestamp=ts)
obj_to_sqa.append((data, sqa_data))
existing_data = data_id_to_existing_data.get(data._db_id)
if existing_data is None:
sqa_data.experiment_id = experiment_id
sqa_datas.append(sqa_data)
else:
existing_data.update(sqa_data)
sqa_datas.append(existing_data)
with session_scope() as session:
session.add_all(sqa_trials)
session.add_all(sqa_datas)
session.flush()
_set_db_ids(obj_to_sqa=obj_to_sqa)
