def test_save_generation_strategy(self):
experiment, generation_strategy = self.init_experiment_and_generation_strategy(
save_generation_strategy=False)
saved = self.with_db_settings._save_generation_strategy_to_db_if_possible(
generation_strategy)
self.assertTrue(saved)
loaded_gs = _load_generation_strategy_by_experiment_name(
experiment_name=experiment.name,
decoder=self.with_db_settings.db_settings.decoder,
)
self.assertIsNotNone(loaded_gs)
self.assertEqual(loaded_gs.name, generation_strategy.name)
def load_experiment_and_generation_strategy(
experiment_name: str, db_settings: DBSettings
) -> Tuple[Experiment, GenerationStrategy]:
"""Load experiment and the corresponding generation strategy from the DB.
Args:
name: Experiment name.
db_settings: Defines behavior for loading/saving experiment to/from db.
Returns:
A tuple of the loaded experiment and generation strategy.
"""
experiment = load_experiment(name=experiment_name, db_settings=db_settings)
generation_strategy = _load_generation_strategy_by_experiment_name(
experiment_name=experiment_name, decoder=db_settings.decoder
)
return experiment, generation_strategy
def _load_experiment_and_generation_strategy(
self, experiment_name: str
) -> Tuple[Optional[Experiment], Optional[GenerationStrategy]]:
"""Loads experiment and its corresponding generation strategy from database
if DB settings are set on this `WithDBSettingsBase` instance.
Args:
experiment_name: Name of the experiment to load, used as unique
identifier by which to find the experiment.
Returns:
- Tuple of `None` and `None` if `DBSettings` are set and no experiment
exists by the given name.
- Tuple of `Experiment` and `None` if experiment exists but does not
have a generation strategy attached to it.
- Tuple of `Experiment` and `GenerationStrategy` if experiment exists
and has a generation strategy attached to it.
"""
if not self.db_settings_set:
raise ValueError("Cannot load from DB in absence of DB settings.")
start_time = time.time()
experiment = _load_experiment(experiment_name,
decoder=self.db_settings.decoder)
if not isinstance(experiment,
Experiment) or experiment.is_simple_experiment:
raise ValueError("Service API only supports `Experiment`.")
logger.debug(
f"Loaded experiment {experiment_name} in "
f"{_round_floats_for_logging(time.time() - start_time)} seconds.")
try:
start_time = time.time()
generation_strategy = _load_generation_strategy_by_experiment_name(
experiment_name=experiment_name,
decoder=self.db_settings.decoder)
logger.debug(
f"Loaded generation strategy for experiment {experiment_name} in "
f"{_round_floats_for_logging(time.time() - start_time)} seconds."
)
except ValueError as err:
if "does not have a generation strategy" in str(err):
return experiment, None
raise # `ValueError` here could signify more than just absence of GS.
return experiment, generation_strategy
def load_experiment_and_generation_strategy(
experiment_name: str, db_settings: DBSettings
) -> Tuple[Experiment, Optional[GenerationStrategy]]:
"""Load experiment and the corresponding generation strategy from the DB.
Args:
name: Experiment name.
db_settings: Defines behavior for loading/saving experiment to/from db.
Returns:
A tuple of the loaded experiment and generation strategy.
"""
experiment = load_experiment(name=experiment_name, db_settings=db_settings)
try:
generation_strategy = _load_generation_strategy_by_experiment_name(
experiment_name=experiment_name, decoder=db_settings.decoder)
except ValueError: # Generation strategy has not yet been attached to this
return experiment, None # experiment.
return experiment, generation_strategy
def test_update_reduced_state_generator_runs(self):
experiment, generation_strategy = self.init_experiment_and_generation_strategy(
save_generation_strategy=True
)
trials = [experiment.new_trial() for _ in range(5)]
grs = []
for t in trials:
gr = generation_strategy.gen(experiment)
grs.append(gr)
t.add_generator_run(gr)
self.with_db_settings._save_or_update_trials_and_generation_strategy_if_possible( # noqa E501
experiment=experiment,
trials=trials,
generation_strategy=generation_strategy,
new_generator_runs=grs,
reduce_state_generator_runs=True,
)
loaded_experiment = _load_experiment(
experiment.name, decoder=self.with_db_settings.db_settings.decoder
)
# Only the last trial's generator run should have large model attributes
for idx, trial in loaded_experiment.trials.items():
for key in [f"_{attr.key}" for attr in GR_LARGE_MODEL_ATTRS]:
if idx < len(loaded_experiment.trials) - 1:
self.assertIsNone(getattr(trial.generator_run, key))
else:
self.assertIsNotNone(getattr(trial.generator_run, key))
loaded_generation_strategy = _load_generation_strategy_by_experiment_name(
experiment.name, decoder=self.with_db_settings.db_settings.decoder
)
# Only the last generator run should have large model attributes
for idx, gr in enumerate(loaded_generation_strategy._generator_runs):
for key in [f"_{attr.key}" for attr in GR_LARGE_MODEL_ATTRS]:
if idx < len(loaded_generation_strategy._generator_runs) - 1:
self.assertIsNone(getattr(gr, key))
else:
self.assertIsNotNone(getattr(gr, key))
def load_experiment_and_generation_strategy(
experiment_name: str, db_settings: DBSettings
) -> Tuple[Experiment, Optional[GenerationStrategy]]:
"""Load experiment and the corresponding generation strategy from the DB.
Args:
name: Experiment name.
db_settings: Defines behavior for loading/saving experiment to/from db.
Returns:
A tuple of the loaded experiment and generation strategy.
"""
experiment = load_experiment(name=experiment_name, db_settings=db_settings)
try:
generation_strategy = _load_generation_strategy_by_experiment_name(
experiment_name=experiment_name, decoder=db_settings.decoder)
except ValueError as err:
if "does not have a generation strategy" in str(err):
return experiment, None
raise # `ValueError` here could signify more than just absence of GS.
return experiment, generation_strategy
def _load_experiment_and_generation_strategy(
self,
experiment_name: str,
reduced_state: bool = False,
) -> Tuple[Optional[Experiment], Optional[GenerationStrategy]]:
"""Loads experiment and its corresponding generation strategy from database
if DB settings are set on this `WithDBSettingsBase` instance.
Args:
experiment_name: Name of the experiment to load, used as unique
identifier by which to find the experiment.
reduced_state: Whether to load experiment and generation strategy
with a slightly reduced state (without abandoned arms on experiment
and model state on each generator run in experiment and generation
strategy; last generator run on generation strategy will still
have its model state).
Returns:
- Tuple of `None` and `None` if `DBSettings` are set and no experiment
exists by the given name.
- Tuple of `Experiment` and `None` if experiment exists but does not
have a generation strategy attached to it.
- Tuple of `Experiment` and `GenerationStrategy` if experiment exists
and has a generation strategy attached to it.
"""
if not self.db_settings_set:
raise ValueError("Cannot load from DB in absence of DB settings.")
logger.info(
"Loading experiment and generation strategy (with reduced state: "
f"{reduced_state})...")
start_time = time.time()
experiment = _load_experiment(
experiment_name,
decoder=self.db_settings.decoder,
reduced_state=reduced_state,
load_trials_in_batches_of_size=LOADING_MINI_BATCH_SIZE,
)
if not isinstance(experiment, Experiment):
raise ValueError("Service API only supports `Experiment`.")
logger.info(
f"Loaded experiment {experiment_name} in "
f"{_round_floats_for_logging(time.time() - start_time)} seconds, "
f"loading trials in mini-batches of {LOADING_MINI_BATCH_SIZE}.")
try:
start_time = time.time()
generation_strategy = _load_generation_strategy_by_experiment_name(
experiment_name=experiment_name,
decoder=self.db_settings.decoder,
experiment=experiment,
reduced_state=reduced_state,
)
logger.info(
f"Loaded generation strategy for experiment {experiment_name} in "
f"{_round_floats_for_logging(time.time() - start_time)} seconds."
)
except ValueError as err:
if "does not have a generation strategy" in str(err):
return experiment, None
raise # `ValueError` here could signify more than just absence of GS.
return experiment, generation_strategy
