def load_generation_strategy_by_id(gs_id: int,
                                   config: Optional[SQAConfig] = None
                                   ) -> GenerationStrategy:
    """Finds a generation strategy stored by a given ID and restores it."""
    config = config or SQAConfig()
    decoder = Decoder(config=config)
    return _load_generation_strategy_by_id(gs_id=gs_id, decoder=decoder)
Ejemplo n.º 2
0
def load_experiment(
    experiment_name: str, config: Optional[SQAConfig] = None
) -> Experiment:
    """Load experiment by name (uses default SQAConfig)."""
    config = config or SQAConfig()
    decoder = Decoder(config=config)
    return _load_experiment(experiment_name=experiment_name, decoder=decoder)
Ejemplo n.º 3
0
 def test_storage_error_handling(self, mock_save_fails):
     """Check that if `suppress_storage_errors` is True, AxClient won't
     visibly fail if encountered storage errors.
     """
     init_test_engine_and_session_factory(force_init=True)
     config = SQAConfig()
     encoder = Encoder(config=config)
     decoder = Decoder(config=config)
     db_settings = DBSettings(encoder=encoder, decoder=decoder)
     ax_client = AxClient(db_settings=db_settings,
                          suppress_storage_errors=True)
     ax_client.create_experiment(
         name="test_experiment",
         parameters=[
             {
                 "name": "x",
                 "type": "range",
                 "bounds": [-5.0, 10.0]
             },
             {
                 "name": "y",
                 "type": "range",
                 "bounds": [0.0, 15.0]
             },
         ],
         minimize=True,
     )
     for _ in range(3):
         parameters, trial_index = ax_client.get_next_trial()
         ax_client.complete_trial(trial_index=trial_index,
                                  raw_data=branin(*parameters.values()))
Ejemplo n.º 4
0
def save_new_trial(
    experiment: Experiment, trial: BaseTrial, config: Optional[SQAConfig] = None
) -> None:
    """Add new trial to the experiment (using default SQAConfig)."""
    config = config or SQAConfig()
    encoder = Encoder(config=config)
    _save_new_trial(experiment=experiment, trial=trial, encoder=encoder)
Ejemplo n.º 5
0
Archivo: save.py Proyecto: linusec/Ax
def update_trial(experiment: Experiment,
                 trial: BaseTrial,
                 config: Optional[SQAConfig] = None) -> None:
    """Update trial and attach data (using default SQAConfig)."""
    config = config or SQAConfig()
    encoder = Encoder(config=config)
    _update_trial(experiment=experiment, trial=trial, encoder=encoder)
Ejemplo n.º 6
0
def save_or_update_trials(
    experiment: Experiment,
    trials: List[BaseTrial],
    config: Optional[SQAConfig] = None,
    batch_size: Optional[int] = None,
    reduce_state_generator_runs: bool = False,
) -> None:
    """Add new trials to the experiment, or update if already exists
    (using default SQAConfig).

    Note that new data objects (whether attached to existing or new trials)
    will also be added to the experiment, but existing data objects in the
    database will *not* be updated or removed.
    """
    config = config or SQAConfig()
    encoder = Encoder(config=config)
    decoder = Decoder(config=config)
    _save_or_update_trials(
        experiment=experiment,
        trials=trials,
        encoder=encoder,
        decoder=decoder,
        batch_size=batch_size,
        reduce_state_generator_runs=reduce_state_generator_runs,
    )
Ejemplo n.º 7
0
def save_or_update_trial(experiment: Experiment,
                         trial: BaseTrial,
                         config: Optional[SQAConfig] = None) -> None:
    """Add new trial to the experiment, or update if already exists
    (using default SQAConfig)."""
    config = config or SQAConfig()
    encoder = Encoder(config=config)
    _save_or_update_trial(experiment=experiment, trial=trial, encoder=encoder)
Ejemplo n.º 8
0
 def setUp(self):
     init_test_engine_and_session_factory(force_init=True)
     self.config = SQAConfig()
     self.encoder = Encoder(config=self.config)
     self.decoder = Decoder(config=self.config)
     self.experiment = get_experiment_with_batch_trial()
     self.dummy_parameters = [
         get_range_parameter(),  # w
         get_range_parameter2(),  # x
     ]
Ejemplo n.º 9
0
Archivo: save.py Proyecto: zorrock/Ax
def save_experiment(experiment: Experiment, config: Optional[SQAConfig] = None) -> None:
    """Save experiment (using default SQAConfig)."""
    if not isinstance(experiment, Experiment):
        raise ValueError("Can only save instances of Experiment")
    if not experiment.has_name:
        raise ValueError("Experiment name must be set prior to saving.")

    config = config or SQAConfig()
    encoder = Encoder(config=config)
    return _save_experiment(experiment=experiment, encoder=encoder)
def load_generation_strategy_by_experiment_name(
        experiment_name: str,
        config: Optional[SQAConfig] = None) -> GenerationStrategy:
    """Finds a generation strategy attached to an experiment specified by a name
    and restores it from its corresponding SQA object.
    """
    config = config or SQAConfig()
    decoder = Decoder(config=config)
    return _load_generation_strategy_by_experiment_name(
        experiment_name=experiment_name, decoder=decoder)
Ejemplo n.º 11
0
Archivo: load.py Proyecto: viotemp1/Ax
def load_generation_strategy_by_id(
    gs_id: int,
    config: Optional[SQAConfig] = None,
    experiment: Optional[Experiment] = None,
    reduced_state: bool = False,
) -> GenerationStrategy:
    """Finds a generation strategy stored by a given ID and restores it."""
    config = config or SQAConfig()
    decoder = Decoder(config=config)
    return _load_generation_strategy_by_id(
        gs_id=gs_id, decoder=decoder, experiment=experiment, reduced_state=reduced_state
    )
Ejemplo n.º 12
0
 def test_sqa_storage(self):
     init_test_engine_and_session_factory(force_init=True)
     config = SQAConfig()
     encoder = Encoder(config=config)
     decoder = Decoder(config=config)
     db_settings = DBSettings(encoder=encoder, decoder=decoder)
     ax_client = AxClient(db_settings=db_settings)
     ax_client.create_experiment(
         name="test_experiment",
         parameters=[
             {"name": "x", "type": "range", "bounds": [-5.0, 10.0]},
             {"name": "y", "type": "range", "bounds": [0.0, 15.0]},
         ],
         minimize=True,
     )
     for _ in range(5):
         parameters, trial_index = ax_client.get_next_trial()
         ax_client.complete_trial(
             trial_index=trial_index, raw_data=branin(*parameters.values())
         )
     gs = ax_client.generation_strategy
     ax_client = AxClient(db_settings=db_settings)
     ax_client.load_experiment_from_database("test_experiment")
     # Trial #4 was completed after the last time the generation strategy
     # generated candidates, so pre-save generation strategy was not
     # "aware" of completion of trial #4. Post-restoration generation
     # strategy is aware of it, however, since it gets restored with most
     # up-to-date experiment data. Do adding trial #4 to the seen completed
     # trials of pre-storage GS to check their equality otherwise.
     gs._seen_trial_indices_by_status[TrialStatus.COMPLETED].add(4)
     self.assertEqual(gs, ax_client.generation_strategy)
     with self.assertRaises(ValueError):
         # Overwriting existing experiment.
         ax_client.create_experiment(
             name="test_experiment",
             parameters=[
                 {"name": "x", "type": "range", "bounds": [-5.0, 10.0]},
                 {"name": "y", "type": "range", "bounds": [0.0, 15.0]},
             ],
             minimize=True,
         )
     with self.assertRaises(ValueError):
         # Overwriting existing experiment with overwrite flag with present
         # DB settings. This should fail as we no longer allow overwriting
         # experiments stored in the DB.
         ax_client.create_experiment(
             name="test_experiment",
             parameters=[{"name": "x", "type": "range", "bounds": [-5.0, 10.0]}],
             overwrite_existing_experiment=True,
         )
     # Original experiment should still be in DB and not have been overwritten.
     self.assertEqual(len(ax_client.experiment.trials), 5)
Ejemplo n.º 13
0
Archivo: save.py Proyecto: Vilashcj/Ax
def update_generation_strategy(
    generation_strategy: GenerationStrategy,
    generator_runs: List[GeneratorRun],
    config: Optional[SQAConfig] = None,
) -> None:
    """Update generation strategy's current step and attach generator runs
    (using default SQAConfig)."""
    config = config or SQAConfig()
    encoder = Encoder(config=config)
    _update_generation_strategy(
        generation_strategy=generation_strategy,
        generator_runs=generator_runs,
        encoder=encoder,
    )
Ejemplo n.º 14
0
Archivo: save.py Proyecto: Vilashcj/Ax
def save_generation_strategy(generation_strategy: GenerationStrategy,
                             config: Optional[SQAConfig] = None) -> int:
    """Save generation strategy (using default SQAConfig if no config is
    specified). If the generation strategy has an experiment set, the experiment
    will be saved first.

    Returns:
        The ID of the saved generation strategy.
    """
    # Start up SQA encoder.
    config = config or SQAConfig()
    encoder = Encoder(config=config)

    return _save_generation_strategy(generation_strategy=generation_strategy,
                                     encoder=encoder)
Ejemplo n.º 15
0
Archivo: save.py Proyecto: ekilic/Ax
def save_experiment(experiment: Experiment,
                    config: Optional[SQAConfig] = None,
                    overwrite: bool = False) -> None:
    """Save experiment (using default SQAConfig)."""
    # pyre-fixme[25]: Assertion will always fail.
    if not isinstance(experiment, Experiment):
        raise ValueError("Can only save instances of Experiment")
    if not experiment.has_name:
        raise ValueError("Experiment name must be set prior to saving.")

    config = config or SQAConfig()
    encoder = Encoder(config=config)
    _save_experiment(experiment=experiment,
                     encoder=encoder,
                     overwrite=overwrite)
Ejemplo n.º 16
0
 def test_suppress_all_storage_errors(self, mock_save_exp, _):
     init_test_engine_and_session_factory(force_init=True)
     config = SQAConfig()
     encoder = Encoder(config=config)
     decoder = Decoder(config=config)
     db_settings = DBSettings(encoder=encoder, decoder=decoder)
     BareBonesTestScheduler(
         experiment=self.branin_experiment,  # Has runner and metrics.
         generation_strategy=self.two_sobol_steps_GS,
         options=SchedulerOptions(
             init_seconds_between_polls=0.1,  # Short between polls so test is fast.
             suppress_storage_errors_after_retries=True,
         ),
         db_settings=db_settings,
     )
     self.assertEqual(mock_save_exp.call_count, 3)
Ejemplo n.º 17
0
def update_properties_on_experiment(
    experiment_with_updated_properties: Experiment,
    config: Optional[SQAConfig] = None,
) -> None:
    config = config or SQAConfig()
    exp_sqa_class = config.class_to_sqa_class[Experiment]

    exp_id = experiment_with_updated_properties.db_id
    if exp_id is None:
        raise ValueError("Experiment must be saved before being updated.")

    with session_scope() as session:
        session.query(exp_sqa_class).filter_by(id=exp_id).update({
            "properties":
            experiment_with_updated_properties._properties,
        })
Ejemplo n.º 18
0
 def test_sqa_storage(self):
     init_test_engine_and_session_factory(force_init=True)
     config = SQAConfig()
     encoder = Encoder(config=config)
     decoder = Decoder(config=config)
     db_settings = DBSettings(encoder=encoder, decoder=decoder)
     ax_client = AxClient(db_settings=db_settings)
     ax_client.create_experiment(
         name="test_experiment",
         parameters=[
             {"name": "x", "type": "range", "bounds": [-5.0, 10.0]},
             {"name": "y", "type": "range", "bounds": [0.0, 15.0]},
         ],
         minimize=True,
     )
     for _ in range(5):
         parameters, trial_index = ax_client.get_next_trial()
         ax_client.complete_trial(
             trial_index=trial_index, raw_data=branin(*parameters.values())
         )
     gs = ax_client.generation_strategy
     ax_client = AxClient(db_settings=db_settings)
     ax_client.load_experiment_from_database("test_experiment")
     self.assertEqual(gs, ax_client.generation_strategy)
     with self.assertRaises(ValueError):
         # Overwriting existing experiment.
         ax_client.create_experiment(
             name="test_experiment",
             parameters=[
                 {"name": "x", "type": "range", "bounds": [-5.0, 10.0]},
                 {"name": "y", "type": "range", "bounds": [0.0, 15.0]},
             ],
             minimize=True,
         )
     with self.assertRaises(ValueError):
         # Overwriting existing experiment with overwrite flag with present
         # DB settings. This should fail as we no longer allow overwriting
         # experiments stored in the DB.
         ax_client.create_experiment(
             name="test_experiment",
             parameters=[{"name": "x", "type": "range", "bounds": [-5.0, 10.0]}],
             overwrite_existing_experiment=True,
         )
     # Original experiment should still be in DB and not have been overwritten.
     self.assertEqual(len(ax_client.experiment.trials), 5)
Ejemplo n.º 19
0
def save_or_update_trials(
    experiment: Experiment,
    trials: List[BaseTrial],
    config: Optional[SQAConfig] = None,
    batch_size: Optional[int] = None,
) -> None:
    """Add new trials to the experiment, or update if already exists
    (using default SQAConfig)."""
    config = config or SQAConfig()
    encoder = Encoder(config=config)
    decoder = Decoder(config=config)
    _save_or_update_trials(
        experiment=experiment,
        trials=trials,
        encoder=encoder,
        decoder=decoder,
        batch_size=batch_size,
    )
Ejemplo n.º 20
0
Archivo: save.py Proyecto: jlin27/Ax
def save_generation_strategy(generation_strategy: GenerationStrategy,
                             config: Optional[SQAConfig] = None) -> int:
    """Save generation strategy (using default SQAConfig if no config is
    specified). If the generation strategy has an experiment set, the experiment
    will be saved first.

    Returns:
        The ID of the saved generation strategy.
    """

    # Start up SQA encoder.
    config = config or SQAConfig()
    encoder = Encoder(config=config)

    # If the generation strategy has not yet generated anything, there will be no
    # experiment set on it.
    if generation_strategy._experiment is None:
        experiment_id = None
    else:
        # Experiment was set on the generation strategy, so we need to save it first.
        save_experiment(experiment=generation_strategy._experiment,
                        config=config)
        experiment_id = _get_experiment_id(
            experiment=generation_strategy._experiment, encoder=encoder)

    gs_sqa = encoder.generation_strategy_to_sqa(
        generation_strategy=generation_strategy, experiment_id=experiment_id)

    with session_scope() as session:
        if generation_strategy._db_id is None:
            session.add(gs_sqa)
            session.flush()  # Ensures generation strategy id is set.
            generation_strategy._db_id = gs_sqa.id
        else:
            existing_gs_sqa = session.query(SQAGenerationStrategy).get(
                generation_strategy._db_id)
            existing_gs_sqa.update(gs_sqa)
            # our update logic ignores foreign keys, i.e. fields ending in _id,
            # because we want SQLAlchemy to handle those relationships for us
            # however, generation_strategy.experiment_id is an exception, so we
            # need to update that manually
            existing_gs_sqa.experiment_id = gs_sqa.experiment_id

    return generation_strategy._db_id
Ejemplo n.º 21
0
 def test_sqa_storage(self):
     init_test_engine_and_session_factory(force_init=True)
     config = SQAConfig()
     encoder = Encoder(config=config)
     decoder = Decoder(config=config)
     db_settings = DBSettings(encoder=encoder, decoder=decoder)
     ax_client = AxClient(db_settings=db_settings)
     ax_client.create_experiment(
         name="test_experiment",
         parameters=[
             {"name": "x1", "type": "range", "bounds": [-5.0, 10.0]},
             {"name": "x2", "type": "range", "bounds": [0.0, 15.0]},
         ],
         minimize=True,
     )
     for _ in range(5):
         parameters, trial_index = ax_client.get_next_trial()
         ax_client.complete_trial(
             trial_index=trial_index, raw_data=branin(*parameters.values())
         )
     gs = ax_client.generation_strategy
     ax_client = AxClient(db_settings=db_settings)
     ax_client.load_experiment_from_database("test_experiment")
     self.assertEqual(gs, ax_client.generation_strategy)
     with self.assertRaises(ValueError):
         # Overwriting existing experiment.
         ax_client.create_experiment(
             name="test_experiment",
             parameters=[
                 {"name": "x1", "type": "range", "bounds": [-5.0, 10.0]},
                 {"name": "x2", "type": "range", "bounds": [0.0, 15.0]},
             ],
             minimize=True,
         )
     # Overwriting existing experiment with overwrite flag.
     ax_client.create_experiment(
         name="test_experiment",
         parameters=[{"name": "x1", "type": "range", "bounds": [-5.0, 10.0]}],
         overwrite_existing_experiment=True,
     )
     # There should be no trials, as we just put in a fresh experiment.
     self.assertEqual(len(ax_client.experiment.trials), 0)
Ejemplo n.º 22
0
def update_generation_strategy(
    generation_strategy: GenerationStrategy,
    generator_runs: List[GeneratorRun],
    config: Optional[SQAConfig] = None,
    batch_size: Optional[int] = None,
    reduce_state_generator_runs: bool = False,
) -> None:
    """Update generation strategy's current step and attach generator runs
    (using default SQAConfig)."""
    config = config or SQAConfig()
    encoder = Encoder(config=config)
    decoder = Decoder(config=config)
    _update_generation_strategy(
        generation_strategy=generation_strategy,
        generator_runs=generator_runs,
        encoder=encoder,
        decoder=decoder,
        batch_size=batch_size,
        reduce_state_generator_runs=reduce_state_generator_runs,
    )
Ejemplo n.º 23
0
Archivo: load.py Proyecto: viotemp1/Ax
def load_experiment(
    experiment_name: str,
    config: Optional[SQAConfig] = None,
    reduced_state: bool = False,
) -> Experiment:
    """Load experiment by name.

    Args:
        experiment_name: Name of the expeirment to load.
        config: `SQAConfig`, from which to retrieve the decoder. Optional,
            defaults to base `SQAConfig`.
        reduced_state: Whether to load experiment with a slightly reduced state
            (without abandoned arms on experiment and withoug model state,
            search space, and optimization config on generator runs).
    """
    config = config or SQAConfig()
    decoder = Decoder(config=config)
    return _load_experiment(
        experiment_name=experiment_name, decoder=decoder, reduced_state=reduced_state
    )
Ejemplo n.º 24
0
 def test_sqa_storage(self):
     init_test_engine_and_session_factory(force_init=True)
     config = SQAConfig()
     encoder = Encoder(config=config)
     decoder = Decoder(config=config)
     db_settings = DBSettings(encoder=encoder, decoder=decoder)
     ax = AxClient(db_settings=db_settings)
     ax.create_experiment(
         name="test_experiment",
         parameters=[
             {"name": "x1", "type": "range", "bounds": [-5.0, 10.0]},
             {"name": "x2", "type": "range", "bounds": [0.0, 15.0]},
         ],
         minimize=True,
     )
     for _ in range(5):
         parameters, trial_index = ax.get_next_trial()
         ax.complete_trial(
             trial_index=trial_index, raw_data=branin(*parameters.values())
         )
     gs = ax.generation_strategy
     ax = AxClient(db_settings=db_settings)
     ax.load_experiment_from_database("test_experiment")
     self.assertEqual(gs, ax.generation_strategy)
Ejemplo n.º 25
0
 def test_sqa_storage(self):
     init_test_engine_and_session_factory(force_init=True)
     config = SQAConfig()
     encoder = Encoder(config=config)
     decoder = Decoder(config=config)
     db_settings = DBSettings(encoder=encoder, decoder=decoder)
     experiment = self.branin_experiment
     # Scheduler currently requires that the experiment be pre-saved.
     with self.assertRaisesRegex(ValueError, ".* must specify a name"):
         experiment._name = None
         scheduler = TestScheduler(
             experiment=experiment,
             generation_strategy=self.two_sobol_steps_GS,
             options=SchedulerOptions(total_trials=1),
             db_settings=db_settings,
         )
     experiment._name = "test_experiment"
     NUM_TRIALS = 5
     scheduler = TestScheduler(
         experiment=experiment,
         generation_strategy=self.two_sobol_steps_GS,
         options=SchedulerOptions(
             total_trials=NUM_TRIALS,
             init_seconds_between_polls=
             0,  # No wait between polls so test is fast.
         ),
         db_settings=db_settings,
     )
     # Check that experiment and GS were saved.
     exp, gs = scheduler._load_experiment_and_generation_strategy(
         experiment.name)
     self.assertEqual(exp, experiment)
     self.assertEqual(gs, self.two_sobol_steps_GS)
     scheduler.run_all_trials()
     # Check that experiment and GS were saved and test reloading with reduced state.
     exp, gs = scheduler._load_experiment_and_generation_strategy(
         experiment.name, reduced_state=True)
     self.assertEqual(len(exp.trials), NUM_TRIALS)
     self.assertEqual(len(gs._generator_runs), NUM_TRIALS)
     # Test `from_stored_experiment`.
     new_scheduler = TestScheduler.from_stored_experiment(
         experiment_name=experiment.name,
         options=SchedulerOptions(
             total_trials=NUM_TRIALS + 1,
             init_seconds_between_polls=
             0,  # No wait between polls so test is fast.
         ),
         db_settings=db_settings,
     )
     # Hack "resumed from storage timestamp" into `exp` to make sure all other fields
     # are equal, since difference in resumed from storage timestamps is expected.
     exp._properties[
         ExperimentStatusProperties.
         RESUMED_FROM_STORAGE_TIMESTAMPS] = new_scheduler.experiment._properties[
             ExperimentStatusProperties.RESUMED_FROM_STORAGE_TIMESTAMPS]
     self.assertEqual(new_scheduler.experiment, exp)
     self.assertEqual(new_scheduler.generation_strategy, gs)
     self.assertEqual(
         len(new_scheduler.experiment._properties[
             ExperimentStatusProperties.RESUMED_FROM_STORAGE_TIMESTAMPS]),
         1,
     )