def test_create_or_load_study_with_409_raises_RuntimeError( self, mock_discovery): """Verify that get_study gracefully handles 409 errors.""" mock_request = mock.MagicMock() mock_request.execute.side_effect = errors.HttpError( httplib2.Response(info={"status": 409}), b"") mock_create_study = mock.MagicMock() mock_create_study.return_value = mock_request mock_discovery.build_from_document.return_value.projects().locations( ).studies().create = (mock_create_study) mock_get_study = mock.MagicMock() mock_get_study.side_effect = [ errors.HttpError(httplib2.Response(info={"status": 400}), b"") ] * 3 mock_discovery.build_from_document.return_value.projects().locations( ).studies().get = (mock_get_study) with self.assertRaisesRegex( RuntimeError, 'GetStudy wasn\'t successful after 3 tries: <HttpError 400 "Ok">', ): optimizer_client.create_or_load_study( project_id=self._project_id, region=self._region, study_id=self._study_id, study_config=self._study_config, )
def test_create_or_load_study_no_study_config_with_404_raises_ValueError( self, mock_discovery): mock_request = mock.MagicMock() mock_request.execute.side_effect = errors.HttpError( httplib2.Response(info={"status": 404}), b"" ) mock_get_study = mock.MagicMock() mock_get_study.return_value = mock_request mock_discovery.build_from_document.return_value.projects().locations( ).studies().get = mock_get_study with self.assertRaisesRegex( ValueError, "GetStudy failed. Study not found: {}.".format(self._study_id), ): optimizer_client.create_or_load_study( project_id=self._project_id, region=self._region, study_id=self._study_id, ) mock_get_study.assert_called_with( name="projects/{}/locations/{}/studies/{}".format( self._project_id, self._region, self._study_id ) )
def test_create_or_load_study_newstudy(self, mock_discovery): mock_create_study = mock.MagicMock() mock_discovery.build_from_document.return_value.projects().locations( ).studies().create = (mock_create_study) client = optimizer_client.create_or_load_study( project_id=self._project_id, region=self._region, study_id=self._study_id, study_config=self._study_config, ) self.assertIsInstance(client, optimizer_client._OptimizerClient) _, mock_kwargs = mock_discovery.build_from_document.call_args self.assertIn("service", mock_kwargs) self.assertIsInstance(mock_kwargs["service"], dict) self.assertEqual( mock_kwargs["service"]["rootUrl"], # Regional endpoint must be specified for Optimizer client. "https://us-central1-ml.googleapis.com/", ) mock_create_study.assert_called_with( body={"study_config": self._study_config}, parent="projects/{}/locations/{}".format(self._project_id, self._region), studyId=self._study_id, )
def test_create_or_load_study_with_409_success(self, mock_discovery): """Verify that get_study gracefully handles 409 errors.""" mock_create_request = mock.MagicMock() mock_create_request.execute.side_effect = errors.HttpError( httplib2.Response(info={"status": 409}), b"") mock_create_study = mock.MagicMock() mock_create_study.return_value = mock_create_request mock_discovery.build_from_document.return_value.projects().locations( ).studies().create = (mock_create_study) mock_get_request = mock.MagicMock() mock_get_request.execute.side_effect = [ errors.HttpError(httplib2.Response(info={"status": 400}), b""), errors.HttpError(httplib2.Response(info={"status": 400}), b""), mock.DEFAULT, ] mock_get_study = mock.MagicMock() mock_get_study.side_effect = mock_get_request mock_discovery.build_from_document.return_value.projects().locations( ).studies().get = (mock_get_study) client = optimizer_client.create_or_load_study( project_id=self._project_id, region=self._region, study_id=self._study_id, study_config=self._study_config, ) self.assertIsInstance(client, optimizer_client._OptimizerClient)
def tearDown(self): super(_CloudTunerIntegrationTestBase, self).tearDown() # Delete the study used in the test, if present if self._study_id: service = optimizer_client.create_or_load_study( _PROJECT_ID, _REGION, self._study_id, None) service.delete_study() tf.keras.backend.clear_session()
def tearDown(self): super(_DistributingCloudTunerIntegrationTestBase, self).tearDown() # Delete the study used in the test, if present if self._study_id: service = optimizer_client.create_or_load_study( _PROJECT_ID, _REGION, self._study_id, None) service.delete_study() tf.keras.backend.clear_session() # Delete log files, saved_models and other training assets self._delete_dir(_REMOTE_DIR)
def __init__( self, project_id: Text, region: Text, objective: Union[Text, oracle_module.Objective] = None, hyperparameters: hp_module.HyperParameters = None, study_config: Optional[Dict[Text, Any]] = None, max_trials: int = None, study_id: Optional[Text] = None, ): """KerasTuner Oracle interface implemented with Optimizer backend. Args: project_id: A GCP project id. region: A GCP region. e.g. 'us-central1'. objective: If a string, the direction of the optimization (min or max) will be inferred. hyperparameters: Mandatory and must include definitions for all hyperparameters used during the search. Can be used to override (or register in advance) hyperparameters in the search space. study_config: Study configuration for CAIP Optimizer service. max_trials: Total number of trials (model configurations) to test at most. If None, it continues the search until it reaches the Optimizer trial limit for each study. Users may stop the search externally (e.g. by killing the job). Note that the Oracle may interrupt the search before `max_trials` models have been tested. study_id: An identifier of the study. If not supplied, system-determined unique ID is given. The full study name will be `projects/{project_id}/locations/{region}/studies/{study_id}`, and the full trial name will be `{study name}/trials/{trial_id}`. """ if study_config: if objective or hyperparameters: raise ValueError( "Please configure either study_config or " '"objective, and hyperparameters".' ) objective = utils.convert_study_config_to_objective(study_config) hyperparameters = utils.convert_study_config_to_hps(study_config) self.study_config = study_config else: if not (objective and hyperparameters): raise ValueError( "If study_config is not set, " "objective and hyperparameters must be set." ) self.study_config = utils.make_study_config(objective, hyperparameters) super(CloudOracle, self).__init__( objective=objective, hyperparameters=hyperparameters, max_trials=max_trials, allow_new_entries=False, tune_new_entries=False, ) if not project_id: raise ValueError('"project_id" is not found.') self._project_id = project_id if not region: raise ValueError('"region" is not found.') self._region = region self.objective = utils.format_objective(objective) self.hyperparameters = hyperparameters self.max_trials = max_trials self.study_id = study_id or "CloudTuner_study_{}".format( datetime.datetime.now().strftime("%Y%m%d_%H%M%S") ) self.service = optimizer_client.create_or_load_study( self._project_id, self._region, self.study_id, self.study_config ) self.trials = {} self._start_time = None