def test_run_on_script(self): track_status = { # TODO(b/172668718) Enable tests after b/172668718 is resolved. # "auto_mirrored_strategy": self.auto_mirrored_strategy(), "auto_tpu_strategy": self.auto_tpu_strategy(), # "auto_one_device_strategy": self.auto_one_device_strategy(), "auto_multi_worker_strategy": self.auto_multi_worker_strategy(), # "none_dist_strat": self.none_dist_strat(), # "docker_config_cloud_build": self.docker_config_cloud_build(), "docker_config_parent_img": self.docker_config_parent_img(), # "docker_config_image": self.docker_config_image(), # "docker_config_cache_from": self.docker_config_cache_from(), # "job_labels": self.job_labels(), "cloud_build_base_image_backward_compatibility": self.cloud_build_base_image_backward_compatibility(), } for test_name, ret_val in track_status.items(): self.assertTrue( google_api_client.wait_for_api_training_job_completion( ret_val["job_id"], _PROJECT_ID), "Job {} generated from the test: {} has failed".format( ret_val["job_id"], test_name))
def test_run_on_script(self): track_status = { "auto_mirrored_strategy": self.auto_mirrored_strategy(), "auto_tpu_strategy": self.auto_tpu_strategy(), "auto_one_device_strategy": self.auto_one_device_strategy(), "auto_one_device_strategy_cloud_build": self.auto_one_device_strategy_cloud_build(), "auto_multi_worker_strategy": self.auto_multi_worker_strategy(), "none_dist_strat_multi_worker_strategy": self.none_dist_strat_multi_worker_strategy(), "auto_dist_strat_mwms_with_parent_img": self.auto_dist_strat_mwms_with_parent_img(), "auto_one_device_job_labels": self.auto_one_device_job_labels(), "auto_one_device_strategy_with_image": self.auto_one_device_strategy_with_image(), "auto_one_device_strategy_cloud_build_image_cache_from": self.auto_one_device_strategy_cloud_build_image_cache_from(), } for test_name, ret_val in track_status.items(): self.assertTrue( google_api_client.wait_for_api_training_job_completion( ret_val["job_id"], _PROJECT_ID), "Job {} generated from the test: {} has failed".format( ret_val["job_id"], test_name))
def test_wait_for_api_training_job_completion_multiple_checks_success( self): self.mock_request.execute.side_effect = [{ "state": "PREPARING" }, { "state": "RUNNING" }, { "state": "SUCCEEDED" }] status = google_api_client.wait_for_api_training_job_completion( self._job_id, self._project_id) self.assertTrue(status) self.assertEqual(3, self.mock_request.execute.call_count)
def test_wait_for_api_training_job_completion_non_blocking_failed( self, mock_log_error): self.mock_request.execute.return_value = { "state": "FAILED", "errorMessage": "test_error_message" } status = google_api_client.wait_for_api_training_job_completion( self._job_id, self._project_id) self.assertFalse(status) self.mock_request.execute.assert_called_once() mock_log_error.assert_called_once_with( "AIP Training job %s failed with error %s.", self._job_id, "test_error_message")
def test_wait_for_api_training_job_completion_non_blocking_cancelled( self, mock_log_error): self.mock_request.execute.return_value = { "state": "CANCELLED", } status = google_api_client.wait_for_api_training_job_completion( self._job_id, self._project_id) self.assertTrue(status) self.mock_request.execute.assert_called_once() job_name = "projects/{}/jobs/{}".format(self._project_id, self._job_id) self.mock_apiclient.projects().jobs().get.assert_called_with( name=job_name) mock_log_error.assert_not_called()
def test_wait_for_api_training_job_completion_multiple_checks_failed( self, mock_log_error): self.mock_request.execute.side_effect = [{ "state": "PREPARING" }, { "state": "RUNNING" }, { "state": "FAILED", "errorMessage": "test_error_message" }] status = google_api_client.wait_for_api_training_job_completion( self._job_id, self._project_id) self.assertFalse(status) self.assertEqual(3, self.mock_request.execute.call_count) mock_log_error.assert_called_once_with( "AIP Training job %s failed with error %s.", self._job_id, "test_error_message")
def test_in_memory_data(self): # This test should only run in tf 2.x if utils.is_tf_v1(): return # Create a folder under remote dir for this test's data tmp_folder = str(uuid.uuid4()) remote_dir = os.path.join(self._remote_dir, tmp_folder) # Keep track of test folders created for final clean up self._test_folders.append(remote_dir) x = np.random.random((2, 3)) y = np.random.randint(0, 2, (2, 2)) job_id = client.cloud_fit( self._model(), x=x, y=y, remote_dir=remote_dir, region=self._region, project_id=self._project_id, image_uri=self._image_uri, job_id="cloud_fit_e2e_test_{}_{}".format( _BUILD_ID.replace("-", "_"), "test_in_memory_data" ), epochs=2, ) # TODO(b/169297404) Replace AIP job status logic with utils wrapper # Wait for AIP Training job to finish successfully self.assertTrue( google_api_client.wait_for_api_training_job_completion( job_id, self._project_id)) # load model from remote dir trained_model = tf.keras.models.load_model(os.path.join( remote_dir, "output")) eval_results = trained_model.evaluate(x, y) # Accuracy should be better than zero self.assertListEqual(trained_model.metrics_names, ["loss", "accuracy"]) self.assertGreater(eval_results[1], 0)
def run_trial(self, trial, *fit_args, **fit_kwargs): """Evaluates a set of hyperparameter values. This method is called during `search` to evaluate a set of hyperparameters using AI Platform training. Arguments: trial: A `Trial` instance that contains the information needed to run this trial. `Hyperparameters` can be accessed via `trial.hyperparameters`. *fit_args: Positional arguments passed by `search`. **fit_kwargs: Keyword arguments passed by `search`. Raises: RuntimeError: If AIP training job fails. """ # Running the training remotely. copied_fit_kwargs = copy.copy(fit_kwargs) # Handle any callbacks passed to `fit`. callbacks = fit_kwargs.pop("callbacks", []) callbacks = self._deepcopy_callbacks(callbacks) # Note: run_trial does not use `TunerCallback` calls, since # training is performed on AI Platform training remotely. # Handle TensorBoard/hyperparameter logging here. The TensorBoard # logs are used for passing metrics back from remote execution. self._add_logging(callbacks, trial) # Creating a save_model checkpoint callback with a saved model file path # specific to this trial. This is to prevent different trials from # overwriting each other. self._add_model_checkpoint_callback( callbacks, trial.trial_id) copied_fit_kwargs["callbacks"] = callbacks model = self.hypermodel.build(trial.hyperparameters) remote_dir = os.path.join(self.directory, str(trial.trial_id)) # TODO(b/170687807) Switch from using "{}".format() to f-string job_id = "{}_{}".format(self._study_id, trial.trial_id) # Create job spec from worker count and config job_spec = self._get_job_spec_from_config(job_id) tf.get_logger().info("Calling cloud_fit with %s", { "model": model, "remote_dir": remote_dir, "region": self._region, "project_id": self._project_id, "image_uri": self._container_uri, "job_id": job_id, "*fit_args": fit_args, "job_spec": job_spec, "**copied_fit_kwargs": copied_fit_kwargs}) cloud_fit_client.cloud_fit( model=model, remote_dir=remote_dir, region=self._region, project_id=self._project_id, image_uri=self._container_uri, job_id=job_id, job_spec=job_spec, *fit_args, **copied_fit_kwargs) # Create an instance of tensorboard DirectoryWatcher to retrieve the # logs for this trial run log_path = os.path.join( self._get_tensorboard_log_dir(trial.trial_id), "train") # Tensorboard log watcher expects the path to exist tf.io.gfile.makedirs(log_path) # TODO(b/170687807) Switch from using "{}".format() to f-string tf.get_logger().info( "Retrieving training logs for trial {} from {}".format( trial.trial_id, log_path)) log_reader = tf_utils.get_tensorboard_log_watcher_from_path(log_path) training_metrics = _TrainingMetrics([], {}) epoch = 0 while google_api_client.is_api_training_job_running( job_id, self._project_id): time.sleep(_POLLING_INTERVAL_IN_SECONDS) # Retrieve available metrics if any training_metrics = self._get_remote_training_metrics( log_reader, training_metrics.partial_epoch_metrics) for epoch_metrics in training_metrics.completed_epoch_metrics: # TODO(b/169197272) Validate metrics contain oracle objective if epoch_metrics: trial.status = self.oracle.update_trial( trial_id=trial.trial_id, metrics=epoch_metrics, step=epoch) epoch += 1 if trial.status == "STOPPED": google_api_client.stop_aip_training_job( job_id, self._project_id) break # Ensure the training job has completed successfully. if not google_api_client.wait_for_api_training_job_completion( job_id, self._project_id): raise RuntimeError( "AIP Training job failed, see logs for details at " "https://console.cloud.google.com/ai-platform/jobs/" "{}/charts/cpu?project={}" .format(job_id, self._project_id)) # Retrieve and report any remaining metrics training_metrics = self._get_remote_training_metrics( log_reader, training_metrics.partial_epoch_metrics) for epoch_metrics in training_metrics.completed_epoch_metrics: # TODO(b/169197272) Validate metrics contain oracle objective # TODO(b/170907612) Support submit partial results to Oracle if epoch_metrics: self.oracle.update_trial( trial_id=trial.trial_id, metrics=epoch_metrics, step=epoch) epoch += 1 # submit final epoch metrics if training_metrics.partial_epoch_metrics: self.oracle.update_trial( trial_id=trial.trial_id, metrics=training_metrics.partial_epoch_metrics, step=epoch)