def test_is_aip_training_job_running_with_running_job(self): self.mock_request.execute.side_effect = [{ "state": "QUEUED" }, { "state": "PREPARING" }, { "state": "RUNNING" }, { "state": "CANCELLING" }] queued_status = google_api_client.is_aip_training_job_running( self._job_id, self._project_id) self.assertTrue(queued_status) job_name = "projects/{}/jobs/{}".format(self._project_id, self._job_id) self.mock_apiclient.projects().jobs().get.assert_called_with( name=job_name) preparing_status = google_api_client.is_aip_training_job_running( self._job_id, self._project_id) self.assertTrue(preparing_status) running_status = google_api_client.is_aip_training_job_running( self._job_id, self._project_id) self.assertTrue(running_status) canceling_status = google_api_client.is_aip_training_job_running( self._job_id, self._project_id) self.assertTrue(canceling_status) self.assertEqual(4, self.mock_request.execute.call_count)
def test_is_aip_training_job_running_with_completed_job(self): self.mock_request.execute.side_effect = [ {"state": "SUCCEEDED"}, {"state": "CANCELLED"}, {"state": "FAILED", "errorMessage": "test_error_message"}] succeeded_status = google_api_client.is_aip_training_job_running( self._job_id, self._project_id) self.assertFalse(succeeded_status) job_name = "projects/{}/jobs/{}".format(self._project_id, self._job_id) self.mock_apiclient.projects().jobs().get.assert_called_with( name=job_name) cancelled_status = google_api_client.is_aip_training_job_running( self._job_id, self._project_id) self.assertFalse(cancelled_status) failed_status = google_api_client.is_aip_training_job_running( self._job_id, self._project_id) self.assertFalse(failed_status) self.assertEqual(3, self.mock_request.execute.call_count)
def run_trial(self, trial, *fit_args, **fit_kwargs): """Evaluates a set of hyperparameter values. This method is called during `search` to evaluate a set of hyperparameters using AI Platform training. Arguments: trial: A `Trial` instance that contains the information needed to run this trial. `Hyperparameters` can be accessed via `trial.hyperparameters`. *fit_args: Positional arguments passed by `search`. **fit_kwargs: Keyword arguments passed by `search`. Raises: RuntimeError: If AIP training job fails. """ # Running the training remotely. copied_fit_kwargs = copy.copy(fit_kwargs) # Handle any callbacks passed to `fit`. callbacks = fit_kwargs.pop("callbacks", []) callbacks = self._deepcopy_callbacks(callbacks) # Note: run_trial does not use `TunerCallback` calls, since # training is performed on AI Platform training remotely. # Handle TensorBoard/hyperparameter logging here. The TensorBoard # logs are used for passing metrics back from remote execution. self._add_logging(callbacks, trial) # Creating a save_model checkpoint callback with a saved model file path # specific to this trial. This is to prevent different trials from # overwriting each other. self._add_model_checkpoint_callback(callbacks, trial.trial_id) copied_fit_kwargs["callbacks"] = callbacks model = self.hypermodel.build(trial.hyperparameters) remote_dir = os.path.join(self.directory, str(trial.trial_id)) job_id = f"{self._study_id}_{trial.trial_id}" # Create job spec from worker count and config job_spec = self._get_job_spec_from_config(job_id) tf.get_logger().info( "Calling cloud_fit with %s", { "model": model, "remote_dir": remote_dir, "region": self._region, "project_id": self._project_id, "image_uri": self._container_uri, "job_id": job_id, "*fit_args": fit_args, "job_spec": job_spec, "**copied_fit_kwargs": copied_fit_kwargs }) cloud_fit_client.cloud_fit(model=model, remote_dir=remote_dir, region=self._region, project_id=self._project_id, image_uri=self._container_uri, job_id=job_id, job_spec=job_spec, *fit_args, **copied_fit_kwargs) # Create an instance of tensorboard DirectoryWatcher to retrieve the # logs for this trial run log_path = os.path.join(self._get_tensorboard_log_dir(trial.trial_id), "train") # Tensorboard log watcher expects the path to exist tf.io.gfile.makedirs(log_path) tf.get_logger().info( f"Retrieving training logs for trial {trial.trial_id} from" f" {log_path}") log_reader = tf_utils.get_tensorboard_log_watcher_from_path(log_path) training_metrics = _TrainingMetrics([], {}) epoch = 0 while google_api_client.is_aip_training_job_running( job_id, self._project_id): time.sleep(_POLLING_INTERVAL_IN_SECONDS) # Retrieve available metrics if any training_metrics = self._get_remote_training_metrics( log_reader, training_metrics.partial_epoch_metrics) for epoch_metrics in training_metrics.completed_epoch_metrics: # TODO(b/169197272) Validate metrics contain oracle objective if epoch_metrics: trial.status = self.oracle.update_trial( trial_id=trial.trial_id, metrics=epoch_metrics, step=epoch) epoch += 1 if trial.status == "STOPPED": google_api_client.stop_aip_training_job( job_id, self._project_id) break # Ensure the training job has completed successfully. if not google_api_client.wait_for_aip_training_job_completion( job_id, self._project_id): raise RuntimeError( "AIP Training job failed, see logs for details at " "https://console.cloud.google.com/ai-platform/jobs/" "{}/charts/cpu?project={}".format(job_id, self._project_id)) # Retrieve and report any remaining metrics training_metrics = self._get_remote_training_metrics( log_reader, training_metrics.partial_epoch_metrics) for epoch_metrics in training_metrics.completed_epoch_metrics: # TODO(b/169197272) Validate metrics contain oracle objective # TODO(b/170907612) Support submit partial results to Oracle if epoch_metrics: self.oracle.update_trial(trial_id=trial.trial_id, metrics=epoch_metrics, step=epoch) epoch += 1 # submit final epoch metrics if training_metrics.partial_epoch_metrics: self.oracle.update_trial( trial_id=trial.trial_id, metrics=training_metrics.partial_epoch_metrics, step=epoch)