def test_run_hyperparameter_tuning_job_with_fail_raises( self, create_hyperparameter_tuning_job_mock, get_hyperparameter_tuning_job_mock_with_fail, sync, ): aiplatform.init( project=_TEST_PROJECT, location=_TEST_LOCATION, staging_bucket=_TEST_STAGING_BUCKET, encryption_spec_key_name=_TEST_DEFAULT_ENCRYPTION_KEY_NAME, ) custom_job = aiplatform.CustomJob( display_name=test_custom_job._TEST_DISPLAY_NAME, worker_pool_specs=test_custom_job._TEST_WORKER_POOL_SPEC, base_output_dir=test_custom_job._TEST_BASE_OUTPUT_DIR, ) job = aiplatform.HyperparameterTuningJob( display_name=_TEST_DISPLAY_NAME, custom_job=custom_job, metric_spec={_TEST_METRIC_SPEC_KEY: _TEST_METRIC_SPEC_VALUE}, parameter_spec={ "lr": hpt.DoubleParameterSpec(min=0.001, max=0.1, scale="log"), "units": hpt.IntegerParameterSpec(min=4, max=1028, scale="linear"), "activation": hpt.CategoricalParameterSpec( values=["relu", "sigmoid", "elu", "selu", "tanh"] ), "batch_size": hpt.DiscreteParameterSpec( values=[16, 32], scale="linear" ), }, parallel_trial_count=_TEST_PARALLEL_TRIAL_COUNT, max_trial_count=_TEST_MAX_TRIAL_COUNT, max_failed_trial_count=_TEST_MAX_FAILED_TRIAL_COUNT, search_algorithm=_TEST_SEARCH_ALGORITHM, measurement_selection=_TEST_MEASUREMENT_SELECTION, labels=_TEST_LABELS, ) with pytest.raises(RuntimeError): job.run( service_account=_TEST_SERVICE_ACCOUNT, network=_TEST_NETWORK, timeout=_TEST_TIMEOUT, restart_job_on_worker_restart=_TEST_RESTART_JOB_ON_WORKER_RESTART, sync=sync, ) job.wait() expected_hyperparameter_tuning_job = _get_hyperparameter_tuning_job_proto() create_hyperparameter_tuning_job_mock.assert_called_once_with( parent=_TEST_PARENT, hyperparameter_tuning_job=expected_hyperparameter_tuning_job, ) assert job._gca_resource.state == gca_job_state_compat.JobState.JOB_STATE_FAILED
def test_serialize_parameters(self): parameters = { 'lr': hpt.DoubleParameterSpec(min=0.001, max=0.1, scale='log'), 'units': hpt.IntegerParameterSpec(min=4, max=128, scale='linear'), 'activation': hpt.CategoricalParameterSpec(values=['relu', 'selu']), 'batch_size': hpt.DiscreteParameterSpec(values=[128, 256], scale='linear') } expected_outputs = [ '{\n "parameterId": "lr",\n "doubleValueSpec": {\n' ' "minValue": 0.001,\n "maxValue": 0.1\n },\n' ' "scaleType": 2,\n "conditionalParameterSpecs": []\n}', '{\n "parameterId": "units",\n "integerValueSpec": {\n' ' "minValue": "4",\n "maxValue": "128"\n },\n' ' "scaleType": 1,\n "conditionalParameterSpecs": []\n}', '{\n "parameterId": "activation",\n "categoricalValueSpec": {\n' ' "values": [\n "relu",\n "selu"\n ]\n },\n' ' "scaleType": 0,\n "conditionalParameterSpecs": []\n}', '{\n "parameterId": "batch_size",\n "discreteValueSpec": {\n' ' "values": [\n 128.0,\n 256.0\n ]\n },\n' ' "scaleType": 1,\n "conditionalParameterSpecs": []\n}', ] outputs = serialize_parameters(parameters) self.assertEqual(outputs, expected_outputs)
def tune_hyperparameters( project: str, location: str, container_uri: str, training_file_path: str, validation_file_path: str, staging_bucket: str, max_trial_count: int, parallel_trial_count: int ) -> NamedTuple('Outputs', [("best_accuracy", float), ("best_alpha", float), ("best_max_iter", int)]): from google.cloud import aiplatform from google.cloud.aiplatform import hyperparameter_tuning as hpt aiplatform.init(project=project, location=location, staging_bucket=staging_bucket) worker_pool_specs = [{ "machine_spec": { "machine_type": "n1-standard-4", "accelerator_type": "NVIDIA_TESLA_K80", "accelerator_count": 1, }, "replica_count": 1, "container_spec": { "image_uri": container_uri, "args": [ f"--training_dataset_path={training_file_path}", f"--validation_dataset_path={validation_file_path}", "--hptune" ], }, }] custom_job = aiplatform.CustomJob(display_name='covertype_kfp_trial_job', worker_pool_specs=worker_pool_specs) hp_job = aiplatform.HyperparameterTuningJob( display_name='covertype_kfp_tuning_job', custom_job=custom_job, metric_spec={ 'accuracy': 'maximize', }, parameter_spec={ 'alpha': hpt.DoubleParameterSpec(min=1.0e-4, max=1.0e-1, scale='linear'), 'max_iter': hpt.DiscreteParameterSpec(values=[1, 2], scale='linear') }, max_trial_count=max_trial_count, parallel_trial_count=parallel_trial_count, ) hp_job.run() metrics = [ trial.final_measurement.metrics[0].value for trial in hp_job.trials ] best_trial = hp_job.trials[metrics.index(max(metrics))] best_accuracy = float(best_trial.final_measurement.metrics[0].value) best_alpha = float(best_trial.parameters[0].value) best_max_iter = int(best_trial.parameters[1].value) return best_accuracy, best_alpha, best_max_iter
def test_serialize_parameters(self): parameters = { 'lr': hpt.DoubleParameterSpec(min=0.001, max=0.1, scale='log'), 'units': hpt.IntegerParameterSpec(min=4, max=128, scale='linear'), 'activation': hpt.CategoricalParameterSpec(values=['relu', 'selu']), 'batch_size': hpt.DiscreteParameterSpec(values=[128, 256], scale='linear') } expected_outputs = [ { 'parameter_id': 'lr', 'double_value_spec': { 'min_value': 0.001, 'max_value': 0.1 }, 'scale_type': 2, 'conditional_parameter_specs': [] }, { 'parameter_id': 'units', 'integer_value_spec': { 'min_value': '4', 'max_value': '128' }, 'scale_type': 1, 'conditional_parameter_specs': [] }, { 'parameter_id': 'activation', 'categorical_value_spec': { 'values': ['relu', 'selu'] }, 'scale_type': 0, 'conditional_parameter_specs': [] }, { 'parameter_id': 'batch_size', 'discrete_value_spec': { 'values': [128.0, 256.0] }, 'scale_type': 1, 'conditional_parameter_specs': [] }, ] outputs = serialize_parameters(parameters) self.assertEqual(outputs, expected_outputs)
def test_hyperparameter_tuning_job_get_state_raises_without_run(self): aiplatform.init( project=_TEST_PROJECT, location=_TEST_LOCATION, staging_bucket=_TEST_STAGING_BUCKET, encryption_spec_key_name=_TEST_DEFAULT_ENCRYPTION_KEY_NAME, ) custom_job = aiplatform.CustomJob( display_name=test_custom_job._TEST_DISPLAY_NAME, worker_pool_specs=test_custom_job._TEST_WORKER_POOL_SPEC, base_output_dir=test_custom_job._TEST_BASE_OUTPUT_DIR, ) job = aiplatform.HyperparameterTuningJob( display_name=_TEST_DISPLAY_NAME, custom_job=custom_job, metric_spec={_TEST_METRIC_SPEC_KEY: _TEST_METRIC_SPEC_VALUE}, parameter_spec={ "lr": hpt.DoubleParameterSpec(min=0.001, max=0.1, scale="log"), "units": hpt.IntegerParameterSpec(min=4, max=1028, scale="linear"), "activation": hpt.CategoricalParameterSpec( values=["relu", "sigmoid", "elu", "selu", "tanh"]), "batch_size": hpt.DiscreteParameterSpec(values=[16, 32, 64], scale="linear"), }, parallel_trial_count=_TEST_PARALLEL_TRIAL_COUNT, max_trial_count=_TEST_MAX_TRIAL_COUNT, max_failed_trial_count=_TEST_MAX_FAILED_TRIAL_COUNT, search_algorithm=_TEST_SEARCH_ALGORITHM, measurement_selection=_TEST_MEASUREMENT_SELECTION, ) with pytest.raises(RuntimeError): print(job.state)
def setUp(self): super(HPTuningJobCompileTest, self).setUp() self._display_name = "test_display_name" self._project = "test_project" self._location = "us-central1" self._package_path = os.path.join( os.getenv('TEST_UNDECLARED_OUTPUTS_DIR'), 'pipeline.json') self._worker_pool_specs = [{ "machine_spec": { "machine_type": "n1-standard-4", "accelerator_type": "NVIDIA_TESLA_T4", "accelerator_count": 1 }, "replica_count": 1, "container_spec": { "image_uri": "gcr.io/project_id/test" } }] self._metrics_spec = {"accuracy": "maximize"} self._parameter_spec = serialize_parameters({ "learning_rate": hpt.DoubleParameterSpec(min=0.001, max=1, scale="log"), }) self._base_output_directory = "gs://my-bucket/blob"
def test_run_hyperparameter_tuning_job_with_fail_at_creation(self): aiplatform.init( project=_TEST_PROJECT, location=_TEST_LOCATION, staging_bucket=_TEST_STAGING_BUCKET, encryption_spec_key_name=_TEST_DEFAULT_ENCRYPTION_KEY_NAME, ) custom_job = aiplatform.CustomJob( display_name=test_custom_job._TEST_DISPLAY_NAME, worker_pool_specs=test_custom_job._TEST_WORKER_POOL_SPEC, base_output_dir=test_custom_job._TEST_BASE_OUTPUT_DIR, ) job = aiplatform.HyperparameterTuningJob( display_name=_TEST_DISPLAY_NAME, custom_job=custom_job, metric_spec={_TEST_METRIC_SPEC_KEY: _TEST_METRIC_SPEC_VALUE}, parameter_spec={ "lr": hpt.DoubleParameterSpec(min=0.001, max=0.1, scale="log"), "units": hpt.IntegerParameterSpec(min=4, max=1028, scale="linear"), "activation": hpt.CategoricalParameterSpec( values=["relu", "sigmoid", "elu", "selu", "tanh"]), "batch_size": hpt.DiscreteParameterSpec(values=[16, 32], scale="linear"), }, parallel_trial_count=_TEST_PARALLEL_TRIAL_COUNT, max_trial_count=_TEST_MAX_TRIAL_COUNT, max_failed_trial_count=_TEST_MAX_FAILED_TRIAL_COUNT, search_algorithm=_TEST_SEARCH_ALGORITHM, measurement_selection=_TEST_MEASUREMENT_SELECTION, ) job.run( service_account=_TEST_SERVICE_ACCOUNT, network=_TEST_NETWORK, timeout=_TEST_TIMEOUT, restart_job_on_worker_restart=_TEST_RESTART_JOB_ON_WORKER_RESTART, sync=False, ) with pytest.raises(RuntimeError) as e: job.wait_for_resource_creation() assert e.match("Mock fail") with pytest.raises(RuntimeError) as e: job.resource_name assert e.match( "HyperparameterTuningJob resource has not been created. Resource failed with: Mock fail" ) with pytest.raises(RuntimeError) as e: job.network assert e.match( "HyperparameterTuningJob resource has not been created. Resource failed with: Mock fail" ) with pytest.raises(RuntimeError) as e: job.trials assert e.match( "HyperparameterTuningJob resource has not been created. Resource failed with: Mock fail" )
def test_create_hyperparameter_tuning_job_with_enable_web_access( self, create_hyperparameter_tuning_job_mock_with_enable_web_access, get_hyperparameter_tuning_job_mock_with_enable_web_access, sync, caplog, ): caplog.set_level(logging.INFO) aiplatform.init( project=_TEST_PROJECT, location=_TEST_LOCATION, staging_bucket=_TEST_STAGING_BUCKET, encryption_spec_key_name=_TEST_DEFAULT_ENCRYPTION_KEY_NAME, ) custom_job = aiplatform.CustomJob( display_name=test_custom_job._TEST_DISPLAY_NAME, worker_pool_specs=test_custom_job._TEST_WORKER_POOL_SPEC, base_output_dir=test_custom_job._TEST_BASE_OUTPUT_DIR, ) job = aiplatform.HyperparameterTuningJob( display_name=_TEST_DISPLAY_NAME, custom_job=custom_job, metric_spec={_TEST_METRIC_SPEC_KEY: _TEST_METRIC_SPEC_VALUE}, parameter_spec={ "lr": hpt.DoubleParameterSpec(min=0.001, max=0.1, scale="log"), "units": hpt.IntegerParameterSpec(min=4, max=1028, scale="linear"), "activation": hpt.CategoricalParameterSpec( values=["relu", "sigmoid", "elu", "selu", "tanh"]), "batch_size": hpt.DiscreteParameterSpec(values=[16, 32], scale="linear"), }, parallel_trial_count=_TEST_PARALLEL_TRIAL_COUNT, max_trial_count=_TEST_MAX_TRIAL_COUNT, max_failed_trial_count=_TEST_MAX_FAILED_TRIAL_COUNT, search_algorithm=_TEST_SEARCH_ALGORITHM, measurement_selection=_TEST_MEASUREMENT_SELECTION, labels=_TEST_LABELS, ) job.run( service_account=_TEST_SERVICE_ACCOUNT, network=_TEST_NETWORK, timeout=_TEST_TIMEOUT, restart_job_on_worker_restart=_TEST_RESTART_JOB_ON_WORKER_RESTART, enable_web_access=test_custom_job._TEST_ENABLE_WEB_ACCESS, sync=sync, create_request_timeout=None, ) job.wait() assert "workerpool0-0" in caplog.text expected_hyperparameter_tuning_job = ( _get_hyperparameter_tuning_job_proto_with_enable_web_access()) create_hyperparameter_tuning_job_mock_with_enable_web_access.assert_called_once_with( parent=_TEST_PARENT, hyperparameter_tuning_job=expected_hyperparameter_tuning_job, timeout=None, ) assert job.state == gca_job_state_compat.JobState.JOB_STATE_SUCCEEDED assert job.network == _TEST_NETWORK assert job.trials == [] caplog.clear()