def test_run_hyperparameter_tuning_job_with_fail_raises(
self,
create_hyperparameter_tuning_job_mock,
get_hyperparameter_tuning_job_mock_with_fail,
sync,
):
aiplatform.init(
project=_TEST_PROJECT,
location=_TEST_LOCATION,
staging_bucket=_TEST_STAGING_BUCKET,
encryption_spec_key_name=_TEST_DEFAULT_ENCRYPTION_KEY_NAME,
)
custom_job = aiplatform.CustomJob(
display_name=test_custom_job._TEST_DISPLAY_NAME,
worker_pool_specs=test_custom_job._TEST_WORKER_POOL_SPEC,
base_output_dir=test_custom_job._TEST_BASE_OUTPUT_DIR,
)
job = aiplatform.HyperparameterTuningJob(
display_name=_TEST_DISPLAY_NAME,
custom_job=custom_job,
metric_spec={_TEST_METRIC_SPEC_KEY: _TEST_METRIC_SPEC_VALUE},
parameter_spec={
"lr": hpt.DoubleParameterSpec(min=0.001, max=0.1, scale="log"),
"units": hpt.IntegerParameterSpec(min=4, max=1028, scale="linear"),
"activation": hpt.CategoricalParameterSpec(
values=["relu", "sigmoid", "elu", "selu", "tanh"]
),
"batch_size": hpt.DiscreteParameterSpec(
values=[16, 32], scale="linear"
),
},
parallel_trial_count=_TEST_PARALLEL_TRIAL_COUNT,
max_trial_count=_TEST_MAX_TRIAL_COUNT,
max_failed_trial_count=_TEST_MAX_FAILED_TRIAL_COUNT,
search_algorithm=_TEST_SEARCH_ALGORITHM,
measurement_selection=_TEST_MEASUREMENT_SELECTION,
labels=_TEST_LABELS,
)
with pytest.raises(RuntimeError):
job.run(
service_account=_TEST_SERVICE_ACCOUNT,
network=_TEST_NETWORK,
timeout=_TEST_TIMEOUT,
restart_job_on_worker_restart=_TEST_RESTART_JOB_ON_WORKER_RESTART,
sync=sync,
)
job.wait()
expected_hyperparameter_tuning_job = _get_hyperparameter_tuning_job_proto()
create_hyperparameter_tuning_job_mock.assert_called_once_with(
parent=_TEST_PARENT,
hyperparameter_tuning_job=expected_hyperparameter_tuning_job,
)
assert job._gca_resource.state == gca_job_state_compat.JobState.JOB_STATE_FAILED
def test_serialize_parameters(self):
parameters = {
'lr':
hpt.DoubleParameterSpec(min=0.001, max=0.1, scale='log'),
'units':
hpt.IntegerParameterSpec(min=4, max=128, scale='linear'),
'activation':
hpt.CategoricalParameterSpec(values=['relu', 'selu']),
'batch_size':
hpt.DiscreteParameterSpec(values=[128, 256], scale='linear')
}
expected_outputs = [
'{\n "parameterId": "lr",\n "doubleValueSpec": {\n'
' "minValue": 0.001,\n "maxValue": 0.1\n },\n'
' "scaleType": 2,\n "conditionalParameterSpecs": []\n}',
'{\n "parameterId": "units",\n "integerValueSpec": {\n'
' "minValue": "4",\n "maxValue": "128"\n },\n'
' "scaleType": 1,\n "conditionalParameterSpecs": []\n}',
'{\n "parameterId": "activation",\n "categoricalValueSpec": {\n'
' "values": [\n "relu",\n "selu"\n ]\n },\n'
' "scaleType": 0,\n "conditionalParameterSpecs": []\n}',
'{\n "parameterId": "batch_size",\n "discreteValueSpec": {\n'
' "values": [\n 128.0,\n 256.0\n ]\n },\n'
' "scaleType": 1,\n "conditionalParameterSpecs": []\n}',
]
outputs = serialize_parameters(parameters)
self.assertEqual(outputs, expected_outputs)
def tune_hyperparameters(
project: str, location: str, container_uri: str, training_file_path: str,
validation_file_path: str, staging_bucket: str, max_trial_count: int,
parallel_trial_count: int
) -> NamedTuple('Outputs', [("best_accuracy", float), ("best_alpha", float),
("best_max_iter", int)]):
from google.cloud import aiplatform
from google.cloud.aiplatform import hyperparameter_tuning as hpt
aiplatform.init(project=project,
location=location,
staging_bucket=staging_bucket)
worker_pool_specs = [{
"machine_spec": {
"machine_type": "n1-standard-4",
"accelerator_type": "NVIDIA_TESLA_K80",
"accelerator_count": 1,
},
"replica_count": 1,
"container_spec": {
"image_uri":
container_uri,
"args": [
f"--training_dataset_path={training_file_path}",
f"--validation_dataset_path={validation_file_path}", "--hptune"
],
},
}]
custom_job = aiplatform.CustomJob(display_name='covertype_kfp_trial_job',
worker_pool_specs=worker_pool_specs)
hp_job = aiplatform.HyperparameterTuningJob(
display_name='covertype_kfp_tuning_job',
custom_job=custom_job,
metric_spec={
'accuracy': 'maximize',
},
parameter_spec={
'alpha':
hpt.DoubleParameterSpec(min=1.0e-4, max=1.0e-1, scale='linear'),
'max_iter':
hpt.DiscreteParameterSpec(values=[1, 2], scale='linear')
},
max_trial_count=max_trial_count,
parallel_trial_count=parallel_trial_count,
)
hp_job.run()
metrics = [
trial.final_measurement.metrics[0].value for trial in hp_job.trials
]
best_trial = hp_job.trials[metrics.index(max(metrics))]
best_accuracy = float(best_trial.final_measurement.metrics[0].value)
best_alpha = float(best_trial.parameters[0].value)
best_max_iter = int(best_trial.parameters[1].value)
return best_accuracy, best_alpha, best_max_iter
def test_serialize_parameters(self):
parameters = {
'lr':
hpt.DoubleParameterSpec(min=0.001, max=0.1, scale='log'),
'units':
hpt.IntegerParameterSpec(min=4, max=128, scale='linear'),
'activation':
hpt.CategoricalParameterSpec(values=['relu', 'selu']),
'batch_size':
hpt.DiscreteParameterSpec(values=[128, 256], scale='linear')
}
expected_outputs = [
{
'parameter_id': 'lr',
'double_value_spec': {
'min_value': 0.001,
'max_value': 0.1
},
'scale_type': 2,
'conditional_parameter_specs': []
},
{
'parameter_id': 'units',
'integer_value_spec': {
'min_value': '4',
'max_value': '128'
},
'scale_type': 1,
'conditional_parameter_specs': []
},
{
'parameter_id': 'activation',
'categorical_value_spec': {
'values': ['relu', 'selu']
},
'scale_type': 0,
'conditional_parameter_specs': []
},
{
'parameter_id': 'batch_size',
'discrete_value_spec': {
'values': [128.0, 256.0]
},
'scale_type': 1,
'conditional_parameter_specs': []
},
]
outputs = serialize_parameters(parameters)
self.assertEqual(outputs, expected_outputs)
def test_hyperparameter_tuning_job_get_state_raises_without_run(self):
aiplatform.init(
project=_TEST_PROJECT,
location=_TEST_LOCATION,
staging_bucket=_TEST_STAGING_BUCKET,
encryption_spec_key_name=_TEST_DEFAULT_ENCRYPTION_KEY_NAME,
)
custom_job = aiplatform.CustomJob(
display_name=test_custom_job._TEST_DISPLAY_NAME,
worker_pool_specs=test_custom_job._TEST_WORKER_POOL_SPEC,
base_output_dir=test_custom_job._TEST_BASE_OUTPUT_DIR,
)
job = aiplatform.HyperparameterTuningJob(
display_name=_TEST_DISPLAY_NAME,
custom_job=custom_job,
metric_spec={_TEST_METRIC_SPEC_KEY: _TEST_METRIC_SPEC_VALUE},
parameter_spec={
"lr":
hpt.DoubleParameterSpec(min=0.001, max=0.1, scale="log"),
"units":
hpt.IntegerParameterSpec(min=4, max=1028, scale="linear"),
"activation":
hpt.CategoricalParameterSpec(
values=["relu", "sigmoid", "elu", "selu", "tanh"]),
"batch_size":
hpt.DiscreteParameterSpec(values=[16, 32, 64], scale="linear"),
},
parallel_trial_count=_TEST_PARALLEL_TRIAL_COUNT,
max_trial_count=_TEST_MAX_TRIAL_COUNT,
max_failed_trial_count=_TEST_MAX_FAILED_TRIAL_COUNT,
search_algorithm=_TEST_SEARCH_ALGORITHM,
measurement_selection=_TEST_MEASUREMENT_SELECTION,
)
with pytest.raises(RuntimeError):
print(job.state)
def setUp(self):
super(HPTuningJobCompileTest, self).setUp()
self._display_name = "test_display_name"
self._project = "test_project"
self._location = "us-central1"
self._package_path = os.path.join(
os.getenv('TEST_UNDECLARED_OUTPUTS_DIR'), 'pipeline.json')
self._worker_pool_specs = [{
"machine_spec": {
"machine_type": "n1-standard-4",
"accelerator_type": "NVIDIA_TESLA_T4",
"accelerator_count": 1
},
"replica_count": 1,
"container_spec": {
"image_uri": "gcr.io/project_id/test"
}
}]
self._metrics_spec = {"accuracy": "maximize"}
self._parameter_spec = serialize_parameters({
"learning_rate": hpt.DoubleParameterSpec(min=0.001, max=1, scale="log"),
})
self._base_output_directory = "gs://my-bucket/blob"
def test_run_hyperparameter_tuning_job_with_fail_at_creation(self):
aiplatform.init(
project=_TEST_PROJECT,
location=_TEST_LOCATION,
staging_bucket=_TEST_STAGING_BUCKET,
encryption_spec_key_name=_TEST_DEFAULT_ENCRYPTION_KEY_NAME,
)
custom_job = aiplatform.CustomJob(
display_name=test_custom_job._TEST_DISPLAY_NAME,
worker_pool_specs=test_custom_job._TEST_WORKER_POOL_SPEC,
base_output_dir=test_custom_job._TEST_BASE_OUTPUT_DIR,
)
job = aiplatform.HyperparameterTuningJob(
display_name=_TEST_DISPLAY_NAME,
custom_job=custom_job,
metric_spec={_TEST_METRIC_SPEC_KEY: _TEST_METRIC_SPEC_VALUE},
parameter_spec={
"lr":
hpt.DoubleParameterSpec(min=0.001, max=0.1, scale="log"),
"units":
hpt.IntegerParameterSpec(min=4, max=1028, scale="linear"),
"activation":
hpt.CategoricalParameterSpec(
values=["relu", "sigmoid", "elu", "selu", "tanh"]),
"batch_size":
hpt.DiscreteParameterSpec(values=[16, 32], scale="linear"),
},
parallel_trial_count=_TEST_PARALLEL_TRIAL_COUNT,
max_trial_count=_TEST_MAX_TRIAL_COUNT,
max_failed_trial_count=_TEST_MAX_FAILED_TRIAL_COUNT,
search_algorithm=_TEST_SEARCH_ALGORITHM,
measurement_selection=_TEST_MEASUREMENT_SELECTION,
)
job.run(
service_account=_TEST_SERVICE_ACCOUNT,
network=_TEST_NETWORK,
timeout=_TEST_TIMEOUT,
restart_job_on_worker_restart=_TEST_RESTART_JOB_ON_WORKER_RESTART,
sync=False,
)
with pytest.raises(RuntimeError) as e:
job.wait_for_resource_creation()
assert e.match("Mock fail")
with pytest.raises(RuntimeError) as e:
job.resource_name
assert e.match(
"HyperparameterTuningJob resource has not been created. Resource failed with: Mock fail"
)
with pytest.raises(RuntimeError) as e:
job.network
assert e.match(
"HyperparameterTuningJob resource has not been created. Resource failed with: Mock fail"
)
with pytest.raises(RuntimeError) as e:
job.trials
assert e.match(
"HyperparameterTuningJob resource has not been created. Resource failed with: Mock fail"
)
def test_create_hyperparameter_tuning_job_with_enable_web_access(
self,
create_hyperparameter_tuning_job_mock_with_enable_web_access,
get_hyperparameter_tuning_job_mock_with_enable_web_access,
sync,
caplog,
):
caplog.set_level(logging.INFO)
aiplatform.init(
project=_TEST_PROJECT,
location=_TEST_LOCATION,
staging_bucket=_TEST_STAGING_BUCKET,
encryption_spec_key_name=_TEST_DEFAULT_ENCRYPTION_KEY_NAME,
)
custom_job = aiplatform.CustomJob(
display_name=test_custom_job._TEST_DISPLAY_NAME,
worker_pool_specs=test_custom_job._TEST_WORKER_POOL_SPEC,
base_output_dir=test_custom_job._TEST_BASE_OUTPUT_DIR,
)
job = aiplatform.HyperparameterTuningJob(
display_name=_TEST_DISPLAY_NAME,
custom_job=custom_job,
metric_spec={_TEST_METRIC_SPEC_KEY: _TEST_METRIC_SPEC_VALUE},
parameter_spec={
"lr":
hpt.DoubleParameterSpec(min=0.001, max=0.1, scale="log"),
"units":
hpt.IntegerParameterSpec(min=4, max=1028, scale="linear"),
"activation":
hpt.CategoricalParameterSpec(
values=["relu", "sigmoid", "elu", "selu", "tanh"]),
"batch_size":
hpt.DiscreteParameterSpec(values=[16, 32], scale="linear"),
},
parallel_trial_count=_TEST_PARALLEL_TRIAL_COUNT,
max_trial_count=_TEST_MAX_TRIAL_COUNT,
max_failed_trial_count=_TEST_MAX_FAILED_TRIAL_COUNT,
search_algorithm=_TEST_SEARCH_ALGORITHM,
measurement_selection=_TEST_MEASUREMENT_SELECTION,
labels=_TEST_LABELS,
)
job.run(
service_account=_TEST_SERVICE_ACCOUNT,
network=_TEST_NETWORK,
timeout=_TEST_TIMEOUT,
restart_job_on_worker_restart=_TEST_RESTART_JOB_ON_WORKER_RESTART,
enable_web_access=test_custom_job._TEST_ENABLE_WEB_ACCESS,
sync=sync,
create_request_timeout=None,
)
job.wait()
assert "workerpool0-0" in caplog.text
expected_hyperparameter_tuning_job = (
_get_hyperparameter_tuning_job_proto_with_enable_web_access())
create_hyperparameter_tuning_job_mock_with_enable_web_access.assert_called_once_with(
parent=_TEST_PARENT,
hyperparameter_tuning_job=expected_hyperparameter_tuning_job,
timeout=None,
)
assert job.state == gca_job_state_compat.JobState.JOB_STATE_SUCCEEDED
assert job.network == _TEST_NETWORK
assert job.trials == []
caplog.clear()
