def test_create_or_load_study_with_409_raises_RuntimeError(
self, mock_discovery):
"""Verify that get_study gracefully handles 409 errors."""
mock_request = mock.MagicMock()
mock_request.execute.side_effect = errors.HttpError(
httplib2.Response(info={"status": 409}), b"")
mock_create_study = mock.MagicMock()
mock_create_study.return_value = mock_request
mock_discovery.build_from_document.return_value.projects().locations(
).studies().create = mock_create_study
mock_get_study = mock.MagicMock()
mock_get_study.side_effect = [
errors.HttpError(httplib2.Response(info={"status": 400}), b"")
] * 3
mock_discovery.build_from_document.return_value.projects().locations(
).studies().get = mock_get_study
with self.assertRaisesRegex(
RuntimeError,
'GetStudy failed. Max retries reached: <HttpError 400 "Ok">',
):
vizier_client.create_or_load_study(
project_id=self._project_id,
region=self._region,
study_id=self._study_id,
study_config=self._study_config,
)
mock_get_study.assert_called_with(
name="projects/{}/locations/{}/studies/{}".format(
self._project_id, self._region, self._study_id))
def test_create_or_load_study_with_409_success(self, mock_discovery):
"""Verify that get_study gracefully handles 409 errors."""
mock_create_request = mock.MagicMock()
mock_create_request.execute.side_effect = errors.HttpError(
httplib2.Response(info={"status": 409}), b"")
mock_create_study = mock.MagicMock()
mock_create_study.return_value = mock_create_request
mock_discovery.build_from_document.return_value.projects().locations(
).studies().create = (mock_create_study)
mock_get_request = mock.MagicMock()
mock_get_request.execute.side_effect = [
errors.HttpError(httplib2.Response(info={"status": 400}), b""),
errors.HttpError(httplib2.Response(info={"status": 400}), b""),
mock.DEFAULT,
]
mock_get_study = mock.MagicMock()
mock_get_study.side_effect = mock_get_request
mock_discovery.build_from_document.return_value.projects().locations(
).studies().get = (mock_get_study)
client = optimizer_client.create_or_load_study(
project_id=self._project_id,
region=self._region,
study_id=self._study_id,
study_config=self._study_config,
)
self.assertIsInstance(client, optimizer_client._OptimizerClient)
def test_create_or_load_study_with_409_raises_RuntimeError(
self, mock_discovery):
"""Verify that get_study gracefully handles 409 errors."""
mock_request = mock.MagicMock()
mock_request.execute.side_effect = errors.HttpError(
httplib2.Response(info={"status": 409}), b"")
mock_create_study = mock.MagicMock()
mock_create_study.return_value = mock_request
mock_discovery.build_from_document.return_value.projects().locations(
).studies().create = (mock_create_study)
mock_get_study = mock.MagicMock()
mock_get_study.side_effect = [
errors.HttpError(httplib2.Response(info={"status": 400}), b"")
] * 3
mock_discovery.build_from_document.return_value.projects().locations(
).studies().get = (mock_get_study)
with self.assertRaisesRegex(
RuntimeError,
'GetStudy wasn\'t successful after 3 tries: <HttpError 400 "Ok">',
):
optimizer_client.create_or_load_study(
project_id=self._project_id,
region=self._region,
study_id=self._study_id,
study_config=self._study_config,
)
def testRetry500(self):
sequence = HttpMockSequence([
('discovery/v1/apis/storage/v1/rest', {
'status': '200'
}, self.DISCOVERY),
('storage/v1/b/makani/o?prefix=misc&alt=json',
gapi_errors.HttpError(
httplib2.Response({'status': '500'}),
'{"code": 500, "message": "Backend Error"}'), None),
('storage/v1/b/makani/o?prefix=misc&alt=json',
gapi_errors.HttpError(
httplib2.Response({'status': '501'}),
'{"code": 500, "message": "Backend Error"}'), None),
('storage/v1/b/makani/o?prefix=misc&alt=json', {
'status': '200'
}, json.dumps({'items': [{
'name': 'a.txt'
}]})),
])
mock_sleep = mock.MagicMock()
gcs = self.GetService(sequence)
with mock.patch('time.sleep', mock_sleep), test_util.DisableWarnings():
names = gcs.List(self.Path('misc'))
self.assertEqual(2, mock_sleep.call_count)
self.assertEqual(['a.txt'], names)
def create(self, name, body):
if 'invalid' in body['accountId']:
return http_fake.HttpRequestFake(
errors.HttpError(http_fake.HttpResponseFake(400),
b'invalid account id'))
elif body['accountId'] in self.service_accounts:
return http_fake.HttpRequestFake(
errors.HttpError(http_fake.HttpResponseFake(409),
b'service account already exists'))
else:
self.service_accounts.append(body['accountId'])
return http_fake.HttpRequestFake({'name': name})
def test_request_with_retry_other_error_code(self):
err1 = errors.HttpError(
http_fake.HttpResponseFake(409), b'service account already exists')
err2 = errors.HttpError(
http_fake.HttpResponseFake(400), b'invalid request')
responses = [err1, err2, err1]
request = http_fake.HttpRequestFakeMultiple(responses)
with mock.patch(
__name__ + '.ProjectsFake.setIamPolicy', return_value=request):
with self.assertRaises(errors.HttpError) as cm:
self._service_account_client._update_iam_policy_with_retry(
PROJECT_ID, FAKE_SERVICE_ACCOUNT, [FAKE_ROLE])
exception = cm.exception
self.assertEqual(exception.resp.status, 400)
def insert(self, project, body):
bucket_name = body['name']
if project != PROJECT_ID:
return http_fake.HttpRequestFake(
errors.HttpError(http_fake.HttpResponseFake(403),
b'permission denied'))
elif 'invalid' in bucket_name:
return http_fake.HttpRequestFake({'invalid': 'response'})
elif bucket_name == EXISTING_BUCKET_NAME:
return http_fake.HttpRequestFake(
errors.HttpError(http_fake.HttpResponseFake(409),
b'permission denied'))
else:
self.buckets.append(bucket_name)
return http_fake.HttpRequestFake(body)
def test_create_or_load_study_no_study_config_with_404_raises_ValueError(
self, mock_discovery):
mock_request = mock.MagicMock()
mock_request.execute.side_effect = errors.HttpError(
httplib2.Response(info={"status": 404}), b""
)
mock_get_study = mock.MagicMock()
mock_get_study.return_value = mock_request
mock_discovery.build_from_document.return_value.projects().locations(
).studies().get = mock_get_study
with self.assertRaisesRegex(
ValueError,
"GetStudy failed. Study not found: {}.".format(self._study_id),
):
optimizer_client.create_or_load_study(
project_id=self._project_id,
region=self._region,
study_id=self._study_id,
)
mock_get_study.assert_called_with(
name="projects/{}/locations/{}/studies/{}".format(
self._project_id, self._region, self._study_id
)
)
def testRetryUpload(self):
bq_client = bigquery.BigQueryClient()
resp = mock.Mock()
resp.status = 503
error = mock.Mock()
error.resp = resp
job = mock.Mock()
# Always raise errors.HttpError on job.execute()
job.configure_mock(
**{"execute.side_effect": errors.HttpError(resp, "nocontent")})
job_id = "hunts_HFFE1D044_Results_1446056474"
with tempfile.NamedTemporaryFile() as fd:
fd.write("{data}")
with mock.patch.object(time, "sleep") as mock_sleep:
with self.assertRaises(bigquery.BigQueryJobUploadError):
bq_client.RetryUpload(job, job_id, error)
# Make sure retry sleeps are correct.
max_calls = config.CONFIG["BigQuery.retry_max_attempts"]
retry_interval = config.CONFIG["BigQuery.retry_interval"]
multiplier = config.CONFIG["BigQuery.retry_multiplier"]
self.assertEqual(job.execute.call_count, max_calls)
mock_sleep.assert_has_calls([
mock.call(retry_interval),
mock.call(retry_interval * multiplier)
])
def test_enforce_policy_firewall_enforcer_deleted_400(self):
"""Verifies that a deleted project returns a status=PROJECT_DELETED.
Setup:
* Switch the ListFirewalls response to be a 400 error with the reason
string set to unknown project.
Expected Result:
A ProjectResult proto showing status=PROJECT_DELETED and the correct
reason string.
"""
deleted_400 = httplib2.Response({
'status': '400',
'content-type': 'application/json'
})
deleted_400.reason = 'Invalid value for project: %s' % self.project
error_deleted_400 = errors.HttpError(deleted_400, ''.encode(), uri='')
err = api_errors.ApiExecutionError(self.project, error_deleted_400)
self.gce_api_client.get_firewall_rules.side_effect = err
result = self.enforcer.enforce_firewall_policy(self.policy)
self.expected_proto.status = project_enforcer.STATUS_DELETED
# Match first part of error reason string
self.assertStartsWith(result.status_reason,
'Project scheduled for deletion')
# Copy reason string into expected proto. The reason includes a long
# error message, which would be ugly to replicate in the test.
self.expected_proto.status_reason = result.status_reason
self.validate_results(self.expected_proto, result)
def testAddUsersGetWithError(self, mock_gds, mock_get_saved_credentials,
mock_response):
"""Test add users get fails gracefully when a resource isn't found.
We need to catch errors from the google directory service module since we
have not yet implemented robust error handling. Here I'm simulating an
exception in the directory service and asserting that we catch it and still
render the add_user page along with the error rather than barfing
completely.
"""
mock_get_saved_credentials.return_value = FAKE_CREDENTIAL
fake_status = '404'
fake_response = MagicMock(status=fake_status)
fake_content = b'some error content'
fake_error = errors.HttpError(fake_response, fake_content)
mock_gds.side_effect = fake_error
mock_response.return_value = ''
response = self.client.get(flask.url_for('add_user'))
args, kwargs = mock_response.call_args
json_output = json.loads(args[0][len(ufo.XSSI_PREFIX):])
self.assertEquals([], json_output['directory_users'])
self.assertEquals(str(fake_error), json_output['error'])
self.assertEquals(ufo.JSON_HEADERS, kwargs['headers'])
def test_run_process_should_record_that_all_items_failed_when_content_api_call_returns_error(
self):
dummy_http_error = errors.HttpError(
mock.MagicMock(status=http.HTTPStatus.BAD_REQUEST,
reason='Bad Request'), b'')
self.mock_content_api_client.return_value.process_items.side_effect = dummy_http_error
dummy_failures = [
failure.Failure(str(item.get('item_id', 'Missing ID')),
dummy_http_error.resp.reason)
for item in DUMMY_ROWS
]
expected_result = process_result.ProcessResult([], dummy_failures, [])
expected_batch_id = int(DUMMY_START_INDEX / DUMMY_BATCH_SIZE) + 1
self.mock_bq_client.from_service_account_json.return_value.load_items.return_value = DUMMY_ROWS
self.test_client.post(INSERT_URL,
data=DUMMY_REQUEST_BODY,
headers={'X-AppEngine-TaskExecutionCount': '0'})
self.mock_recorder.from_service_account_json.return_value.insert_result.assert_called_once_with(
constants.Operation.UPSERT.value,
expected_result,
DUMMY_TIMESTAMP,
expected_batch_id,
)
def get_tabledata(self,
dataset_id,
table_id,
max_results,
page_token=None,
start_index=None,
selected_fields=None,
return_none=False):
"""Mock method of BigQueryCursor.get_tabledata()."""
if return_none:
return None
self.dataset_id = dataset_id
self.table_id = table_id
self.max_results = max_results
self.page_token = page_token
self.start_index = start_index
self.selected_fields = selected_fields
if table_id.startswith('error') and start_index == 1:
response = mock.Mock()
response.reason = 'test_reason'
raise googleapiclient_errors.HttpError(resp=response,
content=b'test')
try:
return self.data_generator.get_data(start_index, max_results)
except IndexError:
return None
def test_verify_iam_roles_http_error(klio_config, mock_discovery_client):
compute_client = mock_discovery_client.build("compute")
err = google_errors.HttpError
resp = httplib2.Response({})
resp.reason = "some resp"
compute_client.projects().get().execute.side_effect = err(
resp, "some content".encode())
iam_client = mock_discovery_client.build("cloudresourcemanager")
job = verify.VerifyJob(klio_config, False)
job._compute_client = compute_client
job._iam_client = iam_client
result = job._verify_iam_roles()
assert result is False
compute_client = mock_discovery_client.build("compute")
compute_client.projects().get().execute.side_effect = None
compute_client.projects().get().execute.return_value = {
"defaultServiceAccount": "the-default-svc-account"
}
iam_client = mock_discovery_client.build("cloudresourcemanager")
iam_client.projects().getIamPolicy(
resource=job.klio_config.pipeline_options.project,
body={}).execute.side_effect = google_errors.HttpError(
resp, "some content".encode())
job._compute_client = compute_client
job._iam_client = iam_client
result = job._verify_iam_roles()
assert result is False
def test_is_status_not_found_404(self):
response = httplib2.Response({
'status': '404',
'content-type': 'application/json'})
response.reason = 'Not Found'
error = errors.HttpError(response, fae.APP_NOT_FOUND.encode(), uri='')
self.assertTrue(ae._is_status_not_found(error))
def testRetryUpload(self):
bq_client = bigquery.BigQueryClient()
resp = mock.Mock()
resp.status = 503
error = mock.Mock()
error.resp = resp
job = mock.Mock()
# Always raise errors.HttpError on job.execute()
job.configure_mock(
**{"execute.side_effect": errors.HttpError(resp, b"nocontent")})
job_id = "hunts_HFFE1D044_Results_1446056474"
with temp.AutoTempFilePath() as filepath:
with io.open(filepath, "w", encoding="utf-8") as filedesc:
filedesc.write("{data}")
with mock.patch.object(time, "sleep") as mock_sleep:
with self.assertRaises(bigquery.BigQueryJobUploadError):
bq_client.RetryUpload(job, job_id, error)
# Make sure retry sleeps are correct.
max_calls = config.CONFIG["BigQuery.retry_max_attempts"]
retry_interval = config.CONFIG["BigQuery.retry_interval"]
multiplier = config.CONFIG["BigQuery.retry_multiplier"]
self.assertEqual(job.execute.call_count, max_calls)
mock_sleep.assert_has_calls([
mock.call(retry_interval.ToFractional(rdfvalue.SECONDS)),
mock.call(
retry_interval.ToFractional(rdfvalue.SECONDS) * multiplier)
])
def testCreateInstance(self):
instance_name = 'foo_instance'
sequence = HttpMockSequence([
('discovery/v1/apis/compute/v1/rest', {
'status': '200'
}, self.DISCOVERY),
('compute/v1/projects/google.com%3Amakani/zones/outer_space/'
'instances/' + instance_name + '?alt=json',
gapi_errors.HttpError(httplib2.Response({'status': '404'}),
('{"code": 404, "message": "The resource ' +
instance_name + ' was not found."')),
'{}'),
('compute/v1/projects/google.com%3Amakani/zones/outer_space/'
'instances?alt=json', {
'status': '200'
}, '{}'),
])
gce = self.GetService(sequence)
gce.CreateInstance('foo_instance',
'foo_image',
image_project='some_project',
zone='outer_space',
machine_type='super_awesome',
network='some_network',
scopes=['foo', 'bar'],
metadata=[{
'key': 'color',
'value': 'red'
}])
def testDeployModelForVertexPredictionError(self):
self._setUpVertexPredictionMocks()
self._mock_endpoint_list.side_effect = [[], [self._mock_endpoint]]
self._mock_model_deploy.side_effect = errors.HttpError(
httplib2.Response(info={'status': 429}), b'')
with self.assertRaises(RuntimeError):
runner.deploy_model_for_aip_prediction(
serving_path=self._serving_path,
model_version_name=self._model_name,
ai_platform_serving_args=self._ai_platform_serving_args_vertex,
labels=self._job_labels,
serving_container_image_uri=self._serving_container_image_uri,
endpoint_region=self._endpoint_region,
enable_vertex=True)
expected_endpoint_create_body = {
'display_name': self._endpoint_name,
'labels': self._job_labels,
}
expected_model_upload_body = {
'display_name': self._model_name,
'artifact_uri': self._serving_path,
'serving_container_image_uri': self._serving_container_image_uri,
}
expected_model_deploy_body = {
'endpoint': self._mock_endpoint,
'traffic_percentage': 100,
}
self._assertDeployModelMockCallsVertex(
expected_endpoint_create_body=expected_endpoint_create_body,
expected_model_upload_body=expected_model_upload_body,
expected_model_deploy_body=expected_model_deploy_body)
def setUp(self):
"""Creates mock objects for googleapi client."""
super(CloudComposerUtilsTest, self).setUp()
self.addCleanup(mock.patch.stopall)
self.project_id = 'project_id'
self.location = cloud_composer._LOCATION
self.environment_name = 'environment_name'
self.zone = 'a'
self.mock_build_service_client = mock.patch.object(
cloud_auth, 'build_service_client', autospec=True).start()
self.mock_wait_for_operation = mock.patch.object(
utils, 'wait_for_operation', autospec=True).start()
self.mock_execute_request = mock.patch.object(utils,
'execute_request',
autospec=True).start()
self.mock_client = mock.Mock()
self.mock_build_service_client.return_value = self.mock_client
self.service_account_key_file = '/tmp/service_account_key.json'
self.composer = cloud_composer.CloudComposerUtils(
self.project_id, self.location, self.service_account_key_file)
self.operation_client = mock.Mock()
self.operation = {}
(self.mock_client.projects.return_value.locations.return_value.
operations.return_value) = self.operation_client
self.mock_execute_request.return_value = self.operation
self.http_error = errors.HttpError(mock.MagicMock(status=400), b'')
self.fully_qualified_name = (f'projects/{self.project_id}/locations/'
f'{self.location}/environments/'
f'{self.environment_name}')
self.mock_environment_client = (self.mock_client.projects.return_value.
locations.return_value.environments)
self.mock_request = mock.Mock(http.HttpRequest)
def get(self, projectId):
for p in self.projects:
if p['projectId'] == projectId:
return http_fake.HttpRequestFake(p)
return http_fake.HttpRequestFake(
errors.HttpError(http_fake.HttpResponseFake(403),
b'permission denied'))
def testCreateInstance_FailIfAlreadyExists(self):
instance_name = 'foo_instance'
zone = 'outer_space'
sequence = HttpMockSequence([
('discovery/v1/apis/compute/v1/rest', {
'status': '200'
}, self.DISCOVERY),
('compute/v1/projects/google.com%3Amakani/zones/' + zone + '/'
'instances?alt=json',
gapi_errors.HttpError(httplib2.Response({'status': '409'}),
('{"code": 409, "message": "The resource ' +
instance_name + ' already exists."')),
'{}')
])
gce = self.GetService(sequence)
with self.assertRaises(gapi_errors.HttpError):
gce.CreateInstance(instance_name,
'foo_image',
image_project='some_project',
zone=zone,
machine_type='super_awesome',
network='some_network',
scopes=['foo', 'bar'],
metadata=[{
'key': 'color',
'value': 'red'
}])
def test_is_status_not_found_403(self):
response = httplib2.Response({
'status': '403',
'content-type': 'application/json'})
response.reason = 'Permission Denied'
error = errors.HttpError(response, fae.PERMISSION_DENIED.encode(), uri='')
self.assertFalse(ae._is_status_not_found(error))
def testCreateInstance_BadGatewayAlreadyExistsIssue(self):
# In this test case, the create instance call initially fails with a 500
# bad gateway error, but the instance does get created in Google's server,
# therefore the retry call fails with an already exists error.
# For more information, see b/136171159
instance_name = 'foo_instance'
zone = 'outer_space'
sequence = HttpMockSequence([
('discovery/v1/apis/compute/v1/rest', {
'status': '200'
}, self.DISCOVERY),
('compute/v1/projects/google.com%3Amakani/zones/outer_space/'
'instances/' + instance_name + '?alt=json',
gapi_errors.HttpError(httplib2.Response({'status': '404'}),
('{"code": 404, "message": "The resource ' +
instance_name + ' was not found."')),
'{}'),
('compute/v1/projects/google.com%3Amakani/zones/' + zone + '/'
'instances?alt=json',
gapi_errors.HttpError(httplib2.Response({'status': '502'}),
('{"code": 502, "message": "Bad Gateway"')),
'{}'),
('compute/v1/projects/google.com%3Amakani/zones/' + zone + '/'
'instances?alt=json',
gapi_errors.HttpError(httplib2.Response({'status': '409'}),
('{"code": 409, "message": "The resource ' +
instance_name + ' already exists."')),
'{}'),
('compute/v1/projects/google.com%3Amakani/zones/' + zone + '/'
'instances/' + instance_name + '?alt=json', {
'status': '200'
}, '{}')
])
gce = self.GetService(sequence)
gce.CreateInstance(instance_name,
'foo_image',
image_project='some_project',
zone=zone,
machine_type='super_awesome',
network='some_network',
scopes=['foo', 'bar'],
metadata=[{
'key': 'color',
'value': 'red'
}])
def _mock_gce_get_project(projectid):
if projectid in results.GCE_GET_PROJECT:
return results.GCE_GET_PROJECT[projectid]
response = httplib2.Response(
{'status': '403', 'content-type': 'application/json'})
content = results.GCE_API_NOT_ENABLED_TEMPLATE.format(id=projectid)
error_403 = errors.HttpError(response, content)
raise api_errors.ApiNotEnabledError('Access Not Configured.', error_403)
def test_delete_cluster_ignore_not_found(self, mock_client, mock_context):
mock_context().__enter__().context_id.return_value = 'ctx1'
mock_client().delete_cluster.side_effect = errors.HttpError(
resp = mock.Mock(status=404),
content = b'not found'
)
delete_cluster('mock-project', 'mock-region', 'mock-cluster')
def create(self, name, body):
if 'invalid' in name:
return http_fake.HttpRequestFake(
errors.HttpError(
http_fake.HttpResponseFake(400), b'invalid resource name'))
else:
self.key_count += 1
return http_fake.HttpRequestFake(FAKE_CREATE_KEY_RESPONSE)
def test_stop_aip_training_job_with_completed_job(self):
self.mock_request.execute.side_effect = errors.HttpError(
httplib2.Response(info={"status": 400}), b"")
google_api_client.stop_aip_training_job(self._job_id, self._project_id)
job_name = "projects/{}/jobs/{}".format(self._project_id, self._job_id)
self.mock_apiclient.projects().jobs().cancel.assert_called_with(
name=job_name)
def test_admin_api_create__creation_error(self):
"""Test the App Engine project creation for an invalid project."""
test_app_engine_admin_api = app_engine.AdminAPI(self.config, mock.Mock())
test_app_engine_admin_api._client.apps.side_effect = errors.HttpError(
httplib2.Response({
'reason': 'Not found.', 'status': httplib.NOT_FOUND}),
'Project not found.')
with self.assertRaises(app_engine.CreationError):
test_app_engine_admin_api.create('us-east1')
def test_execute_request_retries_on_service_unavailable_http_error(self):
mock_request = mock.Mock(http.HttpRequest)
content = b''
error = errors.HttpError(mock.MagicMock(status=503), content)
mock_request.execute.side_effect = [error, None]
utils.execute_request(mock_request)
self.assertEqual(mock_request.execute.call_count, 2)
def test_admin_api_get__not_found_error(self):
"""Test the get API method for an invalid project."""
test_app_engine_admin_api = app_engine.AdminAPI(self.config, mock.Mock())
test_app_engine_admin_api._client.apps.side_effect = errors.HttpError(
httplib2.Response({
'reason': 'Not found.', 'status': httplib.NOT_FOUND}),
'App not found.')
with self.assertRaises(app_engine.NotFoundError):
test_app_engine_admin_api.get()