def test_it_encapsulates_gmail_exceptions(self, mocker):
server_error_response = mocker.MagicMock(status=500)
mocker.patch(
"gmail_wrapper.client.GmailClient._make_client",
return_value=make_gmail_client(mocker,
list_effect=HttpError(
server_error_response,
b"Content")),
)
client = GmailClient(email="*****@*****.**", secrets_json_string="{}")
with pytest.raises(GmailError):
client.get_raw_messages()
def test_add_label_and_retry_no_descriptor(self):
timeseries = 'OPAQUE'
status = ResponseStatus(404, 'Not Found')
self.mockMetricDescriptors.get.side_effect = HttpError(
status, 'Not Found')
self.service.add_label_and_retry('NewLabel', 'ExistingType', timeseries)
self.assertEquals(0, self.mockStub.projects.list.call_count)
self.assertEquals(1, self.mockMetricDescriptors.get.call_count)
self.assertEquals(0, self.mockMetricDescriptors.delete.call_count)
self.assertEquals(0, self.mockMetricDescriptors.create.call_count)
self.assertEquals(0, self.mockTimeSeries.create.call_count)
def generate_response(resp, content):
try:
response_content = json.loads(content.decode('utf-8'))
result = {
'url': response_content['urlNotificationMetadata']['url'],
'status': resp['status'],
'message': 'OK'
}
except Exception as e:
resp.reason = "Something went wrong while processing successful response. Original response %s."\
"Exception: %s " % (resp.reason, str(e))
raise HttpError(resp.reason, content)
return result
def test_acknowledge_fails_on_exception(self, mock_service):
ack_method = (mock_service.return_value.projects.return_value.
subscriptions.return_value.acknowledge)
ack_method.return_value.execute.side_effect = HttpError(
resp={'status': '404'}, content=EMPTY_CONTENT)
with self.assertRaises(Exception) as e:
self.pubsub_hook.acknowledge(TEST_PROJECT, TEST_SUBSCRIPTION,
['1', '2', '3'])
ack_method.assert_called_once_with(
subscription=EXPANDED_SUBSCRIPTION,
body={'ackIds': ['1', '2', '3']})
print(e)
def test_get_dataset_access_raises(self):
"""Test get_dataset_access raises when there is an HTTP exception."""
mock_bq_stub = mock.MagicMock()
self.bq_api_client.service = mock.MagicMock()
self.bq_api_client.service.datasets.return_value = mock_bq_stub
self.bq_api_client._execute = mock.MagicMock(
side_effect=HttpError(self.http_response, '{}')
)
with self.assertRaises(api_errors.ApiExecutionError):
self.bq_api_client.get_dataset_access(fbq.PROJECT_IDS[0],
fbq.DATASET_ID)
def test_pull_fails_on_exception(self, mock_service):
pull_method = (mock_service.return_value.projects.return_value.
subscriptions.return_value.pull)
pull_method.return_value.execute.side_effect = HttpError(
resp={'status': '404'}, content=EMPTY_CONTENT)
with self.assertRaises(Exception):
self.pubsub_hook.pull(TEST_PROJECT, TEST_SUBSCRIPTION, 10)
pull_method.assert_called_with(subscription=EXPANDED_SUBSCRIPTION,
body={
'maxMessages': 10,
'returnImmediately': False
})
def test_gcf_error_silenced_when_function_doesnt_exist(self, mock_hook):
op = GcfFunctionDeleteOperator(
name=self._FUNCTION_NAME,
task_id="id"
)
mock_hook.return_value.delete_function.side_effect = mock.Mock(
side_effect=HttpError(resp=MOCK_RESP_404, content=b'not found'))
op.execute(None)
mock_hook.assert_called_once_with(api_version='v1',
gcp_conn_id='google_cloud_default')
mock_hook.return_value.delete_function.assert_called_once_with(
'projects/project_name/locations/project_location/functions/function_name'
)
def test_it_encapsulates_gmail_exceptions(self, mocker, error_code,
exception_expected):
error_response = mocker.MagicMock(status=error_code)
mocker.patch(
"gmail_wrapper.client.GmailClient._make_client",
return_value=make_gmail_client(mocker,
get_effect=HttpError(
error_response, b"Content")),
)
client = GmailClient(email="*****@*****.**", secrets_json_string="{}")
with pytest.raises(exception_expected):
client.get_raw_message("123AAB")
def test_successful_copy_template_with_smaller_array_fields(self, mock_hook):
mock_hook.return_value.get_instance_template.side_effect = [
HttpError(resp=httplib2.Response({'status': 404}), content=EMPTY_CONTENT),
GCE_INSTANCE_TEMPLATE_BODY_GET,
GCE_INSTANCE_TEMPLATE_BODY_GET_NEW
]
op = ComputeEngineCopyInstanceTemplateOperator(
project_id=GCP_PROJECT_ID,
resource_id=GCE_INSTANCE_TEMPLATE_NAME,
task_id='id',
body_patch={
"name": GCE_INSTANCE_TEMPLATE_NEW_NAME,
"properties": {
"machineType": "n1-standard-1",
"networkInterfaces": [
{
"network": "https://www.googleapis.com/compute/v1/"
"projects/project/global/networks/default",
"accessConfigs": [
{
"type": "ONE_TO_ONE_NAT",
"natIP": "8.8.8.8"
}
]
}
]
}
}
)
result = op.execute(None)
mock_hook.assert_called_once_with(api_version='v1',
gcp_conn_id='google_cloud_default')
body_insert = deepcopy(GCE_INSTANCE_TEMPLATE_BODY_INSERT)
body_insert["properties"]["networkInterfaces"] = [
{
"network": "https://www.googleapis.com/compute/v1/"
"projects/project/global/networks/default",
"accessConfigs": [
{
"type": "ONE_TO_ONE_NAT",
"natIP": "8.8.8.8"
}
]
}
]
mock_hook.return_value.insert_instance_template.assert_called_once_with(
project_id=GCP_PROJECT_ID,
body=body_insert,
request_id=None
)
self.assertEqual(GCE_INSTANCE_TEMPLATE_BODY_GET_NEW, result)
def _retrieve_discovery_doc(url, http, cache_discovery, cache=None):
"""Retrieves the discovery_doc from cache or the internet.
Args:
url: string, the URL of the discovery document.
http: httplib2.Http, An instance of httplib2.Http or something that acts
like it through which HTTP requests will be made.
cache_discovery: Boolean, whether or not to cache the discovery doc.
cache: googleapiclient.discovery_cache.base.Cache, an optional cache
object for the discovery documents.
Returns:
A unicode string representation of the discovery document.
"""
if cache_discovery:
from . import discovery_cache
from .discovery_cache import base
if cache is None:
cache = discovery_cache.autodetect()
if cache:
content = cache.get(url)
if content:
return content
actual_url = url
# REMOTE_ADDR is defined by the CGI spec [RFC3875] as the environment
# variable that contains the network address of the client sending the
# request. If it exists then add that to the request for the discovery
# document to avoid exceeding the quota on discovery requests.
if 'REMOTE_ADDR' in os.environ:
actual_url = _add_query_parameter(url, 'userIp', os.environ['REMOTE_ADDR'])
logger.info('URL being requested: GET %s', actual_url)
resp, content = http.request(actual_url)
if resp.status >= 400:
raise HttpError(resp, content, uri=actual_url)
try:
content = content.decode('utf-8')
except AttributeError:
pass
try:
service = json.loads(content)
except ValueError as e:
logger.error('Failed to parse as JSON: ' + content)
raise InvalidJsonError()
if cache_discovery and cache:
cache.set(url, content)
return content
def test_create_group(self, logger_mock):
with self.subTest("Successful"):
self.sync_service.create_group(
GSuiteSyncService.GroupData(
"new_group",
"some description",
False,
["alias2"],
[f"test2@{settings.GSUITE_DOMAIN}"],
))
self.directory_api.groups().insert.assert_called_once_with(
body={
"email": f"new_group@{settings.GSUITE_DOMAIN}",
"name": "new_group",
"description": "some description",
})
self.settings_api.groups().update.assert_called_once_with(
groupUniqueId=f"new_group@{settings.GSUITE_DOMAIN}",
body=self.sync_service._group_settings(False),
)
self.directory_api.members().list.assert_called()
self.directory_api.groups().aliases().list.assert_called()
self.settings_api.reset_mock()
self.directory_api.reset_mock()
with self.subTest("Failure"):
self.directory_api.groups().insert(
).execute.side_effect = HttpError(Response({"status": 500}),
bytes())
self.sync_service.create_group(
GSuiteSyncService.GroupData(
"new_group",
"some description",
False,
["alias2"],
[f"test2@{settings.GSUITE_DOMAIN}"],
))
self.directory_api.members().list.assert_not_called()
self.directory_api.groups().aliases().list.assert_not_called()
logger_mock.error.assert_called_once_with(
"Could not successfully finish creating the list %s: %s",
"new_group",
bytes(),
)
def test_json_body(self):
"""Test a nicely formed, expected error response."""
resp, content = fake_response(
JSON_ERROR_CONTENT,
{"status": "400", "content-type": "application/json"},
reason="Failed",
)
error = HttpError(resp, content, uri="http://example.org")
self.assertEqual(error.error_details, "error details")
self.assertEqual(error.status_code, 400)
self.assertEqual(
str(error),
'<HttpError 400 when requesting http://example.org returned "country is required". Details: "error details">',
)
def test_delete_key(self, mock_iam_service, mock_logger):
key = {'name': 'key-1'}
mock_service_account_key_delete = mock_iam_service.return_value.projects.return_value. \
serviceAccounts.return_value.keys.return_value.delete
mock_service_account_key_delete.return_value.execute.side_effect = [
dict(), HttpError(mock.Mock(status=404), b'not found')
]
key_rotation.delete_key(key)
mock_service_account_key_delete.assert_called_once_with(name='key-1')
key_rotation.delete_key(key)
mock_logger.info.aasert_called_with(
'{full_key_name} is deleted'.format(full_key_name='key-1'))
def test_get_bigquery_projectids_raises(self):
"""Test that get_bigquery_projectids raises when there is an HTTP
exception.
"""
mock_bq_stub = mock.MagicMock()
self.bq_api_client.service = mock.MagicMock()
self.bq_api_client.service.projects.return_value = mock_bq_stub
self.bq_api_client._execute = mock.MagicMock(
side_effect=HttpError(self.http_response, '{}')
)
with self.assertRaises(api_errors.ApiExecutionError):
self.bq_api_client.get_bigquery_projectids()
def test_update_youtube_statuses_api_quota_exceeded(
mocker, youtube_video_files_processing
):
"""
Test that the update_youtube_statuses task stops without raising an error if the API quota is exceeded.
"""
mock_video_status = mocker.patch(
"videos.tasks.YouTubeApi.video_status",
side_effect=HttpError(
MockHttpErrorResponse(403), str.encode(API_QUOTA_ERROR_MSG, "utf-8")
),
)
update_youtube_statuses()
mock_video_status.assert_called_once()
def test_google_upload_apk_errors_out(edit_resource_mock, http_status_code):
edit_resource_mock.apks().upload().execute.side_effect = HttpError(
# XXX status is presented as a string by googleapiclient
resp={'status': str(http_status_code)},
# XXX content must be bytes
# https://github.com/googleapis/google-api-python-client/blob/ffea1a7fe9d381d23ab59048263c631cc2b45323/googleapiclient/errors.py#L41
content=b'{"error": {"errors": [{"reason": "someRandomReason"}] } }',
)
google_play = GooglePlayEdit(edit_resource_mock, 1, 'dummy_package_name')
with pytest.raises(HttpError):
apk_mock = Mock()
apk_mock.name = '/path/to/dummy.apk'
google_play.upload_apk(apk_mock)
def test_create_subscription_failifexists(self, mock_service):
(mock_service.return_value.projects.return_value.subscriptions.
return_value.create.return_value.execute.side_effect) = HttpError(
resp={'status': '409'}, content=EMPTY_CONTENT)
with self.assertRaises(PubSubException) as e:
self.pubsub_hook.create_subscription(TEST_PROJECT,
TEST_TOPIC,
TEST_SUBSCRIPTION,
fail_if_exists=True)
self.assertEqual(
str(e.exception),
'Subscription already exists: %s' % EXPANDED_SUBSCRIPTION)
def test_delete_nonexisting_subscription_failifnotexists(
self, mock_service):
(mock_service.return_value.projects.return_value.subscriptions.
return_value.delete.return_value.execute.side_effect) = HttpError(
resp={'status': '404'}, content=EMPTY_CONTENT)
with self.assertRaises(PubSubException) as e:
self.pubsub_hook.delete_subscription(TEST_PROJECT,
TEST_SUBSCRIPTION,
fail_if_not_exists=True)
self.assertEqual(
str(e.exception),
'Subscription does not exist: %s' % EXPANDED_SUBSCRIPTION)
def execute(self, num_retries=0):
"""Get the next chunk of the download.
Args:
num_retries: Integer, number of times to retry 500's with randomized
exponential backoff. If all retries fail, the raised HttpError
represents the last request. If zero (default), we attempt the
request only once.
Returns:
(status, done): (MediaDownloadStatus, boolean)
The value of 'done' will be True when the media has been fully
downloaded.
Raises:
googleapiclient.errors.HttpError if the response was not a 2xx.
httplib2.HttpLib2Error if a transport error has occured.
"""
headers = {}
http = self._request.http
for retry_num in range(num_retries + 1):
if retry_num > 0:
self._sleep(self._rand() * 2**retry_num)
logging.warning(
'Retry #%d for media download: GET %s, following status: %d'
% (retry_num, self._uri, resp.status))
func = timeit(http.request)
resp, content = func(self._uri, headers=headers)
if resp.status < 500:
break
if resp.status in [200, 206]:
if 'content-location' in resp and resp[
'content-location'] != self._uri:
self._uri = resp['content-location']
self._progress += len(content)
self._fd.write(content)
if 'content-range' in resp:
content_range = resp['content-range']
length = content_range.rsplit('/', 1)[1]
self._total_size = int(length)
elif 'content-length' in resp:
self._total_size = int(resp['content-length'])
return resp, content
else:
raise HttpError(resp, content, uri=self._uri)
def test_upload_gzip_error(self, mock_service):
test_bucket = 'test_bucket'
test_object = 'test_object'
(mock_service.return_value.objects.return_value.insert.return_value.
execute.side_effect) = HttpError(resp={'status': '404'},
content=EMPTY_CONTENT)
response = self.gcs_hook.upload(test_bucket,
test_object,
self.testfile.name,
gzip=True)
self.assertFalse(os.path.exists(self.testfile.name + '.gz'))
self.assertFalse(response)
def test_non_404_gcf_error_bubbled_up(self, mock_hook):
op = GcfFunctionDeleteOperator(name=self._FUNCTION_NAME, task_id="id")
resp = type('', (object, ), {"status": 500})()
mock_hook.return_value.delete_function.side_effect = mock.Mock(
side_effect=HttpError(resp=resp, content=b'error'))
with self.assertRaises(HttpError):
op.execute(None)
mock_hook.assert_called_once_with(api_version='v1',
gcp_conn_id='google_cloud_default')
mock_hook.return_value.delete_function.assert_called_once_with(
'projects/project_name/locations/project_location/functions/function_name'
)
def test_exceptions_message_not_found(self, mocker):
error_response = mocker.MagicMock(status=404)
mocker.patch(
"gmail_wrapper.client.GmailClient._make_client",
return_value=make_gmail_client(
mocker, get_effect=HttpError(error_response, b"Content")
),
)
client = GmailClient(email="*****@*****.**", secrets_json_string="{}")
message_id = "123AAB"
with pytest.raises(MessageNotFoundError) as error:
raise client.get_raw_message(message_id)
assert str(error.value) == f"MessageNotFoundError: Gmail returned 404 when attempting to get message {message_id}"
def test_delete_expired_google_access_with_one_fail_first(
cloud_manager, app, db_session):
"""
Test the case that there is a failure of removing from google group in GCP.
In this case, we still want the expired record to exist in the db so we can try to
remove it again.
"""
from googleapiclient.errors import HttpError
import fence
fence.settings = MagicMock()
cirrus.config.update = MagicMock()
cloud_manager.return_value.__enter__.return_value.remove_member_from_group.side_effect = [
HttpError(mock.Mock(status=403), bytes("Permission denied", "utf-8")),
{},
]
current_time = int(time.time())
# 1 expired, 2 not expired
access_1_expires = current_time - 3600
access_2_expires = current_time + 3600
_setup_google_access(db_session,
access_1_expires=access_1_expires,
access_2_expires=access_2_expires)
google_access = db_session.query(
GoogleProxyGroupToGoogleBucketAccessGroup).all()
google_proxy_groups = db_session.query(GoogleProxyGroup).all()
google_bucket_access_grps = db_session.query(GoogleBucketAccessGroup).all()
# check database to make sure all the service accounts exist
pre_deletion_google_access_size = len(google_access)
pre_deletion_google_proxy_groups_size = len(google_proxy_groups)
pre_deletion_google_bucket_access_grps_size = len(
google_bucket_access_grps)
# call function to delete expired service account
delete_expired_google_access(config["DB"])
google_access = db_session.query(
GoogleProxyGroupToGoogleBucketAccessGroup).all()
google_proxy_groups = db_session.query(GoogleProxyGroup).all()
google_bucket_access_grps = db_session.query(GoogleBucketAccessGroup).all()
# check database again. Expect nothing is deleted
assert len(google_access) == pre_deletion_google_access_size
assert len(google_proxy_groups) == pre_deletion_google_proxy_groups_size
assert len(google_bucket_access_grps
) == pre_deletion_google_bucket_access_grps_size
def resumable_progress(self):
if self._resumable_progress is None:
upload_range = "bytes */{}".format(self.media_body.size())
status, resp = self.service._http.request(self.resumable_uri,
method='PUT',
headers={
'Content-Length':
'0',
'Content-Range':
upload_range
})
if status['status'] not in ('200', '308'):
#Should 404 result in a FileNotFound error?
raise HttpError(status, resp)
self._range_md5 = hashlib.md5()
def file_in_chunks(start_byte: int,
end_byte: int,
chunksize: int = 4 * 1024**2):
while start_byte < end_byte:
content_length = min(chunksize, end_byte - start_byte)
yield self.media_body.getbytes(start_byte, content_length)
start_byte += content_length
if status['status'] == '200':
self._resumable_progress = self.media_body.size()
for chunk in file_in_chunks(0, self._resumable_progress):
self._range_md5.update(chunk)
elif 'range' in status.keys():
self._resumable_progress = int(status['range'].replace(
'bytes=0-', '', 1)) + 1
for chunk in file_in_chunks(0, self._resumable_progress):
self._range_md5.update(chunk)
logger.debug("Local MD5 (0-%d): %s", self._resumable_progress,
self._range_md5.hexdigest())
logger.debug("Remote MD5 (0-%d): %s", self._resumable_progress,
status['x-range-md5'])
if status['x-range-md5'] != self._range_md5.hexdigest():
raise CheckSumError(
"Checksum mismatch. Need to repeat upload.")
else:
self._resumable_progress = 0
return self._resumable_progress
def test_delete_expired_service_accounts_with_one_fail_first(
cloud_manager, app, db_session
):
"""
Test the case that there is a failure of removing service account from google group
"""
from googleapiclient.errors import HttpError
import fence
fence.settings = MagicMock()
cirrus.config.update = MagicMock()
cloud_manager.return_value.__enter__.return_value.remove_member_from_group.side_effect = [
HttpError(mock.Mock(status=403), bytes("Permission denied", "utf-8")),
{},
]
_setup_service_account_to_google_bucket_access_group(db_session)
service_accounts = db_session.query(UserServiceAccount).all()
google_bucket_access_grps = db_session.query(GoogleBucketAccessGroup).all()
current_time = int(time.time())
# Add expired service account. This acccount is supposed to be deleted
db_session.add(
ServiceAccountToGoogleBucketAccessGroup(
service_account_id=service_accounts[0].id,
expires=current_time - 3600,
access_group_id=google_bucket_access_grps[0].id,
)
)
# Add non-expired service account.
db_session.add(
ServiceAccountToGoogleBucketAccessGroup(
service_account_id=service_accounts[1].id,
expires=current_time + 3600,
access_group_id=google_bucket_access_grps[1].id,
)
)
db_session.commit()
# check database to make sure all the service accounts exist
records = db_session.query(ServiceAccountToGoogleBucketAccessGroup).all()
assert len(records) == 2
# call function to delete expired service account
delete_expired_service_accounts(config["DB"])
# check database again. Expect no service account is deleted
records = db_session.query(ServiceAccountToGoogleBucketAccessGroup).all()
assert len(records) == 2
def test_delete_expired_service_accounts_with_one_fail_second(
cloud_manager, app, db_session
):
"""
Test the case that there is a failure of removing service account from google group
"""
from googleapiclient.errors import HttpError
import fence
fence.settings = MagicMock()
cloud_manager.return_value.__enter__.return_value.remove_member_from_group.side_effect = [
{},
HttpError(mock.Mock(status=403), bytes("Permission denied", "utf-8")),
]
_setup_service_account_to_google_bucket_access_group(db_session)
service_accounts = db_session.query(UserServiceAccount).all()
google_bucket_access_grps = db_session.query(GoogleBucketAccessGroup).all()
current_time = int(time.time())
# Add two expired service account. They are both supposed to be deleted but
# only one is deleted due to a raise exception
service_account1 = ServiceAccountToGoogleBucketAccessGroup(
service_account_id=service_accounts[0].id,
expires=current_time - 3600,
access_group_id=google_bucket_access_grps[0].id,
)
service_account2 = ServiceAccountToGoogleBucketAccessGroup(
service_account_id=service_accounts[1].id,
expires=current_time - 3600,
access_group_id=google_bucket_access_grps[1].id,
)
db_session.add(service_account1)
db_session.add(service_account2)
db_session.commit()
# check database to make sure there are two records in DB
records = db_session.query(ServiceAccountToGoogleBucketAccessGroup).all()
assert len(records) == 2
# call function to delete expired service account
delete_expired_service_accounts(config["DB"])
# check database to make sure only the first one is deleted, the second one
# still exists because of the raised exception
records = db_session.query(ServiceAccountToGoogleBucketAccessGroup).all()
assert len(records) == 1
assert records[0].id == service_account2.id
def test_cancel_mlengine_job_nonexistent_job(self, mock_get_conn):
project_id = "test-project"
job_id = 'test-job-id'
job_cancelled = {}
error_job_does_not_exist = HttpError(resp=mock.MagicMock(status=404),
content=b'Job does not exist')
# fmt: off
(mock_get_conn.return_value.projects.return_value.jobs.return_value.
cancel.return_value.execute.side_effect) = error_job_does_not_exist
(mock_get_conn.return_value.projects.return_value.jobs.return_value.
cancel.return_value.execute.return_value) = job_cancelled
# fmt: on
with self.assertRaises(HttpError):
self.hook.cancel_job(job_id=job_id, project_id=project_id)
def test_empty_project_id_is_ok(self, mock_hook):
mock_hook.return_value.get_function.side_effect = \
HttpError(resp=MOCK_RESP_404, content=b'not found')
operator = CloudFunctionDeployFunctionOperator(
location="test_region", body=deepcopy(VALID_BODY), task_id="id")
operator.execute(None)
mock_hook.assert_called_once_with(api_version='v1',
gcp_conn_id='google_cloud_default')
new_body = deepcopy(VALID_BODY)
new_body['labels'] = {
'airflow-version':
'v' + version.replace('.', '-').replace('+', '-')
}
mock_hook.return_value.create_new_function.assert_called_once_with(
project_id=None, location="test_region", body=new_body)
def test_run_should_ignore_template_for_multivalued_fields_if_sheet_not_found(
self, mock_read_helper, mock_create_tag_template):
error_response = Response({'status': 400, 'reason': 'Not Found'})
mock_read_helper.side_effect = HttpError(resp=error_response,
content=b'{}')
TemplateMaker().run(spreadsheet_id=None,
project_id=None,
template_id='test-template-id',
display_name='Test Template')
mock_read_helper.assert_called_once()
mock_create_tag_template.assert_called_once(
) # Only the master Template is created.
def test_staff_gets_alert_if_http_error_on_search_activity_list(self):
mock = Mock()
mock.configure_mock(**{
"activities.return_value.search.return_value.execute.side_effect":
HttpError(
content=self.error_content,
resp=Response(self.error_resp_dict))
})
with patch("cmsplugin_googleplus.googleplus.build", return_value=mock):
google_plus_api = GooglePlusAPI(TEST_DEVELOPER_KEY)
search_activity_list, alert = google_plus_api.get_search_activity_list(
query='test')
self.assertEqual(search_activity_list, [])
self.assertEqual(alert, 'Google Plus API error: `<HttpError 400 "Bad Request">`')
