def test_list_emails_error(self, harvester, mocker):
error_url = 'https://http.cat/418'
response = mock.MagicMock(status=418, reason='Is teapot')
mocker.patch('googleapiclient.http.HttpRequest.execute',
side_effect=errors.HttpError(response, b'Is teapot', uri=error_url))
emails = harvester.list_unread_emails()
assert emails is None
def testSuccessWithURI(self):
with patch('airflow.contrib.operators.mlengine_operator.MLEngineHook') \
as mock_hook:
input_with_uri = self.INPUT_MISSING_ORIGIN.copy()
input_with_uri['uri'] = 'gs://my_bucket/my_models/savedModel'
success_message = self.SUCCESS_MESSAGE_MISSING_INPUT.copy()
success_message['predictionInput'] = input_with_uri
hook_instance = mock_hook.return_value
hook_instance.get_job.side_effect = errors.HttpError(
resp=httplib2.Response({'status': 404}), content=b'some bytes')
hook_instance.create_job.return_value = success_message
prediction_task = MLEngineBatchPredictionOperator(
job_id='test_prediction',
project_id='test-project',
region=input_with_uri['region'],
data_format=input_with_uri['dataFormat'],
input_paths=input_with_uri['inputPaths'],
output_path=input_with_uri['outputPath'],
uri=input_with_uri['uri'],
dag=self.dag,
task_id='test-prediction')
prediction_output = prediction_task.execute(None)
mock_hook.assert_called_with('google_cloud_default', None)
hook_instance.create_job.assert_called_with(
'test-project', {
'jobId': 'test_prediction',
'predictionInput': input_with_uri
}, ANY)
self.assertEquals(success_message['predictionOutput'],
prediction_output)
def GetAttachments(service, user_id, msg_id, store_dir):
"""Get and store attachment from Message with given id.
Args:
service: Authorized Gmail API service instance.
user_id: User's email address. The special value "me"
can be used to indicate the authenticated user.
msg_id: ID of Message containing attachment.
store_dir: The directory used to store attachments.
"""
try:
message = service.users().messages().get(userId=user_id, id=msg_id).execute()
for part in message['payload']['parts']:
if part['filename']:
if 'data' in part['body']:
data = part['body']['data']
else:
att_id = part['body']['attachmentId']
att = service.users().messages().attachments().get(
userId=user_id, messageId=msg_id, id=att_id).execute()
data = att['data']
file_data = base64.urlsafe_b64decode(data.encode('UTF-8'))
path = part['filename']
with open(path, 'wb') as f:
f.write(file_data)
except errors.HttpError():
print('An error occurred: %s' % sys.exe_info()[0])
def testSuccessWithModel(self):
with patch('airflow.contrib.operators.cloudml_operator.CloudMLHook') \
as mock_hook:
input_with_model = self.INPUT_MISSING_ORIGIN.copy()
input_with_model['modelName'] = \
'projects/test-project/models/test_model'
success_message = self.SUCCESS_MESSAGE_MISSING_INPUT.copy()
success_message['predictionInput'] = input_with_model
hook_instance = mock_hook.return_value
hook_instance.get_job.side_effect = errors.HttpError(
resp=httplib2.Response({'status': 404}), content=b'some bytes')
hook_instance.create_job.return_value = success_message
prediction_task = CloudMLBatchPredictionOperator(
job_id='test_prediction',
project_id='test-project',
region=input_with_model['region'],
data_format=input_with_model['dataFormat'],
input_paths=input_with_model['inputPaths'],
output_path=input_with_model['outputPath'],
model_name=input_with_model['modelName'].split('/')[-1],
dag=self.dag,
task_id='test-prediction')
prediction_output = prediction_task.execute(None)
mock_hook.assert_called_with('google_cloud_default', None)
hook_instance.create_job.assert_called_once_with(
'test-project', {
'jobId': 'test_prediction',
'predictionInput': input_with_model
}, ANY)
self.assertEquals(success_message['predictionOutput'],
prediction_output)
def testHttpError(self):
http_error_code = 403
with patch('airflow.contrib.operators.cloudml_operator.CloudMLHook') \
as mock_hook:
input_with_model = self.INPUT_MISSING_ORIGIN.copy()
input_with_model['modelName'] = \
'projects/experimental/models/test_model'
hook_instance = mock_hook.return_value
hook_instance.create_job.side_effect = errors.HttpError(
resp=httplib2.Response({'status': http_error_code}),
content=b'Forbidden')
with self.assertRaises(errors.HttpError) as context:
prediction_task = CloudMLBatchPredictionOperator(
job_id='test_prediction',
project_id='test-project',
region=input_with_model['region'],
data_format=input_with_model['dataFormat'],
input_paths=input_with_model['inputPaths'],
output_path=input_with_model['outputPath'],
model_name=input_with_model['modelName'].split('/')[-1],
dag=self.dag,
task_id='test-prediction')
prediction_task.execute(None)
mock_hook.assert_called_with('google_cloud_default', None)
hook_instance.create_job.assert_called_with(
'test-project', {
'jobId': 'test_prediction',
'predictionInput': input_with_model
}, ANY)
self.assertEquals(http_error_code, context.exception.resp.status)
def test_subscription_doesnt_exist(self):
"""Ensure that the subscription is created if it doesn't exist."""
mock_subscriptions = mock.Mock()
mock_subscription = mock.Mock()
mock_subscription.execute.side_effect = errors.HttpError(
mock.Mock(status=404), 'not found')
mock_subscriptions.get.return_value = mock_subscription
create_subscription = mock.Mock()
mock_subscriptions.create.return_value = create_subscription
self.mock_pubsub.subscriptions.return_value = mock_subscriptions
self.client.subscribe('foo', 'bar', 'https://baz.com')
mock_subscriptions.get.assert_called_once_with(
subscription='/subscriptions/project/foo')
mock_subscription.execute.assert_called_once_with()
mock_subscriptions.create.assert_called_once_with(
body={
'name': '/subscriptions/project/foo',
'topic': '/topics/project/bar',
'pushConfig': {
'pushEndpoint': 'https://baz.com',
}
})
create_subscription.execute.assert_called_once_with()
def test_gather_logs_http_error(self, log_mock):
"""GSuiteReportsApp - Gather Logs, HTTP Error"""
with patch.object(self._app, '_activities_service') as service_mock:
error = errors.HttpError('response', bytes('bad'))
service_mock.list.return_value.execute.side_effect = error
assert_false(self._app._gather_logs())
log_mock.assert_called_with('Failed to execute activities listing')
def update(self, path, content):
"""Update the file
Args and returns same as put
"""
dbg.dbg(path)
path = util.format_path(path)
metadata = self._path_to_metadata(path)
file_id = metadata['id']
uri = GoogleAPI.UPLOAD_URL + '/files/%s?uploadType=media' % file_id
headers = {
'Content-Type': 'text/plain',
'Content-Length': len(content),
}
for retry_num in xrange(self._num_retries + 1):
resp, data = self.service._http.request(uri,
method='PUT',
body=content,
headers=headers)
if resp.status < 500:
break
if resp.status >= 300:
raise errors.HttpError(resp, data, uri=uri)
if resp.status == 200:
drive_file = json.loads(data)
self._cache_metadata(path, drive_file)
return True
else:
return False
def revision_text(http, file_id, rev_id):
revision = revision_core.revision_details(http, file_id, rev_id)
ret = http.request(revision["downloadUrl"])
# for some reason, ret is a tuple when the request
# succeeds, so try EAFTP
try:
return ret[1]
except KeyError as e:
raise errors.HttpError(ret.status)
def deid_execute():
response = deid_responses[deid_execute.call_count]
deid_execute.call_count += 1
if response == 'Exception':
content = (
'{"error": {"message": "Too many findings to de-identify. '
'Retry with a smaller request."}}')
raise errors.HttpError(httplib2.Response({'status': 400}),
content)
return response
def test_publish_body_raise_on_publish_404(self):
"""Tests if the flush method raises when publish gets a 404 error."""
mocked_resp = mock.MagicMock()
mocked_resp.status = 404
mocked_resp.reason = 'Not Found'
# 404 error
self.topics_publish.execute.side_effect = [
errors.HttpError(mocked_resp, 'Not found')
]
self.assertRaises(errors.HttpError, self.publish)
def test_flush_raise_on_publish_403(self):
"""Tests if the flush method raises when publish gets a 403 error."""
mocked_resp = mock.MagicMock()
mocked_resp.status = 403
mocked_resp.reason = 'Access not allowed'
# 403 error
self.topics_publish.execute.side_effect = [
errors.HttpError(mocked_resp, 'Access not allowed'),
]
self.assertRaises(errors.HttpError, self.publish)
def test_flush_raise_on_publish_404(self):
"""Tests if the flush raises upon 404 error from publish_body."""
self.handler.emit(self.r)
mocked_resp = mock.MagicMock()
mocked_resp.status = 404
mocked_resp.reason = 'Not Found'
# 404 error and None for atexit.
self.publish_body.side_effect = [
errors.HttpError(mocked_resp, 'Not found'), None
]
self.assertRaises(errors.HttpError, self.handler.flush)
def test_get_email_data_error(self, factory, mocker):
error_url = 'https://http.cat/418'
response = mock.MagicMock(status=418, reason='Is teapot')
mocker.patch('googleapiclient.http.HttpRequest.execute',
side_effect=errors.HttpError(response,
b'Is teapot',
uri=error_url))
with pytest.raises(errors.HttpError) as e:
factory._get_email_data('journal_with_year')
assert e.type is errors.HttpError
assert e.value.content == b'Is teapot'
def testMakeCommentRequest_UserCantOwn_RetryMakeCommentRequest(self):
service = issue_tracker_service.IssueTrackerService(mock.MagicMock())
error_content = {
'error': {'message': 'Issue owner must be a project member',
'code': 400}
}
service._ExecuteRequest = mock.Mock(side_effect=errors.HttpError(
mock.Mock(return_value={'status': 404}), json.dumps(error_content)))
service.AddBugComment(12345, 'The comment', owner=['*****@*****.**'])
self.assertEqual(2, service._ExecuteRequest.call_count)
def testNewBug_HttpError_NewBugReturnsError(self):
service = issue_tracker_service.IssueTrackerService(mock.MagicMock())
error_content = {
'error': {'message': 'The user does not exist: [email protected]',
'code': 404}
}
service._ExecuteRequest = mock.Mock(side_effect=errors.HttpError(
mock.Mock(return_value={'status': 404}), json.dumps(error_content)))
response = service.NewBug('Bug title', 'body', owner='*****@*****.**')
self.assertEqual(1, service._ExecuteRequest.call_count)
self.assertTrue('error' in response)
def testMakeCommentRequest_UserDoesNotExist_RetryMakeCommentRequest(self):
service = issue_tracker_service.IssueTrackerService(mock.MagicMock())
error_content = {
'error': {'message': 'The user does not exist: [email protected]',
'code': 404}
}
service._ExecuteRequest = mock.Mock(side_effect=errors.HttpError(
mock.Mock(return_value={'status': 404}), json.dumps(error_content)))
service.AddBugComment(12345, 'The comment', cc_list=['*****@*****.**'],
owner=['*****@*****.**'])
self.assertEqual(2, service._ExecuteRequest.call_count)
def _DownloadFromUrl(self, url):
"""Download file from url using provided credential.
:param url: link of the file to download.
:type url: str.
:returns: str -- content of downloaded file in string.
:raises: ApiRequestError
"""
resp, content = self.http.request(url)
if resp.status != 200:
raise ApiRequestError(errors.HttpError(resp, content, uri=url))
return content
def testMakeCommentRequest_IssueDeleted_ReturnsTrue(self):
service = issue_tracker_service.IssueTrackerService(mock.MagicMock())
error_content = {
'error': {'message': 'User is not allowed to view this issue 12345',
'code': 403}
}
service._ExecuteRequest = mock.Mock(side_effect=errors.HttpError(
mock.Mock(return_value={'status': 403}), json.dumps(error_content)))
comment_posted = service.AddBugComment(12345, 'The comment',
owner='*****@*****.**')
self.assertEqual(1, service._ExecuteRequest.call_count)
self.assertEqual(True, comment_posted)
def test_flush_ignore_recoverable(self):
"""Tests if we raise upon getting 503 error from Cloud Pub/Sub."""
mocked_resp = mock.MagicMock()
mocked_resp.status = 503
mocked_resp.reason = 'Server Error'
# 503 error
self.topics_publish.execute.side_effect = [
errors.HttpError(mocked_resp, 'Server Error'),
]
self.assertRaises(RecoverableError, self.publish)
self.topics.publish.assert_called_once_with(topic=self.topic,
body=self.expected_body)
self.topics_publish.execute.assert_called_once_with(
num_retries=self.RETRY)
def test_create_error(self):
"""Ensure that if the topic create fails, an exception is raised."""
mock_topics = mock.Mock()
mock_topic = mock.Mock()
mock_topic.execute.side_effect = errors.HttpError(
mock.Mock(status=400), 'error')
mock_topics.get.return_value = mock_topic
self.mock_pubsub.topics.return_value = mock_topics
self.assertRaises(errors.HttpError, self.client.create_topic, 'foo')
mock_topics.get.assert_called_once_with(topic='/topics/project/foo')
mock_topic.execute.assert_called_once_with()
def test_delete_doesnt_exist(self):
"""Ensure that nothing happens if the topic doesn't exist."""
mock_topics = mock.Mock()
mock_delete = mock.Mock()
mock_delete.execute.side_effect = errors.HttpError(
mock.Mock(status=404), 'not found')
mock_topics.delete.return_value = mock_delete
self.mock_pubsub.topics.return_value = mock_topics
self.client.delete_topic('foo')
self.mock_pubsub.topics.assert_called_once_with()
mock_topics.delete.assert_called_once_with(topic='/topics/project/foo')
mock_delete.execute.assert_called_once_with()
def test_unsubscribe_doesnt_exist(self):
"""Ensure that nothing happens if the subscription doesn't exist."""
mock_subscriptions = mock.Mock()
mock_delete = mock.Mock()
mock_delete.execute.side_effect = errors.HttpError(
mock.Mock(status=404), 'not found')
mock_subscriptions.delete.return_value = mock_delete
self.mock_pubsub.subscriptions.return_value = mock_subscriptions
self.client.unsubscribe('foo')
self.mock_pubsub.subscriptions.assert_called_once_with()
mock_subscriptions.delete.assert_called_once_with(
subscription='/subscriptions/project/foo')
mock_delete.execute.assert_called_once_with()
def GetMessageId(service, user_id, query):
try:
response = service.users().messages().list(userId=user_id, q=query).execute()
messages = []
if 'messages' in response:
messages.extend(response['messages'])
# print(response)
# print(response['messages'][0]['id'])
messageId = response['messages'][0]['id']
return messageId
except errors.HttpError():
print('An error occurred: %s' % sys.exe_info()[0])
def test_subscribe_error(self):
"""Ensure that if the subscription create fails, an exception is
raised.
"""
mock_subscriptions = mock.Mock()
mock_subscription = mock.Mock()
mock_subscription.execute.side_effect = errors.HttpError(
mock.Mock(status=400), 'error')
mock_subscriptions.get.return_value = mock_subscription
self.mock_pubsub.subscriptions.return_value = mock_subscriptions
self.assertRaises(errors.HttpError, self.client.subscribe, 'foo',
'bar', 'https://baz.com')
mock_subscriptions.get.assert_called_once_with(
subscription='/subscriptions/project/foo')
mock_subscription.execute.assert_called_once_with()
def test_unsubscribe_error(self):
"""Ensure that if the subscription deletion fails, an exception is
raised.
"""
mock_subscriptions = mock.Mock()
mock_delete = mock.Mock()
mock_delete.execute.side_effect = errors.HttpError(
mock.Mock(status=400), 'error')
mock_subscriptions.delete.return_value = mock_delete
self.mock_pubsub.subscriptions.return_value = mock_subscriptions
self.assertRaises(errors.HttpError, self.client.unsubscribe, 'foo')
self.mock_pubsub.subscriptions.assert_called_once_with()
mock_subscriptions.delete.assert_called_once_with(
subscription='/subscriptions/project/foo')
mock_delete.execute.assert_called_once_with()
def test_topic_doesnt_exist(self):
"""Ensure that the topic is created if it doesn't exist."""
mock_topics = mock.Mock()
mock_topic = mock.Mock()
mock_topic.execute.side_effect = errors.HttpError(
mock.Mock(status=404), 'not found')
mock_topics.get.return_value = mock_topic
create_topic = mock.Mock()
mock_topics.create.return_value = create_topic
self.mock_pubsub.topics.return_value = mock_topics
self.client.create_topic('foo')
mock_topics.get.assert_called_once_with(topic='/topics/project/foo')
mock_topic.execute.assert_called_once_with()
mock_topics.create.assert_called_once_with(
body={'name': '/topics/project/foo'})
create_topic.execute.assert_called_once_with()
def test_publish_error(self):
"""Ensure that publish raises an exception when the publish fails."""
mock_topics = mock.Mock()
mock_publish = mock.Mock()
mock_publish.execute.side_effect = errors.HttpError(400, 'error')
mock_topics.publish.return_value = mock_publish
self.mock_pubsub.topics.return_value = mock_topics
self.assertRaises(errors.HttpError, self.client.publish, 'foo', 'bar')
self.mock_pubsub.topics.assert_called_once_with()
mock_topics.publish.assert_called_once_with(
body={
'topic': '/topics/project/foo',
'message': {
'data': base64.b64encode('bar'),
}
})
mock_publish.execute.assert_called_once_with()
def testHttpError(self):
http_error_code = 403
with patch('airflow.contrib.operators.cloudml_operator.CloudMLHook') \
as mock_hook:
hook_instance = mock_hook.return_value
hook_instance.create_job.side_effect = errors.HttpError(
resp=httplib2.Response({'status': http_error_code}),
content=b'Forbidden')
with self.assertRaises(errors.HttpError) as context:
training_op = CloudMLTrainingOperator(
**self.TRAINING_DEFAULT_ARGS)
training_op.execute(None)
mock_hook.assert_called_with(gcp_conn_id='google_cloud_default',
delegate_to=None)
# Make sure only 'create_job' is invoked on hook instance
self.assertEquals(len(hook_instance.mock_calls), 1)
hook_instance.create_job.assert_called_with(
'test-project', self.TRAINING_INPUT, ANY)
self.assertEquals(http_error_code, context.exception.resp.status)
def execute(self):
raise errors.HttpError(self.resp, self.content)
