def test_no_retry_fails_fast(self):
http = HttpMockSequence([({
'status': '500'
}, ''), ({
'status': '200'
}, '{}')])
model = JsonModel()
uri = u'https://www.googleapis.com/someapi/v1/collection/?foo=bar'
method = u'POST'
request = HttpRequest(http,
model.response,
uri,
method=method,
body=u'{}',
headers={'content-type': 'application/json'})
request._rand = lambda: 1.0
request._sleep = lambda _: self.fail(
'sleep should not have been called.')
try:
request.execute()
self.fail('Should have raised an exception.')
except HttpError:
pass
def test_media_io_base_next_chunk_no_retry_403_not_configured(self):
fd = BytesIO(b"i am png")
upload = MediaIoBaseUpload(
fd=fd, mimetype='image/png', chunksize=500, resumable=True)
http = HttpMockSequence([
({'status': '403'}, NOT_CONFIGURED_RESPONSE),
({'status': '200'}, '{}')
])
model = JsonModel()
uri = u'https://www.googleapis.com/someapi/v1/upload/?foo=bar'
method = u'POST'
request = HttpRequest(
http,
model.response,
uri,
method=method,
headers={},
resumable=upload)
request._rand = lambda: 1.0
request._sleep = mock.MagicMock()
with self.assertRaises(HttpError):
request.execute(num_retries=3)
request._sleep.assert_not_called()
def test_media_io_base_next_chunk_no_retry_403_not_configured(self):
fd = BytesIO(b"i am png")
upload = MediaIoBaseUpload(fd=fd,
mimetype='image/png',
chunksize=500,
resumable=True)
http = HttpMockSequence([({
'status': '403'
}, NOT_CONFIGURED_RESPONSE), ({
'status': '200'
}, '{}')])
model = JsonModel()
uri = u'https://www.googleapis.com/someapi/v1/upload/?foo=bar'
method = u'POST'
request = HttpRequest(http,
model.response,
uri,
method=method,
headers={},
resumable=upload)
request._rand = lambda: 1.0
request._sleep = mock.MagicMock()
with self.assertRaises(HttpError):
request.execute(num_retries=3)
request._sleep.assert_not_called()
def test_retry(self):
num_retries = 5
resp_seq = [({'status': '500'}, '')] * num_retries
resp_seq.append(({'status': '200'}, '{}'))
http = HttpMockSequence(resp_seq)
model = JsonModel()
uri = u'https://www.googleapis.com/someapi/v1/collection/?foo=bar'
method = u'POST'
request = HttpRequest(
http,
model.response,
uri,
method=method,
body=u'{}',
headers={'content-type': 'application/json'})
sleeptimes = []
request._sleep = lambda x: sleeptimes.append(x)
request._rand = lambda: 10
request.execute(num_retries=num_retries)
self.assertEqual(num_retries, len(sleeptimes))
for retry_num in xrange(num_retries):
self.assertEqual(10 * 2**(retry_num + 1), sleeptimes[retry_num])
def test_turn_get_into_post(self):
def _postproc(resp, content):
return content
http = HttpMockSequence([
({
'status': '200'
}, 'echo_request_body'),
({
'status': '200'
}, 'echo_request_headers'),
])
# Send a long query parameter.
query = {'q': 'a' * MAX_URI_LENGTH + '?&'}
req = HttpRequest(http,
_postproc,
'http://example.com?' + urlencode(query),
method='GET',
body=None,
headers={},
methodId='foo',
resumable=None)
# Query parameters should be sent in the body.
response = req.execute()
self.assertEqual(b'q=' + b'a' * MAX_URI_LENGTH + b'%3F%26', response)
# Extra headers should be set.
response = req.execute()
self.assertEqual('GET', response['x-http-method-override'])
self.assertEqual(str(MAX_URI_LENGTH + 8), response['content-length'])
self.assertEqual('application/x-www-form-urlencoded',
response['content-type'])
def test_retry(self):
num_retries = 5
resp_seq = [({'status': '500'}, '')] * (num_retries - 3)
resp_seq.append(({'status': '403'}, RATE_LIMIT_EXCEEDED_RESPONSE))
resp_seq.append(({'status': '403'}, USER_RATE_LIMIT_EXCEEDED_RESPONSE))
resp_seq.append(({'status': '429'}, ''))
resp_seq.append(({'status': '200'}, '{}'))
http = HttpMockSequence(resp_seq)
model = JsonModel()
uri = u'https://www.googleapis.com/someapi/v1/collection/?foo=bar'
method = u'POST'
request = HttpRequest(http,
model.response,
uri,
method=method,
body=u'{}',
headers={'content-type': 'application/json'})
sleeptimes = []
request._sleep = lambda x: sleeptimes.append(x)
request._rand = lambda: 10
request.execute(num_retries=num_retries)
self.assertEqual(num_retries, len(sleeptimes))
for retry_num in range(num_retries):
self.assertEqual(10 * 2**(retry_num + 1), sleeptimes[retry_num])
def test_media_io_base_next_chunk_retries(self):
try:
import io
except ImportError:
return
f = open(datafile('small.png'), 'r')
fd = io.BytesIO(f.read())
upload = MediaIoBaseUpload(fd=fd,
mimetype='image/png',
chunksize=500,
resumable=True)
# Simulate 5XXs for both the request that creates the resumable upload and
# the upload itself.
http = HttpMockSequence([
({
'status': '500'
}, ''),
({
'status': '500'
}, ''),
({
'status': '503'
}, ''),
({
'status': '200',
'location': 'location'
}, ''),
({
'status': '500'
}, ''),
({
'status': '500'
}, ''),
({
'status': '503'
}, ''),
({
'status': '200'
}, '{}'),
])
model = JsonModel()
uri = u'https://www.googleapis.com/someapi/v1/upload/?foo=bar'
method = u'POST'
request = HttpRequest(http,
model.response,
uri,
method=method,
headers={},
resumable=upload)
sleeptimes = []
request._sleep = lambda x: sleeptimes.append(x)
request._rand = lambda: 10
request.execute(num_retries=3)
self.assertEqual([20, 40, 80, 20, 40, 80], sleeptimes)
def test_empty_content_type(self):
"""Test for #284"""
http = HttpMock(None, headers={'status': 200})
uri = u'https://www.googleapis.com/someapi/v1/upload/?foo=bar'
method = u'POST'
request = HttpRequest(
http,
_postproc_none,
uri,
method=method,
headers={'content-type': ''})
request.execute()
self.assertEqual('', http.headers.get('content-type'))
def test_empty_content_type(self):
"""Test for #284"""
http = HttpMock(None, headers={'status': 200})
uri = u'https://www.googleapis.com/someapi/v1/upload/?foo=bar'
method = u'POST'
request = HttpRequest(
http,
_postproc_none,
uri,
method=method,
headers={'content-type': ''})
request.execute()
self.assertEqual('', http.headers.get('content-type'))
def log_and_send(self, desc: str, request: http.HttpRequest) -> Response:
log.info(desc)
log.debug(pprint.pformat(request))
response = request.execute()
log.debug(pprint.pformat(response))
self.save_credentials() # They may have been refreshed in the process.
return response
def exponentialBackoff(request: HttpRequest,
time: int = 0,
tries: int = 0,
max_tries: int = 8):
"""Periodically retry a failed request over an increasing amount of time to
handle errors related to rate limits, network volume, or response time.
https://developers.google.com/drive/api/v3/handle-errors
"""
try:
response = None
while response is None:
status, response = request.next_chunk()
if status:
print(
f'Uploading: {round(status.resumable_progress / status.total_size * 100, 2)}%'
)
file = request.execute(num_retries=2)
return file
except HttpError as e:
print(e)
if tries < max_tries:
print(f'Trying again in {time}')
t = Timer(time,
exponentialBackoff,
args=(request, 2**(tries + 1) + random.random(),
tries + 1, max_tries))
t.start()
else:
raise Exception('Max tries reached.')
def _execute_request(
self, request: http.HttpRequest
) -> Mapping[Text, Sequence[Mapping[Text, Any]]]:
result = request.execute()
if 'error' in result:
raise ValueError(result['error'])
return result
def test_ensure_response_callback(self):
m = JsonModel()
request = HttpRequest(
None,
m.response,
'https://www.googleapis.com/someapi/v1/collection/?foo=bar',
method='POST',
body='{}',
headers={'content-type': 'application/json'})
h = HttpMockSequence([ ({'status': 200}, '{}')])
responses = []
def _on_response(resp, responses=responses):
responses.append(resp)
request.add_response_callback(_on_response)
request.execute(http=h)
self.assertEqual(1, len(responses))
def test_unicode(self):
http = HttpMock(datafile('zoo.json'), headers={'status': '200'})
model = JsonModel()
uri = u'https://www.googleapis.com/someapi/v1/collection/?foo=bar'
method = u'POST'
request = HttpRequest(http,
model.response,
uri,
method=method,
body=u'{}',
headers={'content-type': 'application/json'})
request.execute()
self.assertEqual(uri, http.uri)
self.assertEqual(str, type(http.uri))
self.assertEqual(method, http.method)
self.assertEqual(str, type(http.method))
def create(cluster_api: discovery.Resource, creds: Credentials,
request: HttpRequest, project_id: str) -> "Optional[Cluster]":
'''create cluster
Note that this is a blocking call.
Args:
cluster_api: cluster api client
cred: credentials
request: cluster creation http request
project_id: project id
zone: zone
Returns:
Cluster instance on success, None otherwise
'''
daemonset_url = utils.nvidia_daemonset_url(NodeImage.COS)
body = json.loads(request.body)
zone = body['cluster']['zone']
cluster_name = body['cluster']['name']
# execute
rsp = request.execute()
if rsp is None:
logging.error('error: could not create cluster')
return
# wait for creation operation to complete
operation_name = rsp['name']
rsp = utils.wait_for_operation(
cluster_api, 'projects/{}/locations/{}/operations/{}'.format(
project_id, zone, operation_name))
if rsp['status'] != OpStatus.DONE.value:
logging.error('error creating cluster {}!'.format(cluster_name))
return
# get our newly-created cluster
cluster = Cluster.get(name=cluster_name,
project_id=project_id,
zone=zone,
creds=creds)
if cluster is None:
logging.error(
'error: unable to connect to cluster {}'.format(cluster_name))
logging.error(
'nvidia-driver daemonset not applied, to do this manually:')
logging.error('kubectl apply -f {}'.format(daemonset_url))
return
logging.info('created cluster {} successfully'.format(cluster_name))
logging.info('applying nvidia driver daemonset...')
rsp = cluster.apply_daemonset_from_url(
daemonset_url, lambda x: yaml.load(x, Loader=yaml.FullLoader))
return cluster
def test_unicode(self):
http = HttpMock(datafile('zoo.json'), headers={'status': '200'})
model = JsonModel()
uri = u'https://www.googleapis.com/someapi/v1/collection/?foo=bar'
method = u'POST'
request = HttpRequest(
http,
model.response,
uri,
method=method,
body=u'{}',
headers={'content-type': 'application/json'})
request.execute()
self.assertEqual(uri, http.uri)
self.assertEqual(str, type(http.uri))
self.assertEqual(method, http.method)
self.assertEqual(str, type(http.method))
def test_retry_connection_errors_non_resumable(self):
model = JsonModel()
request = HttpRequest(
HttpMockWithErrors(3, {'status': '200'}, '{"foo": "bar"}'),
model.response, u'https://www.example.com/json_api_endpoint')
request._sleep = lambda _x: 0 # do nothing
request._rand = lambda: 10
response = request.execute(num_retries=3)
self.assertEqual({u'foo': u'bar'}, response)
def test_no_retry_connection_errors(self):
model = JsonModel()
request = HttpRequest(
HttpMockWithNonRetriableErrors(1, {'status': '200'}, '{"foo": "bar"}'),
model.response,
u'https://www.example.com/json_api_endpoint')
request._sleep = lambda _x: 0 # do nothing
request._rand = lambda: 10
with self.assertRaises(socket.error):
response = request.execute(num_retries=3)
def test_retry_connection_errors_non_resumable(self):
model = JsonModel()
request = HttpRequest(
HttpMockWithErrors(3, {'status': '200'}, '{"foo": "bar"}'),
model.response,
u'https://www.example.com/json_api_endpoint')
request._sleep = lambda _x: 0 # do nothing
request._rand = lambda: 10
response = request.execute(num_retries=3)
self.assertEqual({u'foo': u'bar'}, response)
def test_no_retry_connection_errors(self):
model = JsonModel()
request = HttpRequest(
HttpMockWithNonRetriableErrors(1, {'status': '200'},
'{"foo": "bar"}'), model.response,
u'https://www.example.com/json_api_endpoint')
request._sleep = lambda _x: 0 # do nothing
request._rand = lambda: 10
with self.assertRaises(socket.error):
response = request.execute(num_retries=3)
def test_media_io_base_next_chunk_retries(self):
try:
import io
except ImportError:
return
f = open(datafile('small.png'), 'r')
fd = io.BytesIO(f.read())
upload = MediaIoBaseUpload(
fd=fd, mimetype='image/png', chunksize=500, resumable=True)
# Simulate 5XXs for both the request that creates the resumable upload and
# the upload itself.
http = HttpMockSequence([
({'status': '500'}, ''),
({'status': '500'}, ''),
({'status': '503'}, ''),
({'status': '200', 'location': 'location'}, ''),
({'status': '500'}, ''),
({'status': '500'}, ''),
({'status': '503'}, ''),
({'status': '200'}, '{}'),
])
model = JsonModel()
uri = u'https://www.googleapis.com/someapi/v1/upload/?foo=bar'
method = u'POST'
request = HttpRequest(
http,
model.response,
uri,
method=method,
headers={},
resumable=upload)
sleeptimes = []
request._sleep = lambda x: sleeptimes.append(x)
request._rand = lambda: 10
request.execute(num_retries=3)
self.assertEqual([20, 40, 80, 20, 40, 80], sleeptimes)
def test_no_retry_401_fails_fast(self):
http = HttpMockSequence([({
'status': '401'
}, ''), ({
'status': '200'
}, '{}')])
model = JsonModel()
uri = u'https://www.googleapis.com/someapi/v1/collection/?foo=bar'
method = u'POST'
request = HttpRequest(http,
model.response,
uri,
method=method,
body=u'{}',
headers={'content-type': 'application/json'})
request._rand = lambda: 1.0
request._sleep = mock.MagicMock()
with self.assertRaises(HttpError):
request.execute()
request._sleep.assert_not_called()
def test_no_retry_401_fails_fast(self):
http = HttpMockSequence([
({'status': '401'}, ''),
({'status': '200'}, '{}')
])
model = JsonModel()
uri = u'https://www.googleapis.com/someapi/v1/collection/?foo=bar'
method = u'POST'
request = HttpRequest(
http,
model.response,
uri,
method=method,
body=u'{}',
headers={'content-type': 'application/json'})
request._rand = lambda: 1.0
request._sleep = mock.MagicMock()
with self.assertRaises(HttpError):
request.execute()
request._sleep.assert_not_called()
def test_no_retry_succeeds(self):
num_retries = 5
resp_seq = [({'status': '200'}, '{}')] * (num_retries)
http = HttpMockSequence(resp_seq)
model = JsonModel()
uri = u'https://www.googleapis.com/someapi/v1/collection/?foo=bar'
method = u'POST'
request = HttpRequest(http,
model.response,
uri,
method=method,
body=u'{}',
headers={'content-type': 'application/json'})
sleeptimes = []
request._sleep = lambda x: sleeptimes.append(x)
request._rand = lambda: 10
request.execute(num_retries=num_retries)
self.assertEqual(0, len(sleeptimes))
def test_turn_get_into_post(self):
def _postproc(resp, content):
return content
http = HttpMockSequence([
({'status': '200'},
'echo_request_body'),
({'status': '200'},
'echo_request_headers'),
])
# Send a long query parameter.
query = {
'q': 'a' * MAX_URI_LENGTH + '?&'
}
req = HttpRequest(
http,
_postproc,
'http://example.com?' + urllib.urlencode(query),
method='GET',
body=None,
headers={},
methodId='foo',
resumable=None)
# Query parameters should be sent in the body.
response = req.execute()
self.assertEqual('q=' + 'a' * MAX_URI_LENGTH + '%3F%26', response)
# Extra headers should be set.
response = req.execute()
self.assertEqual('GET', response['x-http-method-override'])
self.assertEqual(str(MAX_URI_LENGTH + 8), response['content-length'])
self.assertEqual(
'application/x-www-form-urlencoded', response['content-type'])
def execute_request(request: http.HttpRequest) -> Any:
"""Executes an HTTP request and return its response.
This method executes an HTTP request and retries if the response error code is
capable of being retried. Refer to `retry_utils.py` module to see the list of
error codes for which the retry will be attempted.
Args:
request: HTTP request to be executed.
Returns:
response: Response from the HTTP request.
"""
response = request.execute()
return response
def test_no_retry_fails_fast(self):
http = HttpMockSequence([
({'status': '500'}, ''),
({'status': '200'}, '{}')
])
model = JsonModel()
uri = u'https://www.googleapis.com/someapi/v1/collection/?foo=bar'
method = u'POST'
request = HttpRequest(
http,
model.response,
uri,
method=method,
body=u'{}',
headers={'content-type': 'application/json'})
request._rand = lambda: 1.0
request._sleep = lambda _: self.fail('sleep should not have been called.')
try:
request.execute()
self.fail('Should have raised an exception.')
except HttpError:
pass
def test_retry_connection_errors_resumable(self):
with open(datafile('small.png'), 'rb') as small_png_file:
small_png_fd = BytesIO(small_png_file.read())
upload = MediaIoBaseUpload(fd=small_png_fd, mimetype='image/png',
chunksize=500, resumable=True)
model = JsonModel()
request = HttpRequest(
HttpMockWithErrors(
3, {'status': '200', 'location': 'location'}, '{"foo": "bar"}'),
model.response,
u'https://www.example.com/file_upload',
method='POST',
resumable=upload)
request._sleep = lambda _x: 0 # do nothing
request._rand = lambda: 10
response = request.execute(num_retries=3)
self.assertEqual({u'foo': u'bar'}, response)
def _send_whole_upload(request: HttpRequest):
'''Upload a non-resumable media file.
@params:
request: HttpRequest, a MediaUpload object.
@return: tuple(bool, whether or not the upload succeeded
response, the result of the executed request (or None)
'''
if not request or not isinstance(request, HttpRequest):
return (False, None)
try:
resp = request.execute(num_retries=2)
return (True, resp)
except (HttpLib2Error | HttpError) as err:
print('Upload failed:', err)
return (False, None)
def _exponential_backoff(request: HttpRequest) -> Dict[Any, Any]:
"""
Экспоненциальная выдержка для 500 и 503 ошибки.
:param request: Запрос к API Google Drive
:return: Результат запроса
"""
for n in range(0, 5):
try:
return request.execute()
except errors.HttpError as e:
if not re.search(r"HttpError (500|503)", str(e)):
log.error(f"Failed to complete request to Google Drive "
f"API due to error: {str(e)}")
raise e
if n == 4:
error_text = ("Failed to complete request to "
"Google Drive API in 5 attempts due "
"to 500 or 503 error")
log.error(error_text)
raise Exception(error_text)
log.debug(
f"HTTP error: {str(e)[11:14]}. Trying again. Attempt {n + 1}"
)
time.sleep((2**n) + random.random())
def google_api_call(request: HttpRequest):
try:
return request.execute()
except Exception:
logger.error('Call to Google Cloud Resource Manager API Failed!',
sys.exc_info()[0])
def logAndExecute(request: HttpRequest):
"""Executes the given request while adding debug logs for the request and response."""
logging.debug('Request: {}'.format(request.to_json()))
response = request.execute()
logging.debug('Response: {}'.format(response))
return response
