def test_no_retry_fails_fast(self):
http = HttpMockSequence([({
'status': '500'
}, ''), ({
'status': '200'
}, '{}')])
model = JsonModel()
uri = u'https://www.googleapis.com/someapi/v1/collection/?foo=bar'
method = u'POST'
request = HttpRequest(http,
model.response,
uri,
method=method,
body=u'{}',
headers={'content-type': 'application/json'})
request._rand = lambda: 1.0
request._sleep = lambda _: self.fail(
'sleep should not have been called.')
try:
request.execute()
self.fail('Should have raised an exception.')
except HttpError:
pass
def test_turn_get_into_post(self):
def _postproc(resp, content):
return content
http = HttpMockSequence([
({
'status': '200'
}, 'echo_request_body'),
({
'status': '200'
}, 'echo_request_headers'),
])
# Send a long query parameter.
query = {'q': 'a' * MAX_URI_LENGTH + '?&'}
req = HttpRequest(http,
_postproc,
'http://example.com?' + urllib.urlencode(query),
method='GET',
body=None,
headers={},
methodId='foo',
resumable=None)
# Query parameters should be sent in the body.
response = req.execute()
self.assertEqual('q=' + 'a' * MAX_URI_LENGTH + '%3F%26', response)
# Extra headers should be set.
response = req.execute()
self.assertEqual('GET', response['x-http-method-override'])
self.assertEqual(str(MAX_URI_LENGTH + 8), response['content-length'])
self.assertEqual('application/x-www-form-urlencoded',
response['content-type'])
def test_retry(self):
num_retries = 5
resp_seq = [({'status': '500'}, '')] * num_retries
resp_seq.append(({'status': '200'}, '{}'))
http = HttpMockSequence(resp_seq)
model = JsonModel()
uri = u'https://www.googleapis.com/someapi/v1/collection/?foo=bar'
method = u'POST'
request = HttpRequest(
http,
model.response,
uri,
method=method,
body=u'{}',
headers={'content-type': 'application/json'})
sleeptimes = []
request._sleep = lambda x: sleeptimes.append(x)
request._rand = lambda: 10
request.execute(num_retries=num_retries)
self.assertEqual(num_retries, len(sleeptimes))
for retry_num in xrange(num_retries):
self.assertEqual(10 * 2**(retry_num + 1), sleeptimes[retry_num])
def test_retry(self):
num_retries = 5
resp_seq = [({'status': '500'}, '')] * num_retries
resp_seq.append(({'status': '200'}, '{}'))
http = HttpMockSequence(resp_seq)
model = JsonModel()
uri = u'https://www.googleapis.com/someapi/v1/collection/?foo=bar'
method = u'POST'
request = HttpRequest(http,
model.response,
uri,
method=method,
body=u'{}',
headers={'content-type': 'application/json'})
sleeptimes = []
request._sleep = lambda x: sleeptimes.append(x)
request._rand = lambda: 10
request.execute(num_retries=num_retries)
self.assertEqual(num_retries, len(sleeptimes))
for retry_num in xrange(num_retries):
self.assertEqual(10 * 2**(retry_num + 1), sleeptimes[retry_num])
def test_media_io_base_next_chunk_retries(self):
try:
import io
except ImportError:
return
f = open(datafile('small.png'), 'r')
fd = io.BytesIO(f.read())
upload = MediaIoBaseUpload(fd=fd,
mimetype='image/png',
chunksize=500,
resumable=True)
# Simulate 5XXs for both the request that creates the resumable upload and
# the upload itself.
http = HttpMockSequence([
({
'status': '500'
}, ''),
({
'status': '500'
}, ''),
({
'status': '503'
}, ''),
({
'status': '200',
'location': 'location'
}, ''),
({
'status': '500'
}, ''),
({
'status': '500'
}, ''),
({
'status': '503'
}, ''),
({
'status': '200'
}, '{}'),
])
model = JsonModel()
uri = u'https://www.googleapis.com/someapi/v1/upload/?foo=bar'
method = u'POST'
request = HttpRequest(http,
model.response,
uri,
method=method,
headers={},
resumable=upload)
sleeptimes = []
request._sleep = lambda x: sleeptimes.append(x)
request._rand = lambda: 10
request.execute(num_retries=3)
self.assertEqual([20, 40, 80, 20, 40, 80], sleeptimes)
def _make_request(self, request: HttpRequest) -> Dict:
retryable_error_codes = [500, 503]
current_delay = 0.1 #100ms
while True:
try:
return request.execute()
except ConnectionResetError:
message = "Lost connection to the engine."
except HttpError as raw_err:
err = json.loads(raw_err.content).get('error')
message = err.get('message')
# Raise RuntimeError for exceptions that are not retryable.
# Otherwise, pass through to retry.
if not err.get('code') in retryable_error_codes:
raise EngineException(message) from raw_err
current_delay *= 2
if current_delay > self.max_retry_delay:
raise TimeoutError(
'Reached max retry attempts for error: {}'.format(message))
if (self.verbose):
print(message, file=sys.stderr)
print('Waiting ',
current_delay,
'seconds before retrying.',
file=sys.stderr)
time.sleep(current_delay)
def test_unicode(self):
http = HttpMock(datafile('zoo.json'), headers={'status': '200'})
model = JsonModel()
uri = u'https://www.googleapis.com/someapi/v1/collection/?foo=bar'
method = u'POST'
request = HttpRequest(http,
model.response,
uri,
method=method,
body=u'{}',
headers={'content-type': 'application/json'})
request.execute()
self.assertEqual(uri, http.uri)
self.assertEqual(str, type(http.uri))
self.assertEqual(method, http.method)
self.assertEqual(str, type(http.method))
def test_ensure_response_callback(self):
m = JsonModel()
request = HttpRequest(
None,
m.response,
'https://www.googleapis.com/someapi/v1/collection/?foo=bar',
method='POST',
body='{}',
headers={'content-type': 'application/json'})
h = HttpMockSequence([ ({'status': 200}, '{}')])
responses = []
def _on_response(resp, responses=responses):
responses.append(resp)
request.add_response_callback(_on_response)
request.execute(http=h)
self.assertEqual(1, len(responses))
def test_ensure_response_callback(self):
m = JsonModel()
request = HttpRequest(
None,
m.response,
'https://www.googleapis.com/someapi/v1/collection/?foo=bar',
method='POST',
body='{}',
headers={'content-type': 'application/json'})
h = HttpMockSequence([ ({'status': 200}, '{}')])
responses = []
def _on_response(resp, responses=responses):
responses.append(resp)
request.add_response_callback(_on_response)
request.execute(http=h)
self.assertEqual(1, len(responses))
def test_unicode(self):
http = HttpMock(datafile('zoo.json'), headers={'status': '200'})
model = JsonModel()
uri = u'https://www.googleapis.com/someapi/v1/collection/?foo=bar'
method = u'POST'
request = HttpRequest(
http,
model.response,
uri,
method=method,
body=u'{}',
headers={'content-type': 'application/json'})
request.execute()
self.assertEqual(uri, http.uri)
self.assertEqual(str, type(http.uri))
self.assertEqual(method, http.method)
self.assertEqual(str, type(http.method))
def test_media_io_base_next_chunk_retries(self):
try:
import io
except ImportError:
return
f = open(datafile('small.png'), 'r')
fd = io.BytesIO(f.read())
upload = MediaIoBaseUpload(
fd=fd, mimetype='image/png', chunksize=500, resumable=True)
# Simulate 5XXs for both the request that creates the resumable upload and
# the upload itself.
http = HttpMockSequence([
({'status': '500'}, ''),
({'status': '500'}, ''),
({'status': '503'}, ''),
({'status': '200', 'location': 'location'}, ''),
({'status': '500'}, ''),
({'status': '500'}, ''),
({'status': '503'}, ''),
({'status': '200'}, '{}'),
])
model = JsonModel()
uri = u'https://www.googleapis.com/someapi/v1/upload/?foo=bar'
method = u'POST'
request = HttpRequest(
http,
model.response,
uri,
method=method,
headers={},
resumable=upload)
sleeptimes = []
request._sleep = lambda x: sleeptimes.append(x)
request._rand = lambda: 10
request.execute(num_retries=3)
self.assertEqual([20, 40, 80, 20, 40, 80], sleeptimes)
def test_turn_get_into_post(self):
def _postproc(resp, content):
return content
http = HttpMockSequence([
({'status': '200'},
'echo_request_body'),
({'status': '200'},
'echo_request_headers'),
])
# Send a long query parameter.
query = {
'q': 'a' * MAX_URI_LENGTH + '?&'
}
req = HttpRequest(
http,
_postproc,
'http://example.com?' + urllib.parse.urlencode(query),
method='GET',
body=None,
headers={},
methodId='foo',
resumable=None)
# Query parameters should be sent in the body.
response = req.execute()
self.assertEqual('q=' + 'a' * MAX_URI_LENGTH + '%3F%26', response)
# Extra headers should be set.
response = req.execute()
response = simplejson.loads(response.decode())
self.assertEqual('GET', response['x-http-method-override'])
self.assertEqual(str(MAX_URI_LENGTH + 8), response['content-length'])
self.assertEqual(
'application/x-www-form-urlencoded', response['content-type'])
def test_no_retry_fails_fast(self):
http = HttpMockSequence([
({'status': '500'}, ''),
({'status': '200'}, '{}')
])
model = JsonModel()
uri = u'https://www.googleapis.com/someapi/v1/collection/?foo=bar'
method = u'POST'
request = HttpRequest(
http,
model.response,
uri,
method=method,
body=u'{}',
headers={'content-type': 'application/json'})
request._rand = lambda: 1.0
request._sleep = lambda _: self.fail('sleep should not have been called.')
try:
request.execute()
self.fail('Should have raised an exception.')
except HttpError:
pass
def _execute_with_repeat(self,
request: HttpRequest,
retry_num: int = 10) -> Any:
for i in range(retry_num):
try:
response = request.execute()
break
except HttpError:
self._logger.warning(
'An http error occurs during execution, retrying...')
self._logger.warning('Error information: {}'.format(
sys.exc_info()))
else:
raise HttpError(
'http request failed after {} trying.'.format(retry_num))
return response