def test_retry(self): num_retries = 5 resp_seq = [({'status': '500'}, '')] * num_retries resp_seq.append(({'status': '200'}, '{}')) http = HttpMockSequence(resp_seq) model = JsonModel() uri = u'https://www.googleapis.com/someapi/v1/collection/?foo=bar' method = u'POST' request = HttpRequest( http, model.response, uri, method=method, body=u'{}', headers={'content-type': 'application/json'}) sleeptimes = [] request._sleep = lambda x: sleeptimes.append(x) request._rand = lambda: 10 request.execute(num_retries=num_retries) self.assertEqual(num_retries, len(sleeptimes)) for retry_num in xrange(num_retries): self.assertEqual(10 * 2**(retry_num + 1), sleeptimes[retry_num])
def test_media_io_base_next_chunk_no_retry_403_not_configured(self): fd = BytesIO(b"i am png") upload = MediaIoBaseUpload( fd=fd, mimetype='image/png', chunksize=500, resumable=True) http = HttpMockSequence([ ({'status': '403'}, NOT_CONFIGURED_RESPONSE), ({'status': '200'}, '{}') ]) model = JsonModel() uri = u'https://www.googleapis.com/someapi/v1/upload/?foo=bar' method = u'POST' request = HttpRequest( http, model.response, uri, method=method, headers={}, resumable=upload) request._rand = lambda: 1.0 request._sleep = mock.MagicMock() with self.assertRaises(HttpError): request.execute(num_retries=3) request._sleep.assert_not_called()
def test_retry_connection_errors_non_resumable(self): model = JsonModel() request = HttpRequest( HttpMockWithErrors(3, {'status': '200'}, '{"foo": "bar"}'), model.response, u'https://www.example.com/json_api_endpoint') request._sleep = lambda _x: 0 # do nothing request._rand = lambda: 10 response = request.execute(num_retries=3) self.assertEqual({u'foo': u'bar'}, response)
def test_no_retry_connection_errors(self): model = JsonModel() request = HttpRequest( HttpMockWithNonRetriableErrors(1, {'status': '200'}, '{"foo": "bar"}'), model.response, u'https://www.example.com/json_api_endpoint') request._sleep = lambda _x: 0 # do nothing request._rand = lambda: 10 with self.assertRaises(socket.error): response = request.execute(num_retries=3)
def test_retry_connection_errors_resumable(self): with open(datafile('small.png'), 'rb') as small_png_file: small_png_fd = BytesIO(small_png_file.read()) upload = MediaIoBaseUpload(fd=small_png_fd, mimetype='image/png', chunksize=500, resumable=True) model = JsonModel() request = HttpRequest( HttpMockWithErrors( 3, {'status': '200', 'location': 'location'}, '{"foo": "bar"}'), model.response, u'https://www.example.com/file_upload', method='POST', resumable=upload) request._sleep = lambda _x: 0 # do nothing request._rand = lambda: 10 response = request.execute(num_retries=3) self.assertEqual({u'foo': u'bar'}, response)
def test_media_io_base_next_chunk_retries(self): try: import io except ImportError: return f = open(datafile('small.png'), 'r') fd = io.BytesIO(f.read()) upload = MediaIoBaseUpload( fd=fd, mimetype='image/png', chunksize=500, resumable=True) # Simulate 5XXs for both the request that creates the resumable upload and # the upload itself. http = HttpMockSequence([ ({'status': '500'}, ''), ({'status': '500'}, ''), ({'status': '503'}, ''), ({'status': '200', 'location': 'location'}, ''), ({'status': '500'}, ''), ({'status': '500'}, ''), ({'status': '503'}, ''), ({'status': '200'}, '{}'), ]) model = JsonModel() uri = u'https://www.googleapis.com/someapi/v1/upload/?foo=bar' method = u'POST' request = HttpRequest( http, model.response, uri, method=method, headers={}, resumable=upload) sleeptimes = [] request._sleep = lambda x: sleeptimes.append(x) request._rand = lambda: 10 request.execute(num_retries=3) self.assertEqual([20, 40, 80, 20, 40, 80], sleeptimes)
def test_no_retry_401_fails_fast(self): http = HttpMockSequence([ ({'status': '401'}, ''), ({'status': '200'}, '{}') ]) model = JsonModel() uri = u'https://www.googleapis.com/someapi/v1/collection/?foo=bar' method = u'POST' request = HttpRequest( http, model.response, uri, method=method, body=u'{}', headers={'content-type': 'application/json'}) request._rand = lambda: 1.0 request._sleep = mock.MagicMock() with self.assertRaises(HttpError): request.execute() request._sleep.assert_not_called()
def test_no_retry_fails_fast(self): http = HttpMockSequence([ ({'status': '500'}, ''), ({'status': '200'}, '{}') ]) model = JsonModel() uri = u'https://www.googleapis.com/someapi/v1/collection/?foo=bar' method = u'POST' request = HttpRequest( http, model.response, uri, method=method, body=u'{}', headers={'content-type': 'application/json'}) request._rand = lambda: 1.0 request._sleep = mock.MagicMock() with self.assertRaises(HttpError): request.execute() request._sleep.assert_not_called()
def test_no_retry_succeeds(self): num_retries = 5 resp_seq = [({'status': '200'}, '{}')] * (num_retries) http = HttpMockSequence(resp_seq) model = JsonModel() uri = u'https://www.googleapis.com/someapi/v1/collection/?foo=bar' method = u'POST' request = HttpRequest(http, model.response, uri, method=method, body=u'{}', headers={'content-type': 'application/json'}) sleeptimes = [] request._sleep = lambda x: sleeptimes.append(x) request._rand = lambda: 10 request.execute(num_retries=num_retries) self.assertEqual(0, len(sleeptimes))
def test_no_retry_fails_fast(self): http = HttpMockSequence([ ({'status': '500'}, ''), ({'status': '200'}, '{}') ]) model = JsonModel() uri = u'https://www.googleapis.com/someapi/v1/collection/?foo=bar' method = u'POST' request = HttpRequest( http, model.response, uri, method=method, body=u'{}', headers={'content-type': 'application/json'}) request._rand = lambda: 1.0 request._sleep = lambda _: self.fail('sleep should not have been called.') try: request.execute() self.fail('Should have raised an exception.') except HttpError: pass
def test_media_io_base_next_chunk_retries(self): f = open(datafile('small.png'), 'rb') fd = BytesIO(f.read()) upload = MediaIoBaseUpload( fd=fd, mimetype='image/png', chunksize=500, resumable=True) # Simulate errors for both the request that creates the resumable upload # and the upload itself. http = HttpMockSequence([ ({'status': '500'}, ''), ({'status': '500'}, ''), ({'status': '503'}, ''), ({'status': '200', 'location': 'location'}, ''), ({'status': '403'}, USER_RATE_LIMIT_EXCEEDED_RESPONSE), ({'status': '403'}, RATE_LIMIT_EXCEEDED_RESPONSE), ({'status': '429'}, ''), ({'status': '200'}, '{}'), ]) model = JsonModel() uri = u'https://www.googleapis.com/someapi/v1/upload/?foo=bar' method = u'POST' request = HttpRequest( http, model.response, uri, method=method, headers={}, resumable=upload) sleeptimes = [] request._sleep = lambda x: sleeptimes.append(x) request._rand = lambda: 10 request.execute(num_retries=3) self.assertEqual([20, 40, 80, 20, 40, 80], sleeptimes)