def test_no_retry_fails_fast(self):
http = HttpMockSequence([({
'status': '500'
}, ''), ({
'status': '200'
}, '{}')])
model = JsonModel()
uri = 'https://www.googleapis.com/someapi/v1/collection/?foo=bar'
method = 'POST'
request = HttpRequest(http,
model.response,
uri,
method=method,
body='{}',
headers={'content-type': 'application/json'})
request._rand = lambda: 1.0
request._sleep = lambda _: self.fail(
'sleep should not have been called.')
try:
request.execute()
self.fail('Should have raised an exception.')
except HttpError:
pass
def test_http_request_to_from_json(self):
def _postproc(*kwargs):
pass
http = httplib2.Http()
media_upload = MediaFileUpload(datafile("small.png"), chunksize=500, resumable=True)
req = HttpRequest(
http,
_postproc,
"http://example.com",
method="POST",
body="{}",
headers={"content-type": 'multipart/related; boundary="---flubber"'},
methodId="foo",
resumable=media_upload,
)
json = req.to_json()
new_req = HttpRequest.from_json(json, http, _postproc)
self.assertEqual({"content-type": 'multipart/related; boundary="---flubber"'}, new_req.headers)
self.assertEqual("http://example.com", new_req.uri)
self.assertEqual("{}", new_req.body)
self.assertEqual(http, new_req.http)
self.assertEqual(media_upload.to_json(), new_req.resumable.to_json())
def test_retry(self):
num_retries = 5
resp_seq = [({'status': '500'}, '')] * num_retries
resp_seq.append(({'status': '200'}, '{}'))
http = HttpMockSequence(resp_seq)
model = JsonModel()
uri = u'https://www.googleapis.com/someapi/v1/collection/?foo=bar'
method = u'POST'
request = HttpRequest(
http,
model.response,
uri,
method=method,
body=u'{}',
headers={'content-type': 'application/json'})
sleeptimes = []
request._sleep = lambda x: sleeptimes.append(x)
request._rand = lambda: 10
request.execute(num_retries=num_retries)
self.assertEqual(num_retries, len(sleeptimes))
for retry_num in xrange(num_retries):
self.assertEqual(10 * 2**(retry_num + 1), sleeptimes[retry_num])
def test_http_request_to_from_json(self):
def _postproc(*kwargs):
pass
http = httplib2.Http()
media_upload = MediaFileUpload(
datafile('small.png'), chunksize=500, resumable=True)
req = HttpRequest(
http,
_postproc,
'http://example.com',
method='POST',
body='{}',
headers={'content-type': 'multipart/related; boundary="---flubber"'},
methodId='foo',
resumable=media_upload)
json = req.to_json()
new_req = HttpRequest.from_json(json, http, _postproc)
self.assertEquals(new_req.headers,
{'content-type':
'multipart/related; boundary="---flubber"'})
self.assertEquals(new_req.uri, 'http://example.com')
self.assertEquals(new_req.body, '{}')
self.assertEquals(new_req.http, http)
self.assertEquals(new_req.resumable.to_json(), media_upload.to_json())
self.assertEquals(new_req.multipart_boundary, '---flubber--')
def test_http_request_to_from_json(self):
def _postproc(*kwargs):
pass
http = httplib2.Http()
media_upload = MediaFileUpload(datafile('small.png'),
chunksize=500,
resumable=True)
req = HttpRequest(http,
_postproc,
'http://example.com',
method='POST',
body='{}',
headers={
'content-type':
'multipart/related; boundary="---flubber"'
},
methodId='foo',
resumable=media_upload)
json = req.to_json()
new_req = HttpRequest.from_json(json, http, _postproc)
self.assertEqual(
{'content-type': 'multipart/related; boundary="---flubber"'},
new_req.headers)
self.assertEqual('http://example.com', new_req.uri)
self.assertEqual('{}', new_req.body)
self.assertEqual(http, new_req.http)
self.assertEqual(media_upload.to_json(), new_req.resumable.to_json())
self.assertEqual(random.random, new_req._rand)
self.assertEqual(time.sleep, new_req._sleep)
def test_turn_get_into_post(self):
def _postproc(resp, content):
return content
http = HttpMockSequence([
({
'status': '200'
}, 'echo_request_body'),
({
'status': '200'
}, 'echo_request_headers'),
])
# Send a long query parameter.
query = {'q': 'a' * MAX_URI_LENGTH + '?&'}
req = HttpRequest(http,
_postproc,
'http://example.com?' + urllib.urlencode(query),
method='GET',
body=None,
headers={},
methodId='foo',
resumable=None)
# Query parameters should be sent in the body.
response = req.execute()
self.assertEqual('q=' + 'a' * MAX_URI_LENGTH + '%3F%26', response)
# Extra headers should be set.
response = req.execute()
self.assertEqual('GET', response['x-http-method-override'])
self.assertEqual(str(MAX_URI_LENGTH + 8), response['content-length'])
self.assertEqual('application/x-www-form-urlencoded',
response['content-type'])
def test_retry(self):
num_retries = 5
resp_seq = [({'status': '500'}, '')] * num_retries
resp_seq.append(({'status': '200'}, '{}'))
http = HttpMockSequence(resp_seq)
model = JsonModel()
uri = 'https://www.googleapis.com/someapi/v1/collection/?foo=bar'
method = 'POST'
request = HttpRequest(http,
model.response,
uri,
method=method,
body='{}',
headers={'content-type': 'application/json'})
sleeptimes = []
request._sleep = lambda x: sleeptimes.append(x)
request._rand = lambda: 10
request.execute(num_retries=num_retries)
self.assertEqual(num_retries, len(sleeptimes))
for retry_num in range(num_retries):
self.assertEqual(10 * 2**(retry_num + 1), sleeptimes[retry_num])
def test_media_io_base_next_chunk_retries(self):
try:
import io
except ImportError:
return
f = open(datafile('small.png'), 'r')
fd = io.BytesIO(f.read())
upload = MediaIoBaseUpload(fd=fd,
mimetype='image/png',
chunksize=500,
resumable=True)
# Simulate 5XXs for both the request that creates the resumable upload and
# the upload itself.
http = HttpMockSequence([
({
'status': '500'
}, ''),
({
'status': '500'
}, ''),
({
'status': '503'
}, ''),
({
'status': '200',
'location': 'location'
}, ''),
({
'status': '500'
}, ''),
({
'status': '500'
}, ''),
({
'status': '503'
}, ''),
({
'status': '200'
}, '{}'),
])
model = JsonModel()
uri = 'https://www.googleapis.com/someapi/v1/upload/?foo=bar'
method = 'POST'
request = HttpRequest(http,
model.response,
uri,
method=method,
headers={},
resumable=upload)
sleeptimes = []
request._sleep = lambda x: sleeptimes.append(x)
request._rand = lambda: 10
request.execute(num_retries=3)
self.assertEqual([20, 40, 80, 20, 40, 80], sleeptimes)
def test_ensure_response_callback(self):
m = JsonModel()
request = HttpRequest(
None,
m.response,
'https://www.googleapis.com/someapi/v1/collection/?foo=bar',
method='POST',
body='{}',
headers={'content-type': 'application/json'})
h = HttpMockSequence([ ({'status': 200}, '{}')])
responses = []
def _on_response(resp, responses=responses):
responses.append(resp)
request.add_response_callback(_on_response)
request.execute(http=h)
self.assertEqual(1, len(responses))
def _make_request(self, request: HttpRequest) -> Dict:
retryable_error_codes = [500, 503]
current_delay = 0.1 #100ms
while True:
try:
return request.execute()
except ConnectionResetError:
message = "Lost connection to the engine."
except HttpError as raw_err:
err = json.loads(raw_err.content).get('error')
message = err.get('message')
# Raise RuntimeError for exceptions that are not retryable.
# Otherwise, pass through to retry.
if not err.get('code') in retryable_error_codes:
raise EngineException(message) from raw_err
current_delay *= 2
if current_delay > self.max_retry_delay:
raise TimeoutError(
'Reached max retry attempts for error: {}'.format(message))
if (self.verbose):
print(message, file=sys.stderr)
print('Waiting ',
current_delay,
'seconds before retrying.',
file=sys.stderr)
time.sleep(current_delay)
def test_unicode(self):
http = HttpMock(datafile('zoo.json'), headers={'status': '200'})
model = JsonModel()
uri = u'https://www.googleapis.com/someapi/v1/collection/?foo=bar'
method = u'POST'
request = HttpRequest(http,
model.response,
uri,
method=method,
body=u'{}',
headers={'content-type': 'application/json'})
request.execute()
self.assertEqual(uri, http.uri)
self.assertEqual(str, type(http.uri))
self.assertEqual(method, http.method)
self.assertEqual(str, type(http.method))
def setUp(self):
model = JsonModel()
self.request1 = HttpRequest(
None,
model.response,
'https://www.googleapis.com/someapi/v1/collection/?foo=bar',
method='POST',
body='{}',
headers={'content-type': 'application/json'})
self.request2 = HttpRequest(
None,
model.response,
'https://www.googleapis.com/someapi/v1/collection/?foo=bar',
method='GET',
body='',
headers={'content-type': 'application/json'})
def test_unicode(self):
http = HttpMock(datafile('zoo.json'), headers={'status': '200'})
model = JsonModel()
uri = u'https://www.googleapis.com/someapi/v1/collection/?foo=bar'
method = u'POST'
request = HttpRequest(
http,
model.response,
uri,
method=method,
body=u'{}',
headers={'content-type': 'application/json'})
request.execute()
self.assertEqual(uri, http.uri)
self.assertEqual(str, type(http.uri))
self.assertEqual(method, http.method)
self.assertEqual(str, type(http.method))
def test_media_io_base_next_chunk_retries(self):
try:
import io
except ImportError:
return
f = open(datafile('small.png'), 'r')
fd = io.BytesIO(f.read())
upload = MediaIoBaseUpload(
fd=fd, mimetype='image/png', chunksize=500, resumable=True)
# Simulate 5XXs for both the request that creates the resumable upload and
# the upload itself.
http = HttpMockSequence([
({'status': '500'}, ''),
({'status': '500'}, ''),
({'status': '503'}, ''),
({'status': '200', 'location': 'location'}, ''),
({'status': '500'}, ''),
({'status': '500'}, ''),
({'status': '503'}, ''),
({'status': '200'}, '{}'),
])
model = JsonModel()
uri = u'https://www.googleapis.com/someapi/v1/upload/?foo=bar'
method = u'POST'
request = HttpRequest(
http,
model.response,
uri,
method=method,
headers={},
resumable=upload)
sleeptimes = []
request._sleep = lambda x: sleeptimes.append(x)
request._rand = lambda: 10
request.execute(num_retries=3)
self.assertEqual([20, 40, 80, 20, 40, 80], sleeptimes)
def test_serialize_request_no_body(self):
batch = BatchHttpRequest()
request = HttpRequest(
None,
None,
'https://www.googleapis.com/someapi/v1/collection/?foo=bar',
method='POST',
body='',
headers={'content-type': 'application/json'},
methodId=None,
resumable=None)
s = batch._serialize_request(request).splitlines()
self.assertEqual(NO_BODY_EXPECTED.splitlines(), s)
def method(self, **kwargs):
for name in kwargs.iterkeys():
if name not in argmap:
raise TypeError('Got an unexpected keyword argument "%s"' %
name)
for name in required_params:
if name not in kwargs:
raise TypeError('Missing required parameter "%s"' % name)
for name, regex in pattern_params.iteritems():
if name in kwargs:
if re.match(regex, kwargs[name]) is None:
raise TypeError(
'Parameter "%s" value "%s" does not match the pattern "%s"'
% (name, kwargs[name], regex))
actual_query_params = {}
actual_path_params = {}
for key, value in kwargs.iteritems():
if key in query_params:
actual_query_params[argmap[key]] = value
if key in path_params:
actual_path_params[argmap[key]] = value
body_value = kwargs.get('body', None)
if self._developerKey:
actual_query_params['key'] = self._developerKey
headers = {}
headers, params, query, body = self._model.request(
headers, actual_path_params, actual_query_params, body_value)
# TODO(ade) This exists to fix a bug in V1 of the Buzz discovery document.
# Base URLs should not contain any path elements. If they do then urlparse.urljoin will strip them out
# This results in an incorrect URL which returns a 404
url_result = urlparse.urlsplit(self._baseUrl)
new_base_url = url_result.scheme + '://' + url_result.netloc
expanded_url = uritemplate.expand(pathUrl, params)
url = urlparse.urljoin(new_base_url,
url_result.path + expanded_url + query)
logging.info('URL being requested: %s' % url)
return HttpRequest(self._http,
url,
method=httpMethod,
body=body,
headers=headers,
postproc=self._model.response)
def test_turn_get_into_post(self):
def _postproc(resp, content):
return content
http = HttpMockSequence([
({'status': '200'},
'echo_request_body'),
({'status': '200'},
'echo_request_headers'),
])
# Send a long query parameter.
query = {
'q': 'a' * MAX_URI_LENGTH + '?&'
}
req = HttpRequest(
http,
_postproc,
'http://example.com?' + urllib.parse.urlencode(query),
method='GET',
body=None,
headers={},
methodId='foo',
resumable=None)
# Query parameters should be sent in the body.
response = req.execute()
self.assertEqual('q=' + 'a' * MAX_URI_LENGTH + '%3F%26', response)
# Extra headers should be set.
response = req.execute()
response = simplejson.loads(response.decode())
self.assertEqual('GET', response['x-http-method-override'])
self.assertEqual(str(MAX_URI_LENGTH + 8), response['content-length'])
self.assertEqual(
'application/x-www-form-urlencoded', response['content-type'])
def test_no_retry_fails_fast(self):
http = HttpMockSequence([
({'status': '500'}, ''),
({'status': '200'}, '{}')
])
model = JsonModel()
uri = u'https://www.googleapis.com/someapi/v1/collection/?foo=bar'
method = u'POST'
request = HttpRequest(
http,
model.response,
uri,
method=method,
body=u'{}',
headers={'content-type': 'application/json'})
request._rand = lambda: 1.0
request._sleep = lambda _: self.fail('sleep should not have been called.')
try:
request.execute()
self.fail('Should have raised an exception.')
except HttpError:
pass
def _execute_with_repeat(self,
request: HttpRequest,
retry_num: int = 10) -> Any:
for i in range(retry_num):
try:
response = request.execute()
break
except HttpError:
self._logger.warning(
'An http error occurs during execution, retrying...')
self._logger.warning('Error information: {}'.format(
sys.exc_info()))
else:
raise HttpError(
'http request failed after {} trying.'.format(retry_num))
return response
def test_serialize_request_media_body(self):
batch = BatchHttpRequest()
f = open(datafile('small.png'))
body = f.read()
f.close()
request = HttpRequest(
None,
None,
'https://www.googleapis.com/someapi/v1/collection/?foo=bar',
method='POST',
body=body,
headers={'content-type': 'application/json'},
methodId=None,
resumable=None)
# Just testing it shouldn't raise an exception.
s = batch._serialize_request(request).splitlines()
def test_ensure_response_callback(self):
m = JsonModel()
request = HttpRequest(
None,
m.response,
'https://www.googleapis.com/someapi/v1/collection/?foo=bar',
method='POST',
body='{}',
headers={'content-type': 'application/json'})
h = HttpMockSequence([({'status': 200}, '{}')])
responses = []
def _on_response(resp, responses=responses):
responses.append(resp)
request.add_response_callback(_on_response)
request.execute(http=h)
self.assertEqual(1, len(responses))
def method(self, previous):
"""
Takes a single argument, 'body', which is the results
from the last call, and returns the next set of items
in the collection.
Returns None if there are no more items in
the collection.
"""
if methodDesc['type'] != 'uri':
raise UnknownLinkType(methodDesc['type'])
try:
p = previous
for key in methodDesc['location']:
p = p[key]
url = p
except (KeyError, TypeError):
return None
if self._developerKey:
parsed = list(urlparse.urlparse(url))
q = parse_qsl(parsed[4])
q.append(('key', self._developerKey))
parsed[4] = urllib.urlencode(q)
url = urlparse.urlunparse(parsed)
headers = {}
headers, params, query, body = self._model.request(
headers, {}, {}, None)
logging.info('URL being requested: %s' % url)
resp, content = self._http.request(url,
method='GET',
headers=headers)
return HttpRequest(self._http,
url,
method='GET',
headers=headers,
postproc=self._model.response)
def _build_request(self, _http, *args, **kwargs):
# Create a new Http() object for every request
http = self.credentials.authorize(httplib2.Http())
return HttpRequest(http, *args, **kwargs)
