def HandleResult(response, urlfetch_exception, allow_truncated):
"""Returns (result, user_exception) to return from a fetch() call."""
result = None
user_exception = None
if urlfetch_exception:
user_exception = urlfetch_exception
if hasattr(urlfetch_exception, 'application_error'):
if (urlfetch_exception.application_error ==
urlfetch_service_pb.URLFetchServiceError.INVALID_URL):
user_exception = urlfetch.InvalidURLError(str(urlfetch_exception))
elif (urlfetch_exception.application_error ==
urlfetch_service_pb.URLFetchServiceError.UNSPECIFIED_ERROR):
user_exception = urlfetch.DownloadError(str(urlfetch_exception))
elif (urlfetch_exception.application_error ==
urlfetch_service_pb.URLFetchServiceError.FETCH_ERROR):
user_exception = urlfetch.DownloadError(str(urlfetch_exception))
elif (urlfetch_exception.application_error ==
urlfetch_service_pb.URLFetchServiceError.RESPONSE_TOO_LARGE):
user_exception = urlfetch.ResponseTooLargeError(None)
else:
result = urlfetch._URLFetchResult(response)
if not allow_truncated and response.contentwastruncated():
user_exception = urlfetch.ResponseTooLargeError(result)
return result, user_exception
def _UpdateCatalogIfChanged(cls, catalog, url):
"""Returns the contents of a url passed to URLFetch.fetch().
Args:
catalog: models.AppleSUSCatalog entity to update.
url: str url to fetch.
Returns:
Boolean. True if the catalog was updated, False otherwise.
Raises:
urlfetch.Error on failures.
"""
headers = {'If-Modified-Since': catalog.last_modified_header}
response = urlfetch.fetch(url,
headers=headers,
deadline=30,
validate_certificate=True)
if response.status_code == httplib.NOT_MODIFIED:
return False
elif response.status_code == httplib.OK:
xml = response.content
# TODO(user): validate response plist here.
#logging.info(
# '%s SUS catalog is old. Updating...', catalog.key().name())
header_date_str = response.headers.get('Last-Modified', '')
cls._UpdateCatalog(xml,
entity=catalog,
last_modified=header_date_str)
return True
else:
raise urlfetch.DownloadError('Non-200 status_code: %s' %
response.status_code)
def get_result(self):
results = []
if self.do_timeout:
raise urlfetch.DownloadError(
"Deadline exceeded while waiting for HTTP response from URL")
elif self.expired_token:
return FakeResult(
400, {
'error': {
'message':
u'Error validating access token: Session has expired on Jun 9, 2014 10:05am. The current time is Jun 9, 2014 10:32am.',
'code': 190,
'type': 'OAuthException',
'error_subcode': 463
}
})
else:
urls = [x['relative_url'] for x in self.batch_list]
for url in urls:
if url in MemoryFBAPI.results:
result = MemoryFBAPI.results[url]
if result is RESULT_TIMEOUT:
status_code = None
content = None
else:
status_code, content = result
else:
status_code = 404
content = None
if status_code:
results.append(
dict(code=status_code, body=json.dumps(content)))
else:
results.append(None)
return FakeResult(200, results)
def _Dynamic_Fetch(self, request, response):
request_method = {
request.GET: 'GET',
request.POST: 'POST',
request.PUT: 'PUT',
request.PATCH: 'PATCH',
request.DELETE: 'DELETE',
}[request.method()]
for method, path in self._routes:
if request.url().startswith(
path) and method.upper() == request_method:
data = self._routes[(method, path)]
break
else:
data = {
'status': self._status,
'headers': self._headers,
'content': self._content,
}
if data['status'] is None:
raise urlfetch.DownloadError(
'Unable to fetch URL: {0} Error: test route not setup'.format(
request.url()))
response.set_finalurl(request.url)
response.set_contentwastruncated(False)
response.set_statuscode(data['status'])
response.set_content(data['content'])
for header, value in data['headers'].items():
new_header = response.add_header()
new_header.set_key(header)
new_header.set_value(value)
self.request = request
self.response = response
def get_builders(master):
"""Get the list of builders from a master from CBE.
Returns the list in a (platform, master name, builder name) format.
"""
url = CBE_MASTER_URL % master
max_retries = 3
retries = max_retries
data = None
while retries > 0 and not data:
try:
data = json.loads(urlfetch.fetch(url, deadline=300).content)
except urlfetch.DownloadError:
# Exponential backoff
time.sleep(2**(max_retries - retries))
continue
if not data:
raise urlfetch.DownloadError('Could not fetch %s' % url)
platforms = ['ios', 'android', 'cros', 'win', 'mac', 'linux']
results = []
for builder in data['builders']:
platform = 'other'
for cur_platform in platforms:
if cur_platform in builder:
platform = cur_platform
break
results.append((platform, master, builder))
return results
def test_url_fetch_rpc_error(urlfetch_mock, app):
urlfetch_mock.DownloadError = urlfetch.DownloadError
get_result_mock = mock.Mock(side_effect=urlfetch.DownloadError())
urlfetch_mock.create_rpc = mock.Mock(return_value=mock.Mock(
get_result=get_result_mock))
response = app.get('/', status=500)
assert 'Error fetching URL' in response.body
def testNoRetryAfterDoRequestUrlFetchTimeout(self):
retry_params = api_utils.RetryParams(max_retries=0)
api = rest_api._RestApi('scope', retry_params=retry_params)
fut = ndb.Future()
fut.set_exception(urlfetch.DownloadError())
ndb.Context.urlfetch = mock.create_autospec(ndb.Context.urlfetch,
return_value=fut)
self.assertRaises(urlfetch.DownloadError, api.do_request, 'foo')
def ok_resp(rs):
if rs.status_code == 200:
return rs
else:
raise urlfetch.DownloadError(
"%d response:\n\n%r\n\n%s"
% (rs.status_code,
rs.headers,
rs.content))
def testRetryFailWithResponseTimeOut(self):
self.results.extend([urlfetch.DownloadError()] *
(self.max_retries - 1))
self.results.append(
test_utils.MockUrlFetchResult(httplib.REQUEST_TIMEOUT, None, None))
with mock.patch.object(api_utils.urlfetch, 'fetch') as f:
f.side_effect = self._SideEffect
self.assertEqual(
httplib.REQUEST_TIMEOUT,
api_utils._retry_fetch('foo', self.retry_params).status_code)
def testRetryAfterDoRequestUrlFetchTimeout(self):
api = rest_api._RestApi('scope')
fut = ndb.Future()
fut.set_exception(urlfetch.DownloadError())
ndb.Context.urlfetch = mock.create_autospec(ndb.Context.urlfetch,
return_value=fut)
with mock.patch('google.appengine.api.urlfetch' '.fetch') as f:
f.return_value = test_utils.MockUrlFetchResult(
httplib.ACCEPTED, None, None)
self.assertEqual(httplib.ACCEPTED, api.do_request('foo')[0])
def testRetryAfterDoRequestUrlFetchTimeout(self):
api = rest_api._RestApi('scope')
resp_fut1 = ndb.Future()
resp_fut1.set_exception(urlfetch.DownloadError())
resp_fut2 = ndb.Future()
resp_fut2.set_result(
test_utils.MockUrlFetchResult(httplib.ACCEPTED, None, None))
ndb.Context.urlfetch = mock.create_autospec(
ndb.Context.urlfetch, side_effect=[resp_fut1, resp_fut2])
self.assertEqual(httplib.ACCEPTED, api.do_request('foo')[0])
self.assertEqual(2, ndb.Context.urlfetch.call_count)
def testRetrySuccess(self):
self.results.append(
test_utils.MockUrlFetchResult(httplib.REQUEST_TIMEOUT, None, None))
self.results.append(
test_utils.MockUrlFetchResult(httplib.SERVICE_UNAVAILABLE, None,
None))
self.results.append(urlfetch.DownloadError())
self.results.append(apiproxy_errors.Error())
self.results.append(
test_utils.MockUrlFetchResult(httplib.ACCEPTED, None, None))
with mock.patch.object(api_utils.urlfetch, 'fetch') as f:
f.side_effect = self._SideEffect
self.assertEqual(
httplib.ACCEPTED,
api_utils._retry_fetch('foo', self.retry_params,
deadline=1000).status_code)
self.assertEqual(1000, f.call_args[1]['deadline'])
def testUrlFetchCalledWithUserProvidedDeadline(self):
retry_params = api_utils.RetryParams(urlfetch_timeout=90)
api = rest_api._RestApi('scope', retry_params=retry_params)
resp_fut1 = ndb.Future()
resp_fut1.set_exception(urlfetch.DownloadError())
resp_fut2 = ndb.Future()
resp_fut2.set_result(
test_utils.MockUrlFetchResult(httplib.ACCEPTED, None, None))
ndb.Context.urlfetch = mock.create_autospec(
ndb.Context.urlfetch, side_effect=[resp_fut1, resp_fut2])
self.assertEqual(httplib.ACCEPTED, api.do_request('foo')[0])
self.assertEqual(90,
ndb.Context.urlfetch.call_args_list[0][1]['deadline'])
self.assertEqual(90,
ndb.Context.urlfetch.call_args_list[1][1]['deadline'])
