def HandleResult(response, urlfetch_exception, allow_truncated): """Returns (result, user_exception) to return from a fetch() call.""" result = None user_exception = None if urlfetch_exception: user_exception = urlfetch_exception if hasattr(urlfetch_exception, 'application_error'): if (urlfetch_exception.application_error == urlfetch_service_pb.URLFetchServiceError.INVALID_URL): user_exception = urlfetch.InvalidURLError(str(urlfetch_exception)) elif (urlfetch_exception.application_error == urlfetch_service_pb.URLFetchServiceError.UNSPECIFIED_ERROR): user_exception = urlfetch.DownloadError(str(urlfetch_exception)) elif (urlfetch_exception.application_error == urlfetch_service_pb.URLFetchServiceError.FETCH_ERROR): user_exception = urlfetch.DownloadError(str(urlfetch_exception)) elif (urlfetch_exception.application_error == urlfetch_service_pb.URLFetchServiceError.RESPONSE_TOO_LARGE): user_exception = urlfetch.ResponseTooLargeError(None) else: result = urlfetch._URLFetchResult(response) if not allow_truncated and response.contentwastruncated(): user_exception = urlfetch.ResponseTooLargeError(result) return result, user_exception
def _UpdateCatalogIfChanged(cls, catalog, url): """Returns the contents of a url passed to URLFetch.fetch(). Args: catalog: models.AppleSUSCatalog entity to update. url: str url to fetch. Returns: Boolean. True if the catalog was updated, False otherwise. Raises: urlfetch.Error on failures. """ headers = {'If-Modified-Since': catalog.last_modified_header} response = urlfetch.fetch(url, headers=headers, deadline=30, validate_certificate=True) if response.status_code == httplib.NOT_MODIFIED: return False elif response.status_code == httplib.OK: xml = response.content # TODO(user): validate response plist here. #logging.info( # '%s SUS catalog is old. Updating...', catalog.key().name()) header_date_str = response.headers.get('Last-Modified', '') cls._UpdateCatalog(xml, entity=catalog, last_modified=header_date_str) return True else: raise urlfetch.DownloadError('Non-200 status_code: %s' % response.status_code)
def get_result(self): results = [] if self.do_timeout: raise urlfetch.DownloadError( "Deadline exceeded while waiting for HTTP response from URL") elif self.expired_token: return FakeResult( 400, { 'error': { 'message': u'Error validating access token: Session has expired on Jun 9, 2014 10:05am. The current time is Jun 9, 2014 10:32am.', 'code': 190, 'type': 'OAuthException', 'error_subcode': 463 } }) else: urls = [x['relative_url'] for x in self.batch_list] for url in urls: if url in MemoryFBAPI.results: result = MemoryFBAPI.results[url] if result is RESULT_TIMEOUT: status_code = None content = None else: status_code, content = result else: status_code = 404 content = None if status_code: results.append( dict(code=status_code, body=json.dumps(content))) else: results.append(None) return FakeResult(200, results)
def _Dynamic_Fetch(self, request, response): request_method = { request.GET: 'GET', request.POST: 'POST', request.PUT: 'PUT', request.PATCH: 'PATCH', request.DELETE: 'DELETE', }[request.method()] for method, path in self._routes: if request.url().startswith( path) and method.upper() == request_method: data = self._routes[(method, path)] break else: data = { 'status': self._status, 'headers': self._headers, 'content': self._content, } if data['status'] is None: raise urlfetch.DownloadError( 'Unable to fetch URL: {0} Error: test route not setup'.format( request.url())) response.set_finalurl(request.url) response.set_contentwastruncated(False) response.set_statuscode(data['status']) response.set_content(data['content']) for header, value in data['headers'].items(): new_header = response.add_header() new_header.set_key(header) new_header.set_value(value) self.request = request self.response = response
def get_builders(master): """Get the list of builders from a master from CBE. Returns the list in a (platform, master name, builder name) format. """ url = CBE_MASTER_URL % master max_retries = 3 retries = max_retries data = None while retries > 0 and not data: try: data = json.loads(urlfetch.fetch(url, deadline=300).content) except urlfetch.DownloadError: # Exponential backoff time.sleep(2**(max_retries - retries)) continue if not data: raise urlfetch.DownloadError('Could not fetch %s' % url) platforms = ['ios', 'android', 'cros', 'win', 'mac', 'linux'] results = [] for builder in data['builders']: platform = 'other' for cur_platform in platforms: if cur_platform in builder: platform = cur_platform break results.append((platform, master, builder)) return results
def test_url_fetch_rpc_error(urlfetch_mock, app): urlfetch_mock.DownloadError = urlfetch.DownloadError get_result_mock = mock.Mock(side_effect=urlfetch.DownloadError()) urlfetch_mock.create_rpc = mock.Mock(return_value=mock.Mock( get_result=get_result_mock)) response = app.get('/', status=500) assert 'Error fetching URL' in response.body
def testNoRetryAfterDoRequestUrlFetchTimeout(self): retry_params = api_utils.RetryParams(max_retries=0) api = rest_api._RestApi('scope', retry_params=retry_params) fut = ndb.Future() fut.set_exception(urlfetch.DownloadError()) ndb.Context.urlfetch = mock.create_autospec(ndb.Context.urlfetch, return_value=fut) self.assertRaises(urlfetch.DownloadError, api.do_request, 'foo')
def ok_resp(rs): if rs.status_code == 200: return rs else: raise urlfetch.DownloadError( "%d response:\n\n%r\n\n%s" % (rs.status_code, rs.headers, rs.content))
def testRetryFailWithResponseTimeOut(self): self.results.extend([urlfetch.DownloadError()] * (self.max_retries - 1)) self.results.append( test_utils.MockUrlFetchResult(httplib.REQUEST_TIMEOUT, None, None)) with mock.patch.object(api_utils.urlfetch, 'fetch') as f: f.side_effect = self._SideEffect self.assertEqual( httplib.REQUEST_TIMEOUT, api_utils._retry_fetch('foo', self.retry_params).status_code)
def testRetryAfterDoRequestUrlFetchTimeout(self): api = rest_api._RestApi('scope') fut = ndb.Future() fut.set_exception(urlfetch.DownloadError()) ndb.Context.urlfetch = mock.create_autospec(ndb.Context.urlfetch, return_value=fut) with mock.patch('google.appengine.api.urlfetch' '.fetch') as f: f.return_value = test_utils.MockUrlFetchResult( httplib.ACCEPTED, None, None) self.assertEqual(httplib.ACCEPTED, api.do_request('foo')[0])
def testRetryAfterDoRequestUrlFetchTimeout(self): api = rest_api._RestApi('scope') resp_fut1 = ndb.Future() resp_fut1.set_exception(urlfetch.DownloadError()) resp_fut2 = ndb.Future() resp_fut2.set_result( test_utils.MockUrlFetchResult(httplib.ACCEPTED, None, None)) ndb.Context.urlfetch = mock.create_autospec( ndb.Context.urlfetch, side_effect=[resp_fut1, resp_fut2]) self.assertEqual(httplib.ACCEPTED, api.do_request('foo')[0]) self.assertEqual(2, ndb.Context.urlfetch.call_count)
def testRetrySuccess(self): self.results.append( test_utils.MockUrlFetchResult(httplib.REQUEST_TIMEOUT, None, None)) self.results.append( test_utils.MockUrlFetchResult(httplib.SERVICE_UNAVAILABLE, None, None)) self.results.append(urlfetch.DownloadError()) self.results.append(apiproxy_errors.Error()) self.results.append( test_utils.MockUrlFetchResult(httplib.ACCEPTED, None, None)) with mock.patch.object(api_utils.urlfetch, 'fetch') as f: f.side_effect = self._SideEffect self.assertEqual( httplib.ACCEPTED, api_utils._retry_fetch('foo', self.retry_params, deadline=1000).status_code) self.assertEqual(1000, f.call_args[1]['deadline'])
def testUrlFetchCalledWithUserProvidedDeadline(self): retry_params = api_utils.RetryParams(urlfetch_timeout=90) api = rest_api._RestApi('scope', retry_params=retry_params) resp_fut1 = ndb.Future() resp_fut1.set_exception(urlfetch.DownloadError()) resp_fut2 = ndb.Future() resp_fut2.set_result( test_utils.MockUrlFetchResult(httplib.ACCEPTED, None, None)) ndb.Context.urlfetch = mock.create_autospec( ndb.Context.urlfetch, side_effect=[resp_fut1, resp_fut2]) self.assertEqual(httplib.ACCEPTED, api.do_request('foo')[0]) self.assertEqual(90, ndb.Context.urlfetch.call_args_list[0][1]['deadline']) self.assertEqual(90, ndb.Context.urlfetch.call_args_list[1][1]['deadline'])