def _paginated_fetch(self, url, params, callback, page_size=None, index=1, total=None): """Returns an iterator to a paginated resource. """ if page_size is None: page_size = self.page_size # update the page number scope_params = self._params(page_size, index) scope_params.update(params) # get the page log.debug("url = '%s', params = '%s'" % (url, json.dumps(scope_params))) res = request_with_retry('GET', url=url, params=scope_params, headers=self._headers()) if res.status_code != 200: raise ValueError("could not fetch Google resource: '%s'" % url) # extract the information data = res.json()['feed'] entry = data.get('entry', None) if entry is None: # collection is empty, return. return # yield every item as a generator for item in data['entry']: yield callback(item) # iterate every page total_results = total if total else g_json_value(data, 'totalResults', 'openSearch') remaining_results = total_results - (index + page_size - 1) if remaining_results > 0: for item in self._paginated_fetch(url, params, callback, page_size, index + page_size, total): yield item
def request_with_retry(method, n_retries=settings.N_MAX_ATTEMPTS, **kwargs): s = requests.Session() req = requests.Request(method, **kwargs) prepped = req.prepare() try: res = s.send(prepped) except (ValueError, IOError, SSLError, ConnectionError): log.debug("request: '%s %s' failed, retrying." % (method, kwargs['url'])) if n_retries <= 1: raise return request_with_retry(method, n_retries=n_retries-1, **kwargs) if res.status_code not in (200, 201, 202, 204): return request_with_retry(method, n_retries=n_retries-1, **kwargs) return res
def request_with_retry(method, n_retries=settings.N_MAX_ATTEMPTS, **kwargs): s = requests.Session() req = requests.Request(method, **kwargs) prepped = req.prepare() try: res = s.send(prepped) except (ValueError, IOError, SSLError, ConnectionError): log.debug("request: '%s %s' failed, retrying." % (method, kwargs['url'])) if n_retries <= 1: raise return request_with_retry(method, n_retries=n_retries - 1, **kwargs) if res.status_code not in (200, 201, 202, 204): return request_with_retry(method, n_retries=n_retries - 1, **kwargs) return res
def _paginated_fetch(self, url, params, callback, page_size=None, index=1, total=None): """Returns an iterator to a paginated resource. """ if page_size is None: page_size = self.page_size # update the page number scope_params = self._params(page_size, index) scope_params.update(params) # get the page log.debug("url = '%s', params = '%s'" % (url, json.dumps(scope_params))) res = request_with_retry('GET', url=url, params=scope_params, headers=self._headers()) if res.status_code != 200: raise ValueError("could not fetch Google resource: '%s'" % url) # extract the information data = res.json()['feed'] entry = data.get('entry', None) if entry is None: # collection is empty, return. return # yield every item as a generator for item in data['entry']: yield callback(item) # iterate every page total_results = total if total else g_json_value( data, 'totalResults', 'openSearch') remaining_results = total_results - (index + page_size - 1) if remaining_results > 0: for item in self._paginated_fetch(url, params, callback, page_size, index + page_size, total): yield item