def _paginated_fetch(self, url, params, callback, page_size=None, index=1, total=None):
"""Returns an iterator to a paginated resource.
"""
if page_size is None:
page_size = self.page_size
# update the page number
scope_params = self._params(page_size, index)
scope_params.update(params)
# get the page
log.debug("url = '%s', params = '%s'" % (url, json.dumps(scope_params)))
res = request_with_retry('GET', url=url, params=scope_params, headers=self._headers())
if res.status_code != 200:
raise ValueError("could not fetch Google resource: '%s'" % url)
# extract the information
data = res.json()['feed']
entry = data.get('entry', None)
if entry is None: # collection is empty, return.
return
# yield every item as a generator
for item in data['entry']:
yield callback(item)
# iterate every page
total_results = total if total else g_json_value(data, 'totalResults', 'openSearch')
remaining_results = total_results - (index + page_size - 1)
if remaining_results > 0:
for item in self._paginated_fetch(url, params, callback, page_size, index + page_size, total):
yield item
def request_with_retry(method, n_retries=settings.N_MAX_ATTEMPTS, **kwargs):
s = requests.Session()
req = requests.Request(method, **kwargs)
prepped = req.prepare()
try:
res = s.send(prepped)
except (ValueError, IOError, SSLError, ConnectionError):
log.debug("request: '%s %s' failed, retrying." % (method, kwargs['url']))
if n_retries <= 1:
raise
return request_with_retry(method, n_retries=n_retries-1, **kwargs)
if res.status_code not in (200, 201, 202, 204):
return request_with_retry(method, n_retries=n_retries-1, **kwargs)
return res
def request_with_retry(method, n_retries=settings.N_MAX_ATTEMPTS, **kwargs):
s = requests.Session()
req = requests.Request(method, **kwargs)
prepped = req.prepare()
try:
res = s.send(prepped)
except (ValueError, IOError, SSLError, ConnectionError):
log.debug("request: '%s %s' failed, retrying." %
(method, kwargs['url']))
if n_retries <= 1:
raise
return request_with_retry(method, n_retries=n_retries - 1, **kwargs)
if res.status_code not in (200, 201, 202, 204):
return request_with_retry(method, n_retries=n_retries - 1, **kwargs)
return res
def _paginated_fetch(self,
url,
params,
callback,
page_size=None,
index=1,
total=None):
"""Returns an iterator to a paginated resource.
"""
if page_size is None:
page_size = self.page_size
# update the page number
scope_params = self._params(page_size, index)
scope_params.update(params)
# get the page
log.debug("url = '%s', params = '%s'" %
(url, json.dumps(scope_params)))
res = request_with_retry('GET',
url=url,
params=scope_params,
headers=self._headers())
if res.status_code != 200:
raise ValueError("could not fetch Google resource: '%s'" % url)
# extract the information
data = res.json()['feed']
entry = data.get('entry', None)
if entry is None: # collection is empty, return.
return
# yield every item as a generator
for item in data['entry']:
yield callback(item)
# iterate every page
total_results = total if total else g_json_value(
data, 'totalResults', 'openSearch')
remaining_results = total_results - (index + page_size - 1)
if remaining_results > 0:
for item in self._paginated_fetch(url, params, callback, page_size,
index + page_size, total):
yield item
