def _request(self, page_url, params=None, url_data=None):
"""Given a page url and a dict of params, opens and returns the page.
:param page_url: the url to grab content from.
:param params: the extra url data to submit
:param url_data: the GET data to put in the url
:returns: the open page
"""
return _request(self, page_url, params, url_data, self._opener)
def _request(self, page_url, params=None, url_data=None):
"""Given a page url and a dict of params, opens and returns the page.
:param page_url: the url to grab content from.
:param params: the extra url data to submit
:param url_data: the GET data to put in the url
:returns: the open page
"""
return _request(self, page_url, params, url_data, self._opener)
def _request(self, page_url, params=None, url_data=None, timeout=None):
"""Given a page url and a dict of params, opens and returns the page.
:param page_url: the url to grab content from.
:param params: a dictionary containing the extra url data to submit
:param url_data: a dictionary containing the GET data to put in the url
:returns: the open page
"""
# pylint: disable-msg=W0212
timeout = self.config.timeout if timeout is None else timeout
remaining_attempts = 3
while True:
try:
return helpers._request(self, page_url, params,
url_data, timeout)
except HTTPError as error:
remaining_attempts -= 1
if (error.code not in self.RETRY_CODES or
remaining_attempts == 0):
raise
except http_client.IncompleteRead:
remaining_attempts -= 1
if remaining_attempts == 0:
raise