def _mech_open(self, url, data=None, update_history=True):
try:
url.get_full_url
except AttributeError:
# string URL -- convert to absolute URL if required
scheme, netloc = urlparse.urlparse(url)[:2]
if not scheme:
# relative URL
assert not netloc, "malformed URL"
if self._response is None:
raise BrowserStateError(
"can't fetch relative URL: not viewing any document")
url = urlparse.urljoin(self._response.geturl(), url)
if self.request is not None and update_history:
self._history.append((self.request, self._response))
self._response = None
# we want self.request to be assigned even if UserAgent.open fails
self.request = self._request(url, data)
self._previous_scheme = self.request.get_type()
self._response = UserAgent.open(self, self.request, data)
if not hasattr(self._response, "seek"):
self._response = response_seek_wrapper(self._response)
self._parse_html(self._response)
return self._response
def processLink(self, link):
"""Process a link."""
url = link.absoluteURL
# Whatever will happen, we have looked at the URL
self.visited.append(url)
# Retrieve the content
try:
self.browser.open(link.callableURL)
except urllib2.HTTPError, error:
# Something went wrong with retrieving the page.
self.linkErrors += 1
self.sendMessage(
'%s (%i): %s' % (error.msg, error.code, link.callableURL), 2)
self.sendMessage('+-> Reference: ' + link.referenceURL, 2)
# Now set the error page as the response
from ClientCookie._Util import response_seek_wrapper
self.browser._response = response_seek_wrapper(error)