def build_opener(self, debug=False):
"""Create handlers with the appropriate debug level.
We intentionally create new ones because the OpenerDirector
class in urllib2 is smart enough to replace its internal
versions with ours if we pass them into the
urllib2.build_opener method. This is much easier than
trying to introspect into the OpenerDirector to find the
existing handlers.
Based on http://code.activestate.com/recipes/440574/#c1
TODO: Implement workaround for http://bugs.python.org/issue7152
"""
http_handler = HTTPHandler(debuglevel=debug)
https_handler = HTTPSHandler(debuglevel=debug)
proxy_handler = ProxyHandler(debuglevel=debug)
unknown_handler = UnknownHandler(debuglevel=debug)
http_default_error_handler = HTTPDefaultErrorHandler(debuglevel=debug)
http_redirect_handler = HTTPRedirectHandler(debuglevel=debug)
http_error_processor = HTTPErrorProcessor(debuglevel=debug)
handlers = [http_handler, https_handler, proxy_handler, \
unknown_handler, http_default_error_handler, \
http_redirect_handler, http_error_processor]
opener = build_opener(handlers)
return opener
def build_opener(source_address=None, timeout=10):
"""Function similar to ``urllib2.build_opener`` that will build
an ``OpenerDirector`` with the explicit handlers we want,
``source_address`` for binding, ``timeout`` and our custom
`User-Agent`
"""
# printer('Timeout set to %d' % timeout, debug=True)
if source_address:
source_address_tuple = (source_address, 0)
# printer('Binding to source address: %r' % (source_address_tuple,), debug=True)
else:
source_address_tuple = None
handlers = [
ProxyHandler(),
SpeedtestHTTPHandler(source_address=source_address_tuple,
timeout=timeout),
SpeedtestHTTPSHandler(source_address=source_address_tuple,
timeout=timeout),
HTTPDefaultErrorHandler(),
HTTPRedirectHandler(),
HTTPErrorProcessor()
]
opener = OpenerDirector()
opener.addheaders = [('User-agent', build_user_agent())]
for handler in handlers:
opener.add_handler(handler)
return opener
def http_response(self, request, response):
try:
return HTTPErrorProcessor.http_response(self, request, response)
except HTTPError, e:
error_json = parse_json(e.read())
if e.code == 401:
raise UnauthorizedError(error_json)
elif e.code in ( 400, 500 ):
raise BadRequestError(error_json)
else:
raise ClientException(error_json)
def http_response(self, request, response):
try:
return HTTPErrorProcessor.http_response(self, request, response)
except HTTPError, e:
error_json = parse_json(e.read())
if e.code == 401:
raise UnauthorizedError(error_json)
elif e.code in (400, 500):
raise BadRequestError(error_json)
else:
raise ClientException(error_json)
def http_response(self, request, response):
try:
return HTTPErrorProcessor.http_response(self, request, response)
except HTTPError, e:
data = e.read()
try:
error_json = parse_json(data)
except ValueError:
logging.debug('Value error when trying to parse JSON from response data:\n%s' % response)
raise
if e.code == 401:
raise UnauthorizedError(error_json)
elif e.code in ( 400, 500 ):
raise BadRequestError(error_json)
else:
raise ClientException(error_json)
def http_response(self, request, response):
try:
return HTTPErrorProcessor.http_response(self, request, response)
except HTTPError, e:
data = e.read()
try:
error_json = parse_json(data)
except ValueError:
logging.debug(
'Value error when trying to parse JSON from response data:\n%s'
% response)
raise
if e.code == 401:
raise UnauthorizedError(error_json)
elif e.code in (400, 500):
raise BadRequestError(error_json)
else:
raise ClientException(error_json)
def http_response(self, request, response):
if response.code in [200, 201, 204]:
return response
return HTTPErrorProcessor.http_response(self, request, response)
