def _normalize_urllib_response(response, request=None):
if six.PY2:
if not isinstance(response, urllib.addinfourl):
raise TypeError("Cannot normalize this response object")
else:
if not isinstance(response, http.client.HTTPResponse):
raise TypeError("Cannot normalize this response object")
url = response.url
status_code = response.getcode()
content_type = response.headers.get('Content-Type')
return Response(
request=request,
content=response.read(),
url=url,
status_code=status_code,
content_type=content_type,
response=response,
)
def _normalize_urllib_response(response, request=None):
if six.PY2:
if not isinstance(response, urllib.addinfourl):
raise TypeError("Cannot normalize this response object")
else:
if not isinstance(response, http.client.HTTPResponse):
raise TypeError("Cannot normalize this response object")
url = response.url
status_code = response.getcode()
content_type = response.headers.get('Content-Type')
return Response(
request=request,
content=response.read(),
url=url,
status_code=status_code,
content_type=content_type,
response=response,
)
def _do_service(self, req, timeout=10, content_to_file=True):
resp_meta = {
"content_to_file":content_to_file,
"content_path":None,
"content":None,
"error":None,
"redirect":None,
"require_auth":False,
"headers":None,
"encoding":None
}
try:
response = self.opener.open(req)
resp_meta["headers"] = response.info()
content_type = resp_meta["headers"].getheader('content-type', None)
if content_type:
_, param = cgi.parse_header(content_type)
resp_meta["encoding"] = param.get("charset", None)
if content_to_file:
temp_path = self._save_to_file(response)
if not temp_path:
resp_meta["error"] = "fail to save content to file"
else:
resp_meta["content_path"] = temp_path
else:
resp_meta["content"] = response.read()
except urllib2.HTTPError as e:
resp_meta["headers"] = e.info()
self.logger.exception("req http error")
if 300 <= e.code < 400:
resp_meta['redirect'] = e.info().getheader("Location", None)
elif e.code == 403:
resp_meta["require_auth"] = True
resp_meta["error"] = e.read()
e.close()
except urllib2.URLError as e:
self.logger.exception("req url error")
resp_meta["error"] = "url error"
return resp_meta