def open(self, url, opts=None, data=None, referer=None, size=-1,
add_headers=None):
"""Open URL and return unicode content"""
url = list(urlparse.urlparse(url))
if opts:
for key in opts:
val = opts[key]
if isinstance(val, unicode):
opts[key] = encode(val, 'utf-8')
query = [urllib.urlencode(opts)]
if url[4]:
query.append(url[4])
url[4] = u'&'.join(query)
realurl = urlparse.urlunparse(url)
if self.debug:
print 'url = %r' % realurl
request = urllib2.Request(realurl, data)
if referer:
request.add_header(u'Referer', referer)
if add_headers:
for item in add_headers.items():
request.add_header(*item)
response = self.opener.open(request)
data = response.read(size)
import google
if isinstance(response, google.Response):
headers = None
else:
headers = response.headers
if headers and headers.get('content-encoding') == 'gzip':
data = GzipFile(fileobj=StringIO(data)).read()
return encoding.convert(data, headers)
def open(self, url, opts=None, data=None, referer=None, size=-1, add_headers=None, logger=None, **kwargs):
"""Open URL and return unicode content"""
realurl = buildurl(opts=opts, **dict(urlparse.urlparse(url)._asdict(), **kwargs))
if logger is not None:
logger.debug("open url: {}".format(realurl))
request = urllib2.Request(realurl, data)
if referer:
request.add_header(u"Referer", referer)
if add_headers:
for item in add_headers.items():
request.add_header(*item)
response = self.opener.open(request)
data = response.read(size)
import google
headers = None if isinstance(response, google.Response) else response.headers
if headers and headers.get("content-encoding") == "gzip":
data = GzipFile(fileobj=StringIO(data)).read()
return encoding.convert(data, headers)
def open(self,
url,
opts=None,
data=None,
referer=None,
size=-1,
add_headers=None):
"""Open URL and return unicode content"""
url = list(urlparse.urlparse(url))
if opts:
for key in opts:
val = opts[key]
if isinstance(val, unicode):
opts[key] = encode(val, 'utf-8')
query = [urllib.urlencode(opts)]
if url[4]:
query.append(url[4])
url[4] = u'&'.join(query)
realurl = urlparse.urlunparse(url)
if self.debug:
print 'url = %r' % realurl
request = urllib2.Request(realurl, data)
if referer:
request.add_header(u'Referer', referer)
if add_headers:
for item in add_headers.items():
request.add_header(*item)
response = self.opener.open(request)
data = response.read(size)
import google
if isinstance(response, google.Response):
headers = None
else:
headers = response.headers
if headers and headers.get('content-encoding') == 'gzip':
data = GzipFile(fileobj=StringIO(data)).read()
return encoding.convert(data, headers)
