def open(self, url, opts=None, data=None, referer=None, size=-1, add_headers=None): """Open URL and return unicode content""" url = list(urlparse.urlparse(url)) if opts: for key in opts: val = opts[key] if isinstance(val, unicode): opts[key] = encode(val, 'utf-8') query = [urllib.urlencode(opts)] if url[4]: query.append(url[4]) url[4] = u'&'.join(query) realurl = urlparse.urlunparse(url) if self.debug: print 'url = %r' % realurl request = urllib2.Request(realurl, data) if referer: request.add_header(u'Referer', referer) if add_headers: for item in add_headers.items(): request.add_header(*item) response = self.opener.open(request) data = response.read(size) import google if isinstance(response, google.Response): headers = None else: headers = response.headers if headers and headers.get('content-encoding') == 'gzip': data = GzipFile(fileobj=StringIO(data)).read() return encoding.convert(data, headers)
def open(self, url, opts=None, data=None, referer=None, size=-1, add_headers=None, logger=None, **kwargs): """Open URL and return unicode content""" realurl = buildurl(opts=opts, **dict(urlparse.urlparse(url)._asdict(), **kwargs)) if logger is not None: logger.debug("open url: {}".format(realurl)) request = urllib2.Request(realurl, data) if referer: request.add_header(u"Referer", referer) if add_headers: for item in add_headers.items(): request.add_header(*item) response = self.opener.open(request) data = response.read(size) import google headers = None if isinstance(response, google.Response) else response.headers if headers and headers.get("content-encoding") == "gzip": data = GzipFile(fileobj=StringIO(data)).read() return encoding.convert(data, headers)