def unicode_urlopen(url, accept_lang=None): """Returns a *Unicode* file-like object for non-local documents. Client must ensure that the URL points to non-binary data. Pass in an Accept-Language value to configure the FancyURLopener we use.""" opener = FancyURLopener() if accept_lang: opener.addheader("Accept-Language", accept_lang) # We want to convert the bytes file-like object returned by # urllib, which is bytes in both Python 2 and Python 3 # fortunately, and turn it into a Unicode file-like object # with a little help from our StringIO friend. page = opener.open(url) encoding = page.headers['content-type'] encoding = encoding.split('charset=') if len(encoding) > 1: encoding = encoding[-1] page = page.read().decode(encoding) else: page = page.read() encoding = meta_encoding(page) or 'utf8' page = page.decode(encoding) page = StringIO(page) return page
def get(self, url, headers=None): o = FancyURLopener() if headers: for k, v in headers.items(): o.addheader(k, v) self.req = o.open(url) return self
def _get_sector_url(self, sector, length): start = sector * 2048 if self._buff: self._buff.close() opener = FancyURLopener() opener.http_error_206 = lambda *a, **k: None opener.addheader("Range", "bytes=%d-%d" % (start, start + length - 1)) self._buff = opener.open(self._url)
def __load_page(self, url): res = None body = None opener = FancyURLopener() # Clear default User-Agent header which is defined in addheaders. opener.addheaders = [] for key, value in request_headers.iteritems(): opener.addheader(key, value) opener.addheader("Cookie", self.cookie) try: res = opener.open(url) body = res.read() except IOError, error: logging.error(error.strerror)
def fetchURL(url, file='', params=None, headers={}, isBinary=False, encodeURL=True): log("> bbbLib.fetchURL() %s isBinary=%s encodeURL=%s" % (url, isBinary, encodeURL)) if encodeURL: safe_url = quote_plus(url,'/:&?=+#@') else: safe_url = url success = False data = None if not file: # create temp file if needed file = xbmc.translatePath(os.path.join(os.getcwd(), "temp.html")) # remove destination file if exists already deleteFile(file) # fetch from url try: opener = FancyURLopener() # add headers if supplied # if headers: if not headers.has_key('User-Agent') and not headers.has_key('User-agent'): headers['User-Agent'] = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)' for name, value in headers.items(): opener.addheader(name, value) fn, resp = opener.retrieve(safe_url, file, data=params) # print fn, resp content_type = resp.get("Content-Type",'').lower() # fail if expecting an image but not corrent type returned if isBinary and (find(content_type,"text") != -1): raise "Not Binary" opener.close() del opener urlcleanup() except IOError, errobj: ErrorCode(errobj)
def fetchURL(url, file='', params=None, headers={}, isBinary=False, encodeURL=True): log("> bbbLib.fetchURL() %s isBinary=%s encodeURL=%s" % (url, isBinary, encodeURL)) if encodeURL: safe_url = quote_plus(url,'/:&?=+#@') else: safe_url = url success = False data = None if not file: # create temp file file = xbmc.translatePath( "special://temp/temp.html" ) # remove destination file if exists already deleteFile(file) # fetch from url try: opener = FancyURLopener() # add headers if supplied if not headers.has_key('User-Agent') and not headers.has_key('User-agent'): headers['User-Agent'] = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)' for name, value in headers.items(): opener.addheader(name, value) fn, resp = opener.retrieve(safe_url, file, data=params) # print fn, resp content_type = resp.get("Content-Type",'').lower() # fail if expecting an image but not correct type returned if isBinary and (find(content_type,"text") != -1): raise "Not Binary" opener.close() del opener urlcleanup() except IOError, errobj: ErrorCode(errobj)