def unicode_urlopen(url, accept_lang=None):
"""Returns a *Unicode* file-like object for non-local documents.
Client must ensure that the URL points to non-binary data. Pass in
an Accept-Language value to configure the FancyURLopener we
use."""
opener = FancyURLopener()
if accept_lang:
opener.addheader("Accept-Language", accept_lang)
# We want to convert the bytes file-like object returned by
# urllib, which is bytes in both Python 2 and Python 3
# fortunately, and turn it into a Unicode file-like object
# with a little help from our StringIO friend.
page = opener.open(url)
encoding = page.headers['content-type']
encoding = encoding.split('charset=')
if len(encoding) > 1:
encoding = encoding[-1]
page = page.read().decode(encoding)
else:
page = page.read()
encoding = meta_encoding(page) or 'utf8'
page = page.decode(encoding)
page = StringIO(page)
return page
def unicode_urlopen(url, accept_lang=None):
"""Returns a *Unicode* file-like object for non-local documents.
Client must ensure that the URL points to non-binary data. Pass in
an Accept-Language value to configure the FancyURLopener we
use."""
opener = FancyURLopener()
if accept_lang:
opener.addheader("Accept-Language", accept_lang)
# We want to convert the bytes file-like object returned by
# urllib, which is bytes in both Python 2 and Python 3
# fortunately, and turn it into a Unicode file-like object
# with a little help from our StringIO friend.
page = opener.open(url)
encoding = page.headers['content-type']
encoding = encoding.split('charset=')
if len(encoding) > 1:
encoding = encoding[-1]
page = page.read().decode(encoding)
else:
page = page.read()
encoding = meta_encoding(page) or 'utf8'
page = page.decode(encoding)
page = StringIO(page)
return page
def get(self, url, headers=None):
o = FancyURLopener()
if headers:
for k, v in headers.items():
o.addheader(k, v)
self.req = o.open(url)
return self
def _get_sector_url(self, sector, length):
start = sector * 2048
if self._buff:
self._buff.close()
opener = FancyURLopener()
opener.http_error_206 = lambda *a, **k: None
opener.addheader("Range", "bytes=%d-%d" % (start, start + length - 1))
self._buff = opener.open(self._url)
def __load_page(self, url):
res = None
body = None
opener = FancyURLopener()
# Clear default User-Agent header which is defined in addheaders.
opener.addheaders = []
for key, value in request_headers.iteritems():
opener.addheader(key, value)
opener.addheader("Cookie", self.cookie)
try:
res = opener.open(url)
body = res.read()
except IOError, error:
logging.error(error.strerror)
def fetchURL(url, file='', params=None, headers={}, isBinary=False, encodeURL=True):
log("> bbbLib.fetchURL() %s isBinary=%s encodeURL=%s" % (url, isBinary, encodeURL))
if encodeURL:
safe_url = quote_plus(url,'/:&?=+#@')
else:
safe_url = url
success = False
data = None
if not file:
# create temp file if needed
file = xbmc.translatePath(os.path.join(os.getcwd(), "temp.html"))
# remove destination file if exists already
deleteFile(file)
# fetch from url
try:
opener = FancyURLopener()
# add headers if supplied
# if headers:
if not headers.has_key('User-Agent') and not headers.has_key('User-agent'):
headers['User-Agent'] = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'
for name, value in headers.items():
opener.addheader(name, value)
fn, resp = opener.retrieve(safe_url, file, data=params)
# print fn, resp
content_type = resp.get("Content-Type",'').lower()
# fail if expecting an image but not corrent type returned
if isBinary and (find(content_type,"text") != -1):
raise "Not Binary"
opener.close()
del opener
urlcleanup()
except IOError, errobj:
ErrorCode(errobj)
def fetchURL(url, file='', params=None, headers={}, isBinary=False, encodeURL=True):
log("> bbbLib.fetchURL() %s isBinary=%s encodeURL=%s" % (url, isBinary, encodeURL))
if encodeURL:
safe_url = quote_plus(url,'/:&?=+#@')
else:
safe_url = url
success = False
data = None
if not file:
# create temp file
file = xbmc.translatePath( "special://temp/temp.html" )
# remove destination file if exists already
deleteFile(file)
# fetch from url
try:
opener = FancyURLopener()
# add headers if supplied
if not headers.has_key('User-Agent') and not headers.has_key('User-agent'):
headers['User-Agent'] = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'
for name, value in headers.items():
opener.addheader(name, value)
fn, resp = opener.retrieve(safe_url, file, data=params)
# print fn, resp
content_type = resp.get("Content-Type",'').lower()
# fail if expecting an image but not correct type returned
if isBinary and (find(content_type,"text") != -1):
raise "Not Binary"
opener.close()
del opener
urlcleanup()
except IOError, errobj:
ErrorCode(errobj)
