def get(self, url): url = self._get_canonical_url(url) try: parsed, rv = self._cache[url] except KeyError: return None # We have already gotten this file and we've # decoded the response, so just return if parsed: return rv # Otherwise, we have a 2-tuple that needs to be applied body, encoding = rv # Our body is lazily evaluated if it # comes from libsourcemap if callable(body): body = body() body = body.decode(codec_lookup(encoding, 'utf-8').name, 'replace').split(u'\n') # Set back a marker to indicate we've parsed this url self._cache[url] = (True, body) return body
def assert_match(enc, ref=None): if ref is None: ref = enc rv = codec_lookup(enc) if rv is None: assert ref is rv is None else: assert rv.name == ref
def test_codec_lookup(): def assert_match(enc, ref=None): if ref is None: ref = enc rv = codec_lookup(enc) if rv is None: assert ref is rv is None else: assert rv.name == ref assert codec_lookup("utf-8").name == "utf-8" assert codec_lookup("utf8").name == "utf-8" assert codec_lookup("zlib").name == "utf-8" assert codec_lookup("utf16").name == "utf-16" assert codec_lookup("undefined").name == "utf-8" assert codec_lookup("undefined", default=None) is None assert codec_lookup("undefined", default="latin1").name == "iso8859-1" assert codec_lookup("zlib", default="latin1").name == "iso8859-1" assert codec_lookup("unknownshit", default="latin1").name == "iso8859-1"
def test_codec_lookup(): def assert_match(enc, ref=None): if ref is None: ref = enc rv = codec_lookup(enc) if rv is None: assert ref is rv is None else: assert rv.name == ref assert codec_lookup('utf-8').name == 'utf-8' assert codec_lookup('utf8').name == 'utf-8' if sys.version_info[:3] >= (2, 7, 12): assert codec_lookup('zlib').name == 'utf-8' assert codec_lookup('utf16').name == 'utf-16' assert codec_lookup('undefined').name == 'utf-8' assert codec_lookup('undefined', default=None) is None assert codec_lookup('undefined', default='latin1').name == 'iso8859-1' if sys.version_info[:3] >= (2, 7, 12): assert codec_lookup('zlib', default='latin1').name == 'iso8859-1' assert codec_lookup('unknownshit', default='latin1').name == 'iso8859-1'
def is_utf8(codec): name = codec_lookup(codec).name return name in ("utf-8", "ascii")
def is_utf8(codec): name = codec_lookup(codec).name return name in ('utf-8', 'ascii')