Python to_unicode Examples

Example #1

File: test_encoding.py Project: zanachka/w3lib

 def test_python_crash(self):
     import random
     from io import BytesIO
     random.seed(42)
     buf = BytesIO()
     for i in range(150000):
         buf.write(bytes([random.randint(0, 255)]))
     to_unicode(buf.getvalue(), 'utf-16-le')
     to_unicode(buf.getvalue(), 'utf-16-be')
     to_unicode(buf.getvalue(), 'utf-32-le')
     to_unicode(buf.getvalue(), 'utf-32-be')

Example #2

File: test_encoding.py Project: scrapy/w3lib

 def test_python_crash(self):
     import random
     from io import BytesIO
     random.seed(42)
     buf = BytesIO()
     for i in range(150000):
         buf.write(bytes([random.randint(0, 255)]))
     to_unicode(buf.getvalue(), 'utf-16-le')
     to_unicode(buf.getvalue(), 'utf-16-be')
     to_unicode(buf.getvalue(), 'utf-32-le')
     to_unicode(buf.getvalue(), 'utf-32-be')

Example #3

File: test_encoding.py Project: azizur77/w3lib

 def test_invalid_utf8(self):
     self.assertEqual(to_unicode(b'\xc2\xc2\xa3', 'utf-8'), u'\ufffd\xa3')

Example #4

File: test_encoding.py Project: azizur77/w3lib

 def test_utf8(self):
     self.assertEqual(to_unicode(b'\xc2\xa3', 'utf-8'), u'\xa3')

Example #5

File: html.py Project: AaronChiu2017/scrapy_redis_splash_spider

def bytes_to_unicode(bytes, encoding):
    """
	将给定的bytes，通过encoding转换成unicode
	"""
    return to_unicode(bytes, encoding)

Example #6

 def body_as_unicode(self):
     """Return body as unicode"""
     if self._cached_ubody is None:
         self._cached_ubody = to_unicode(self.body, self.encoding)
     return self._cached_ubody

Example #7

File: test_encoding.py Project: Dior222/w3lib

 def test_invalid_utf8(self):
     self.assertEqual(to_unicode('\xc2\xc2\xa3', 'utf-8'), u'\ufffd\xa3')

Example #8

File: test_encoding.py Project: Dior222/w3lib

 def test_utf8(self):
     self.assertEqual(to_unicode('\xc2\xa3', 'utf-8'), u'\xa3')

Example #9

File: text.py Project: LucianU/scrapy

 def body_as_unicode(self):
     """Return body as unicode"""
     if self._cached_ubody is None:
         self._cached_ubody = to_unicode(self.body, self.encoding)
     return self._cached_ubody

Example #10

 def test_invalid_utf8(self):
     self.assertEqual(to_unicode(b"\xc2\xc2\xa3", "utf-8"), "\ufffd\xa3")

Example #11

 def test_utf8(self):
     self.assertEqual(to_unicode(b"\xc2\xa3", "utf-8"), "\xa3")