def test_load_unicode(self):
self.assertStrictEqual(self.load(b'u\4\0\0\0test', (2, 7)),
compat.UnicodeType(u'test'))
self.assertStrictEqual(self.load(b'u\4\0\0\0test', (3, 6)), 'test')
# This character is \u00e4 (umlaut a).
s = b'u\2\0\0\0\xc3\xa4'
if sys.version_info[0] == 2:
self.assertStrictEqual(self.load(s, (3, 6)), '\xc3\xa4')
else:
self.assertStrictEqual(self.load(s, (3, 6)), '\xe4')
def load_unicode(self):
n = self._read_long()
s = self._read(n)
# We need to convert bytes to a unicode string for any of the following:
# - We are analysing python2 code
# - We are running in a python3 host
# If we are analysing python3 code in a python2 host, we leave the string as
# a utf-8 encoded bytestring.
if (self.python_version[0] < 3 or sys.version_info[0] == 3):
s = s.decode('utf8')
if self.python_version[0] < 3:
# In Python 2, unicode and str are different classes.
s = compat.UnicodeType(s)
return s
def _get_literal_value(self, pyval):
if pyval == self.vm.lookup_builtin("__builtin__.True"):
return True
elif pyval == self.vm.lookup_builtin("__builtin__.False"):
return False
elif isinstance(pyval, str):
prefix, value = parser_constants.STRING_RE.match(
pyval).groups()[:2]
value = value[1:-1] # remove quotation marks
if "b" in prefix and not self.vm.PY2:
value = compat.bytestring(value)
elif "u" in prefix and self.vm.PY2:
value = compat.UnicodeType(value)
return value
else:
return pyval
def load_unicode(self):
n = self._read_long()
s = self._read(n)
# We need to convert bytes to a unicode string for any of the following:
# - We are analysing python2 code
# - We are running in a python3 host
# If we are analysing python3 code in a python2 host, we leave the string as
# a utf-8 encoded bytestring.
if (self.python_version[0] < 3 or sys.version_info[0] == 3):
# We use the 'backslashreplace' error mode in order to handle non-utf8
# backslash-escaped string literals correctly.
s = s.decode('utf8', 'backslashreplace')
if self.python_version[0] < 3:
# In Python 2, unicode and str are different classes.
s = compat.UnicodeType(s)
return s