def test_load_unicode(self): self.assertStrictEqual(self.load(b'u\4\0\0\0test', (2, 7)), compat.UnicodeType(u'test')) self.assertStrictEqual(self.load(b'u\4\0\0\0test', (3, 6)), 'test') # This character is \u00e4 (umlaut a). s = b'u\2\0\0\0\xc3\xa4' if sys.version_info[0] == 2: self.assertStrictEqual(self.load(s, (3, 6)), '\xc3\xa4') else: self.assertStrictEqual(self.load(s, (3, 6)), '\xe4')
def load_unicode(self): n = self._read_long() s = self._read(n) # We need to convert bytes to a unicode string for any of the following: # - We are analysing python2 code # - We are running in a python3 host # If we are analysing python3 code in a python2 host, we leave the string as # a utf-8 encoded bytestring. if (self.python_version[0] < 3 or sys.version_info[0] == 3): s = s.decode('utf8') if self.python_version[0] < 3: # In Python 2, unicode and str are different classes. s = compat.UnicodeType(s) return s
def _get_literal_value(self, pyval): if pyval == self.vm.lookup_builtin("__builtin__.True"): return True elif pyval == self.vm.lookup_builtin("__builtin__.False"): return False elif isinstance(pyval, str): prefix, value = parser_constants.STRING_RE.match( pyval).groups()[:2] value = value[1:-1] # remove quotation marks if "b" in prefix and not self.vm.PY2: value = compat.bytestring(value) elif "u" in prefix and self.vm.PY2: value = compat.UnicodeType(value) return value else: return pyval
def load_unicode(self): n = self._read_long() s = self._read(n) # We need to convert bytes to a unicode string for any of the following: # - We are analysing python2 code # - We are running in a python3 host # If we are analysing python3 code in a python2 host, we leave the string as # a utf-8 encoded bytestring. if (self.python_version[0] < 3 or sys.version_info[0] == 3): # We use the 'backslashreplace' error mode in order to handle non-utf8 # backslash-escaped string literals correctly. s = s.decode('utf8', 'backslashreplace') if self.python_version[0] < 3: # In Python 2, unicode and str are different classes. s = compat.UnicodeType(s) return s