def __init__(self): f = open("data/scripts/custom/easyEnchant/itemname-tw.txt", "r") f.readline() for line in f: temp = line.split("\t") if temp[0].isdigit(): self.itemNameTable[int(temp[0])] = [_codecs.utf_8_decode(temp[1])[0],_codecs.utf_8_decode(temp[2])[0]] f.close()
def internal_cp34951(sample1): AreEqual(codecs.utf_8_decode(sample1), ('12\u20ac\x0a', 6)) sample1 = sample1[:-1] # 12<euro> AreEqual(codecs.utf_8_decode(sample1), ('12\u20ac', 5)) sample1 = sample1[:-1] # 12<uncomplete euro> AreEqual(codecs.utf_8_decode(sample1), ('12', 2)) sample1 = sample1 + 'x7f' # makes it invalid try: r = codecs.utf_8_decode(sample1) Assert(False, "expected UncodeDecodeError not raised") except Exception as e: AreEqual(type(e), UnicodeDecodeError)
def test_utf_8_decode(): ''' ''' #sanity new_str, size = codecs.utf_8_decode("abc") AreEqual(new_str, u'abc') AreEqual(size, 3)
def test_partial_utf8(self): import _codecs encoding = 'utf-8' check_partial = [ u"\x00", u"\x00", u"\x00\xff", u"\x00\xff", u"\x00\xff\u07ff", u"\x00\xff\u07ff", u"\x00\xff\u07ff", u"\x00\xff\u07ff\u0800", u"\x00\xff\u07ff\u0800", u"\x00\xff\u07ff\u0800", u"\x00\xff\u07ff\u0800\uffff", u"\x00\xff\u07ff\u0800\uffff", u"\x00\xff\u07ff\u0800\uffff", u"\x00\xff\u07ff\u0800\uffff", u"\x00\xff\u07ff\u0800\uffff\U00010000", ] buffer = '' result = u"" for (c, partialresult) in zip(u"\x00\xff\u07ff\u0800\uffff\U00010000".encode(encoding), check_partial): buffer += c res = _codecs.utf_8_decode(buffer,'strict',False) if res[1] >0 : buffer = '' result += res[0] assert result == partialresult
def test_partial_utf8(self): import _codecs encoding = 'utf-8' check_partial = [ u"\x00", u"\x00", u"\x00\xff", u"\x00\xff", u"\x00\xff\u07ff", u"\x00\xff\u07ff", u"\x00\xff\u07ff", u"\x00\xff\u07ff\u0800", u"\x00\xff\u07ff\u0800", u"\x00\xff\u07ff\u0800", u"\x00\xff\u07ff\u0800\uffff", ] buffer = '' result = u"" for (c, partialresult) in zip(u"\x00\xff\u07ff\u0800\uffff".encode(encoding), check_partial): buffer += c res = _codecs.utf_8_decode(buffer,'strict',False) if res[1] >0 : buffer = '' result += res[0] assert result == partialresult
def __init__(self): f = open("data/scripts/custom/rbtools/npcname-tw.txt", "r") f.readline() for line in f: temp = line.split("\t") if temp[0].isdigit(): self.npcNameTable[int(temp[0])] = _codecs.utf_8_decode(temp[1])[0] f.close()
def readUTF8(self): l = self.readUShort() if not self.has(l): raise ReadException() ret = utf_8_decode(self.data[self.index:self.index + l])[0] self.advance(l) return ret
def __init__(self): f = open("data/scripts/custom/rbtools/npcname-tw.txt", "r") f.readline() for line in f: temp = line.split("\t") if temp[0].isdigit(): self.npcNameTable[int(temp[0])] = _codecs.utf_8_decode( temp[1])[0] f.close()
def load_unicode(self, level): """ Unicode is stored as: - 4 bytes - n - number of UTF8 bytes to follow - n bytes - UTF8-encoded string """ self._logger.unicode(level) self._logger.field(level, self._offset, 'n') n = self.r_long() self._logger.value(level, n) self._logger.field(level, self._offset, 's') s = self._read(n) self._logger.value(level, '%s' % s) #ret = s.decode('utf8') ret, len_ret = utf_8_decode(s) return ret
def load_unicode(self, level): """ Unicode is stored as: - 4 bytes - n - number of UTF8 bytes to follow - n bytes - UTF8-encoded string """ self._logger.unicode(level) self._logger.field(level, self._offset, "n") n = self.r_long() self._logger.value(level, n) self._logger.field(level, self._offset, "s") s = self._read(n) self._logger.value(level, "%s" % s) # ret = s.decode('utf8') ret, len_ret = utf_8_decode(s) return ret
def handle(self, exchange): if not exchange.getRemoteAddress().getAddress() == InetAddress.getByAddress([127, 0, 0, 1]): exchange.sendResponseHeaders(403, 0) exchange.close() return try: i = exchange.getRequestBody() ibuff = "" while True: temp = i.read() if temp == -1: break ibuff += chr(temp) ibuff = _codecs.utf_8_decode(ibuff)[0] self.handleImpl(exchange, ibuff) except: exchange.sendResponseHeaders(500, 0) exchange.close()
def test_decode_bytearray(self): import _codecs b = bytearray() assert _codecs.ascii_decode(b) == (u'', 0) assert _codecs.latin_1_decode(b) == (u'', 0) assert _codecs.utf_7_decode(b) == (u'', 0) assert _codecs.utf_8_decode(b) == (u'', 0) assert _codecs.utf_16_be_decode(b) == (u'', 0) assert _codecs.utf_16_decode(b) == (u'', 0) assert _codecs.utf_16_le_decode(b) == (u'', 0) assert _codecs.utf_16_ex_decode(b) == (u'', 0, 0) assert _codecs.utf_32_decode(b) == (u'', 0) assert _codecs.utf_32_be_decode(b) == (u'', 0) assert _codecs.utf_32_le_decode(b) == (u'', 0) assert _codecs.utf_32_ex_decode(b) == (u'', 0, 0) assert _codecs.charmap_decode(b) == (u'', 0) assert _codecs.unicode_escape_decode(b) == (u'', 0) assert _codecs.raw_unicode_escape_decode(b) == (u'', 0) assert _codecs.unicode_internal_decode(b) == (u'', 0)
def handle(self, exchange): if not exchange.getRemoteAddress().getAddress( ) == InetAddress.getByAddress([127, 0, 0, 1]): exchange.sendResponseHeaders(403, 0) exchange.close() return try: i = exchange.getRequestBody() ibuff = "" while True: temp = i.read() if temp == -1: break ibuff += chr(temp) ibuff = _codecs.utf_8_decode(ibuff)[0] self.handleImpl(exchange, ibuff) except: exchange.sendResponseHeaders(500, 0) exchange.close()
def load_unicode(self): n = self.r_long() s = self._read(n) #ret = s.decode('utf8') ret, len_ret = utf_8_decode(s) return ret