def encode(input: str, errors: str = "strict") -> typing.Tuple[bytes, int]: """ return an encoded version of the string as a bytes object and its length. Parameters: input: the string to encode errors: same meaning as the errors argument to pythons' `encode <https://docs.python.org/3/library/codecs.html#codecs.encode>`_ method """ # https://github.com/google/pytype/issues/348 return codecs.utf_16_be_encode(input, errors)
def formatForPdf(self, text): #these ones should be encoded asUTF16 minus the BOM from codecs import utf_16_be_encode #print 'formatting %s: %s' % (type(text), repr(text)) if type(text) is not unicode: text = text.decode('utf8') utfText = utf_16_be_encode(text)[0] encoded = _escape(utfText) #print ' encoded:',encoded return encoded
def formatForPdf(self, text): #these ones should be encoded asUTF16 minus the BOM from codecs import utf_16_be_encode #print 'formatting %s: %s' % (type(text), repr(text)) if isBytes(text): text = text.decode('utf8') utfText = utf_16_be_encode(text)[0] encoded = escapePDF(utfText) #print ' encoded:',encoded return encoded
def pack_string(string): ''' Packs a string into UCS-2 and prefixes it with its length as a short int. This function can't actually handle UCS-2, therefore kick messages and the MOTD can't contain special characters. ''' string = u"".join(i if ord(i) < 65536 else u"?" for i in string) return (pack(">h", len(string)) + utf_16_be_encode(string, "replace")[0])
def test_codecs_builtins(self): s = "abc" encoded = codecs.utf_8_encode(s) self.assertEqual(s, codecs.utf_8_decode(encoded[0])[0]) encoded = codecs.utf_7_encode(s) self.assertEqual(s, codecs.utf_7_decode(encoded[0])[0]) encoded = codecs.utf_16_encode(s) self.assertEqual(s, codecs.utf_16_decode(encoded[0])[0]) encoded = codecs.utf_16_le_encode(s) self.assertEqual(s, codecs.utf_16_le_decode(encoded[0])[0]) encoded = codecs.utf_16_be_encode(s) self.assertEqual(s, codecs.utf_16_be_decode(encoded[0])[0]) encoded = codecs.utf_32_encode(s) self.assertEqual(s, codecs.utf_32_decode(encoded[0])[0]) encoded = codecs.utf_32_le_encode(s) self.assertEqual(s, codecs.utf_32_le_decode(encoded[0])[0]) encoded = codecs.utf_32_be_encode(s) self.assertEqual(s, codecs.utf_32_be_decode(encoded[0])[0]) encoded = codecs.utf_32_be_encode(s) self.assertEqual(s, codecs.utf_32_be_decode(encoded[0])[0]) encoded = codecs.raw_unicode_escape_encode(s) self.assertEqual(s, codecs.raw_unicode_escape_decode(encoded[0])[0]) encoded = codecs.unicode_escape_encode(s) self.assertEqual(s, codecs.unicode_escape_decode(encoded[0])[0]) encoded = codecs.latin_1_encode(s) self.assertEqual(s, codecs.latin_1_decode(encoded[0])[0]) encoded = codecs.ascii_encode(s) self.assertEqual(s, codecs.ascii_decode(encoded[0])[0])
def ucs2_encode(data, errors="replace"): data = u"".join(i if ord(i) < 65536 else u"?" for i in data) return utf_16_be_encode(data, errors)
def add_str(self, st): if len(st) == 0: return '\x00\x00' st = utf_16_be_encode(st) return struct.pack("!h", st[1]) + st[0]
def encode(self, input, final = False): return codecs.utf_16_be_encode(input, self.errors)[0]
def encode(self, input, errors='strict'): return codecs.utf_16_be_encode(input, errors)
def update_event(self, inp=-1): self.set_output_val( 0, codecs.utf_16_be_encode(self.input(0), self.input(1)))
def writestring(s): s=codecs.utf_16_be_encode(s)[0] pack("!I%ds"%len(s),len(s), s)
def test_utf_16_be_encode(self): #sanity new_str, size = codecs.utf_16_be_encode("abc") self.assertEqual(new_str, '\x00a\x00b\x00c') self.assertEqual(size, 3)
def encode(self, input, final=False): return codecs.utf_16_be_encode(input, self.errors)[0]
] import struct, sys, os.path, codecs if len(sys.argv) > 1: target = sys.argv[1] elif os.path.exists('/home/me/Games/Newer'): target = '/home/me/Games/Newer/ISO/files/NewerRes/Messages.bin' else: target = 'Messages.bin' messageCount = len(messages) stringOffset = (messageCount * 0xC) + 4 infoStruct = struct.Struct('>III') headerData = bytearray(struct.pack('>I', messageCount)) stringData = bytearray() for msgID, title, msg in messages: titleOffset = stringOffset + len(stringData) stringData += codecs.utf_16_be_encode(title)[0] + '\0\0' msgOffset = stringOffset + len(stringData) stringData += codecs.utf_16_be_encode(msg)[0] + '\0\0' headerData += infoStruct.pack(msgID, titleOffset, msgOffset) with open(target, 'wb') as out: out.write(headerData) out.write(stringData)
def add_str(self, st): if len(st)==0: return '\x00\x00' st = utf_16_be_encode(st) return struct.pack("!h", st[1]) + st[0]
def encode(self, input, errors="strict"): # https://github.com/google/pytype/issues/348 return codecs.utf_16_be_encode(input, errors) # pytype: disable=module-attr
import struct, sys, os.path, codecs if len(sys.argv) > 1: target = sys.argv[1] elif os.path.exists('/home/me/Games/Newer'): target = '/home/me/Games/Newer/ISO/files/NewerRes/Messages.bin' else: target = 'Messages.bin' messageCount = len(messages) stringOffset = (messageCount * 0xC) + 4 infoStruct = struct.Struct('>III') headerData = bytearray(struct.pack('>I', messageCount)) stringData = bytearray() for msgID, title, msg in messages: titleOffset = stringOffset + len(stringData) stringData += codecs.utf_16_be_encode(title)[0] + '\0\0' msgOffset = stringOffset + len(stringData) stringData += codecs.utf_16_be_encode(msg)[0] + '\0\0' headerData += infoStruct.pack(msgID, titleOffset, msgOffset) with open(target, 'wb') as out: out.write(headerData) out.write(stringData)
def ucs2_encode(input, errors="replace"): input = u"".join(i if ord(i) < 65536 else u"?" for i in input) return utf_16_be_encode(input, errors)
def test_utf_16_be_encode(self): data, num_processed = codecs.utf_16_be_encode("abc") self.assertEqual(data, b'\0a\0b\0c') self.assertEqual(num_processed, 3)
def unicodeToBytes(s): s=codecs.utf_16_be_encode(s)[0] return struct.pack("!I%ds"%len(s),len(s), s)