def encode(input: str, errors: str = "strict") -> typing.Tuple[bytes, int]:
"""
return an encoded version of the string as a bytes object and its length.
Parameters:
input: the string to encode
errors: same meaning as the errors argument to pythons' `encode <https://docs.python.org/3/library/codecs.html#codecs.encode>`_ method
"""
# https://github.com/google/pytype/issues/348
return codecs.utf_16_be_encode(input, errors)
def formatForPdf(self, text):
#these ones should be encoded asUTF16 minus the BOM
from codecs import utf_16_be_encode
#print 'formatting %s: %s' % (type(text), repr(text))
if type(text) is not unicode:
text = text.decode('utf8')
utfText = utf_16_be_encode(text)[0]
encoded = _escape(utfText)
#print ' encoded:',encoded
return encoded
def formatForPdf(self, text):
#these ones should be encoded asUTF16 minus the BOM
from codecs import utf_16_be_encode
#print 'formatting %s: %s' % (type(text), repr(text))
if isBytes(text):
text = text.decode('utf8')
utfText = utf_16_be_encode(text)[0]
encoded = escapePDF(utfText)
#print ' encoded:',encoded
return encoded
def pack_string(string):
'''
Packs a string into UCS-2 and prefixes it with its length as a short int.
This function can't actually handle UCS-2, therefore kick messages and
the MOTD can't contain special characters.
'''
string = u"".join(i if ord(i) < 65536 else u"?" for i in string)
return (pack(">h", len(string)) +
utf_16_be_encode(string, "replace")[0])
def test_codecs_builtins(self):
s = "abc"
encoded = codecs.utf_8_encode(s)
self.assertEqual(s, codecs.utf_8_decode(encoded[0])[0])
encoded = codecs.utf_7_encode(s)
self.assertEqual(s, codecs.utf_7_decode(encoded[0])[0])
encoded = codecs.utf_16_encode(s)
self.assertEqual(s, codecs.utf_16_decode(encoded[0])[0])
encoded = codecs.utf_16_le_encode(s)
self.assertEqual(s, codecs.utf_16_le_decode(encoded[0])[0])
encoded = codecs.utf_16_be_encode(s)
self.assertEqual(s, codecs.utf_16_be_decode(encoded[0])[0])
encoded = codecs.utf_32_encode(s)
self.assertEqual(s, codecs.utf_32_decode(encoded[0])[0])
encoded = codecs.utf_32_le_encode(s)
self.assertEqual(s, codecs.utf_32_le_decode(encoded[0])[0])
encoded = codecs.utf_32_be_encode(s)
self.assertEqual(s, codecs.utf_32_be_decode(encoded[0])[0])
encoded = codecs.utf_32_be_encode(s)
self.assertEqual(s, codecs.utf_32_be_decode(encoded[0])[0])
encoded = codecs.raw_unicode_escape_encode(s)
self.assertEqual(s, codecs.raw_unicode_escape_decode(encoded[0])[0])
encoded = codecs.unicode_escape_encode(s)
self.assertEqual(s, codecs.unicode_escape_decode(encoded[0])[0])
encoded = codecs.latin_1_encode(s)
self.assertEqual(s, codecs.latin_1_decode(encoded[0])[0])
encoded = codecs.ascii_encode(s)
self.assertEqual(s, codecs.ascii_decode(encoded[0])[0])
def ucs2_encode(data, errors="replace"):
data = u"".join(i if ord(i) < 65536 else u"?" for i in data)
return utf_16_be_encode(data, errors)
def add_str(self, st):
if len(st) == 0:
return '\x00\x00'
st = utf_16_be_encode(st)
return struct.pack("!h", st[1]) + st[0]
def encode(self, input, final = False):
return codecs.utf_16_be_encode(input, self.errors)[0]
def encode(self, input, errors='strict'):
return codecs.utf_16_be_encode(input, errors)
def update_event(self, inp=-1):
self.set_output_val(
0, codecs.utf_16_be_encode(self.input(0), self.input(1)))
def writestring(s):
s=codecs.utf_16_be_encode(s)[0]
pack("!I%ds"%len(s),len(s), s)
def test_utf_16_be_encode(self):
#sanity
new_str, size = codecs.utf_16_be_encode("abc")
self.assertEqual(new_str, '\x00a\x00b\x00c')
self.assertEqual(size, 3)
def encode(self, input, final=False):
return codecs.utf_16_be_encode(input, self.errors)[0]
]
import struct, sys, os.path, codecs
if len(sys.argv) > 1:
target = sys.argv[1]
elif os.path.exists('/home/me/Games/Newer'):
target = '/home/me/Games/Newer/ISO/files/NewerRes/Messages.bin'
else:
target = 'Messages.bin'
messageCount = len(messages)
stringOffset = (messageCount * 0xC) + 4
infoStruct = struct.Struct('>III')
headerData = bytearray(struct.pack('>I', messageCount))
stringData = bytearray()
for msgID, title, msg in messages:
titleOffset = stringOffset + len(stringData)
stringData += codecs.utf_16_be_encode(title)[0] + '\0\0'
msgOffset = stringOffset + len(stringData)
stringData += codecs.utf_16_be_encode(msg)[0] + '\0\0'
headerData += infoStruct.pack(msgID, titleOffset, msgOffset)
with open(target, 'wb') as out:
out.write(headerData)
out.write(stringData)
def add_str(self, st):
if len(st)==0:
return '\x00\x00'
st = utf_16_be_encode(st)
return struct.pack("!h", st[1]) + st[0]
def encode(self, input, errors="strict"):
# https://github.com/google/pytype/issues/348
return codecs.utf_16_be_encode(input, errors) # pytype: disable=module-attr
import struct, sys, os.path, codecs
if len(sys.argv) > 1:
target = sys.argv[1]
elif os.path.exists('/home/me/Games/Newer'):
target = '/home/me/Games/Newer/ISO/files/NewerRes/Messages.bin'
else:
target = 'Messages.bin'
messageCount = len(messages)
stringOffset = (messageCount * 0xC) + 4
infoStruct = struct.Struct('>III')
headerData = bytearray(struct.pack('>I', messageCount))
stringData = bytearray()
for msgID, title, msg in messages:
titleOffset = stringOffset + len(stringData)
stringData += codecs.utf_16_be_encode(title)[0] + '\0\0'
msgOffset = stringOffset + len(stringData)
stringData += codecs.utf_16_be_encode(msg)[0] + '\0\0'
headerData += infoStruct.pack(msgID, titleOffset, msgOffset)
with open(target, 'wb') as out:
out.write(headerData)
out.write(stringData)
def test_utf_16_be_encode(self):
#sanity
new_str, size = codecs.utf_16_be_encode("abc")
self.assertEqual(new_str, '\x00a\x00b\x00c')
self.assertEqual(size, 3)
def ucs2_encode(data, errors="replace"):
data = u"".join(i if ord(i) < 65536 else u"?" for i in data)
return utf_16_be_encode(data, errors)
def encode(self, input, errors='strict'):
return codecs.utf_16_be_encode(input, errors)
def ucs2_encode(input, errors="replace"):
input = u"".join(i if ord(i) < 65536 else u"?" for i in input)
return utf_16_be_encode(input, errors)
def test_utf_16_be_encode(self):
data, num_processed = codecs.utf_16_be_encode("abc")
self.assertEqual(data, b'\0a\0b\0c')
self.assertEqual(num_processed, 3)
def unicodeToBytes(s):
s=codecs.utf_16_be_encode(s)[0]
return struct.pack("!I%ds"%len(s),len(s), s)
def ucs2_encode(input, errors="replace"):
input = u"".join(i if ord(i) < 65536 else u"?" for i in input)
return utf_16_be_encode(input, errors)
