def encodeex(encodebuf, unicodedata, errors="strict", errorcb=None,
namecb=None, ignore_error=0):
inleft = len(unicodedata)
inbuf = rffi.get_nonmoving_unicodebuffer(unicodedata)
try:
if pypy_cjk_enc_init(encodebuf, inbuf, inleft) < 0:
raise MemoryError
if ignore_error == 0:
flags = MBENC_FLUSH | MBENC_RESET
else:
flags = 0
while True:
r = pypy_cjk_enc_chunk(encodebuf, flags)
if r == 0 or r == ignore_error:
break
multibytecodec_encerror(encodebuf, r, errors,
errorcb, namecb, unicodedata)
while flags & MBENC_RESET:
r = pypy_cjk_enc_reset(encodebuf)
if r == 0:
break
multibytecodec_encerror(encodebuf, r, errors,
errorcb, namecb, unicodedata)
src = pypy_cjk_enc_outbuf(encodebuf)
length = pypy_cjk_enc_outlen(encodebuf)
return rffi.charpsize2str(src, length)
#
finally:
rffi.free_nonmoving_unicodebuffer(unicodedata, inbuf)
def unicode_encode_mbcs(p, size, errors, errorhandler=None):
dataptr = rffi.get_nonmoving_unicodebuffer(p)
try:
# first get the size of the result
if size > 0:
mbcssize = WideCharToMultiByte(CP_ACP, 0, dataptr, size, None,
0, None, None)
if mbcssize == 0:
raise rwin32.lastWindowsError()
else:
mbcssize = 0
raw_buf, gc_buf = rffi.alloc_buffer(mbcssize)
try:
# do the conversion
if WideCharToMultiByte(CP_ACP, 0, dataptr, size, raw_buf,
mbcssize, None, None) == 0:
raise rwin32.lastWindowsError()
return rffi.str_from_buffer(raw_buf, gc_buf, mbcssize,
mbcssize)
finally:
rffi.keep_buffer_alive_until_here(raw_buf, gc_buf)
finally:
rffi.free_nonmoving_unicodebuffer(p, dataptr)
def multibytecodec_decerror(decodebuf, e, errors,
errorcb, namecb, stringdata):
if e > 0:
reason = "illegal multibyte sequence"
esize = e
elif e == MBERR_TOOFEW:
reason = "incomplete multibyte sequence"
esize = pypy_cjk_dec_inbuf_remaining(decodebuf)
elif e == MBERR_NOMEMORY:
raise MemoryError
else:
raise RuntimeError
#
# compute the unicode to use as a replacement -> 'replace', and
# the current position in the input 'unicodedata' -> 'end'
start = pypy_cjk_dec_inbuf_consumed(decodebuf)
end = start + esize
if errors == "strict":
raise EncodeDecodeError(start, end, reason)
elif errors == "ignore":
replace = u""
elif errors == "replace":
replace = UNICODE_REPLACEMENT_CHARACTER
else:
assert errorcb
replace, end = errorcb(errors, namecb, reason,
stringdata, start, end)
inbuf = rffi.get_nonmoving_unicodebuffer(replace)
try:
r = pypy_cjk_dec_replace_on_error(decodebuf, inbuf, len(replace), end)
finally:
rffi.free_nonmoving_unicodebuffer(replace, inbuf)
if r == MBERR_NOMEMORY:
raise MemoryError
def unicode_encode_mbcs(p, size, errors, errorhandler=None):
dataptr = rffi.get_nonmoving_unicodebuffer(p)
try:
# first get the size of the result
if size > 0:
mbcssize = WideCharToMultiByte(CP_ACP, 0,
dataptr, size, None, 0,
None, None)
if mbcssize == 0:
raise rwin32.lastWindowsError()
else:
mbcssize = 0
raw_buf, gc_buf = rffi.alloc_buffer(mbcssize)
try:
# do the conversion
if WideCharToMultiByte(CP_ACP, 0,
dataptr, size, raw_buf, mbcssize,
None, None) == 0:
raise rwin32.lastWindowsError()
return rffi.str_from_buffer(raw_buf, gc_buf, mbcssize, mbcssize)
finally:
rffi.keep_buffer_alive_until_here(raw_buf, gc_buf)
finally:
rffi.free_nonmoving_unicodebuffer(p, dataptr)
def encode(codec, unicodedata, errors="strict", errorcb=None, namecb=None):
inleft = len(unicodedata)
inbuf = rffi.get_nonmoving_unicodebuffer(unicodedata)
try:
encodebuf = pypy_cjk_enc_init(codec, inbuf, inleft)
if not encodebuf:
raise MemoryError
try:
while True:
r = pypy_cjk_enc_chunk(encodebuf)
if r == 0:
break
multibytecodec_encerror(encodebuf, r, errors,
codec, errorcb, namecb, unicodedata)
while True:
r = pypy_cjk_enc_reset(encodebuf)
if r == 0:
break
multibytecodec_encerror(encodebuf, r, errors,
codec, errorcb, namecb, unicodedata)
src = pypy_cjk_enc_outbuf(encodebuf)
length = pypy_cjk_enc_outlen(encodebuf)
return rffi.charpsize2str(src, length)
#
finally:
pypy_cjk_enc_free(encodebuf)
#
finally:
rffi.free_nonmoving_unicodebuffer(unicodedata, inbuf)
def multibytecodec_decerror(decodebuf, e, errors,
errorcb, namecb, stringdata):
if e > 0:
reason = "illegal multibyte sequence"
esize = e
elif e == MBERR_TOOFEW:
reason = "incomplete multibyte sequence"
esize = pypy_cjk_dec_inbuf_remaining(decodebuf)
elif e == MBERR_NOMEMORY:
raise MemoryError
else:
raise RuntimeError
#
# compute the unicode to use as a replacement -> 'replace', and
# the current position in the input 'unicodedata' -> 'end'
start = pypy_cjk_dec_inbuf_consumed(decodebuf)
end = start + esize
if errors == "strict":
raise EncodeDecodeError(start, end, reason)
elif errors == "ignore":
replace = u""
elif errors == "replace":
replace = UNICODE_REPLACEMENT_CHARACTER
else:
assert errorcb
replace, end = errorcb(errors, namecb, reason,
stringdata, start, end)
inbuf = rffi.get_nonmoving_unicodebuffer(replace)
try:
r = pypy_cjk_dec_replace_on_error(decodebuf, inbuf, len(replace), end)
finally:
rffi.free_nonmoving_unicodebuffer(replace, inbuf)
if r == MBERR_NOMEMORY:
raise MemoryError
def encodeex(encodebuf, unicodedata, errors="strict", errorcb=None, namecb=None, ignore_error=0):
inleft = len(unicodedata)
inbuf = rffi.get_nonmoving_unicodebuffer(unicodedata)
try:
if pypy_cjk_enc_init(encodebuf, inbuf, inleft) < 0:
raise MemoryError
if ignore_error == 0:
flags = MBENC_FLUSH | MBENC_RESET
else:
flags = 0
while True:
r = pypy_cjk_enc_chunk(encodebuf, flags)
if r == 0 or r == ignore_error:
break
multibytecodec_encerror(encodebuf, r, errors, errorcb, namecb, unicodedata)
while flags & MBENC_RESET:
r = pypy_cjk_enc_reset(encodebuf)
if r == 0:
break
multibytecodec_encerror(encodebuf, r, errors, errorcb, namecb, unicodedata)
src = pypy_cjk_enc_outbuf(encodebuf)
length = pypy_cjk_enc_outlen(encodebuf)
return rffi.charpsize2str(src, length)
#
finally:
rffi.free_nonmoving_unicodebuffer(unicodedata, inbuf)
