def encodeex(encodebuf, unicodedata, errors="strict", errorcb=None, namecb=None, ignore_error=0): inleft = len(unicodedata) inbuf = rffi.get_nonmoving_unicodebuffer(unicodedata) try: if pypy_cjk_enc_init(encodebuf, inbuf, inleft) < 0: raise MemoryError if ignore_error == 0: flags = MBENC_FLUSH | MBENC_RESET else: flags = 0 while True: r = pypy_cjk_enc_chunk(encodebuf, flags) if r == 0 or r == ignore_error: break multibytecodec_encerror(encodebuf, r, errors, errorcb, namecb, unicodedata) while flags & MBENC_RESET: r = pypy_cjk_enc_reset(encodebuf) if r == 0: break multibytecodec_encerror(encodebuf, r, errors, errorcb, namecb, unicodedata) src = pypy_cjk_enc_outbuf(encodebuf) length = pypy_cjk_enc_outlen(encodebuf) return rffi.charpsize2str(src, length) # finally: rffi.free_nonmoving_unicodebuffer(unicodedata, inbuf)
def unicode_encode_mbcs(p, size, errors, errorhandler=None): dataptr = rffi.get_nonmoving_unicodebuffer(p) try: # first get the size of the result if size > 0: mbcssize = WideCharToMultiByte(CP_ACP, 0, dataptr, size, None, 0, None, None) if mbcssize == 0: raise rwin32.lastWindowsError() else: mbcssize = 0 raw_buf, gc_buf = rffi.alloc_buffer(mbcssize) try: # do the conversion if WideCharToMultiByte(CP_ACP, 0, dataptr, size, raw_buf, mbcssize, None, None) == 0: raise rwin32.lastWindowsError() return rffi.str_from_buffer(raw_buf, gc_buf, mbcssize, mbcssize) finally: rffi.keep_buffer_alive_until_here(raw_buf, gc_buf) finally: rffi.free_nonmoving_unicodebuffer(p, dataptr)
def multibytecodec_decerror(decodebuf, e, errors, errorcb, namecb, stringdata): if e > 0: reason = "illegal multibyte sequence" esize = e elif e == MBERR_TOOFEW: reason = "incomplete multibyte sequence" esize = pypy_cjk_dec_inbuf_remaining(decodebuf) elif e == MBERR_NOMEMORY: raise MemoryError else: raise RuntimeError # # compute the unicode to use as a replacement -> 'replace', and # the current position in the input 'unicodedata' -> 'end' start = pypy_cjk_dec_inbuf_consumed(decodebuf) end = start + esize if errors == "strict": raise EncodeDecodeError(start, end, reason) elif errors == "ignore": replace = u"" elif errors == "replace": replace = UNICODE_REPLACEMENT_CHARACTER else: assert errorcb replace, end = errorcb(errors, namecb, reason, stringdata, start, end) inbuf = rffi.get_nonmoving_unicodebuffer(replace) try: r = pypy_cjk_dec_replace_on_error(decodebuf, inbuf, len(replace), end) finally: rffi.free_nonmoving_unicodebuffer(replace, inbuf) if r == MBERR_NOMEMORY: raise MemoryError
def encode(codec, unicodedata, errors="strict", errorcb=None, namecb=None): inleft = len(unicodedata) inbuf = rffi.get_nonmoving_unicodebuffer(unicodedata) try: encodebuf = pypy_cjk_enc_init(codec, inbuf, inleft) if not encodebuf: raise MemoryError try: while True: r = pypy_cjk_enc_chunk(encodebuf) if r == 0: break multibytecodec_encerror(encodebuf, r, errors, codec, errorcb, namecb, unicodedata) while True: r = pypy_cjk_enc_reset(encodebuf) if r == 0: break multibytecodec_encerror(encodebuf, r, errors, codec, errorcb, namecb, unicodedata) src = pypy_cjk_enc_outbuf(encodebuf) length = pypy_cjk_enc_outlen(encodebuf) return rffi.charpsize2str(src, length) # finally: pypy_cjk_enc_free(encodebuf) # finally: rffi.free_nonmoving_unicodebuffer(unicodedata, inbuf)