def PyUnicode_EncodeDecimal(space, s, length, output, llerrors):
"""Takes a Unicode string holding a decimal value and writes it
into an output buffer using standard ASCII digit codes.
The output buffer has to provide at least length+1 bytes of
storage area. The output string is 0-terminated.
The encoder converts whitespace to ' ', decimal characters to
their corresponding ASCII digit and all other Latin-1 characters
except \0 as-is. Characters outside this range (Unicode ordinals
1-256) are treated as errors. This includes embedded NULL bytes.
Returns 0 on success, -1 on failure.
"""
u = rffi.wcharpsize2unicode(s, length)
if llerrors:
errors = rffi.charp2str(llerrors)
else:
errors = None
state = space.fromcache(CodecState)
result = runicode.unicode_encode_decimal(u, length, errors,
state.encode_error_handler)
i = len(result)
output[i] = '\0'
i -= 1
while i >= 0:
output[i] = result[i]
i -= 1
return 0
def llimpl_FormatErrorW(code):
"Return a unicode message corresponding to the given Windows error code."
buf = lltype.malloc(rffi.CWCHARPP.TO, 1, flavor='raw')
buf[0] = lltype.nullptr(rffi.CWCHARP.TO)
try:
msglen = FormatMessageW(FORMAT_MESSAGE_ALLOCATE_BUFFER |
FORMAT_MESSAGE_FROM_SYSTEM |
FORMAT_MESSAGE_IGNORE_INSERTS,
None,
rffi.cast(DWORD, code),
DEFAULT_LANGUAGE,
rffi.cast(rffi.CWCHARP, buf),
0, None)
buflen = intmask(msglen)
# remove trailing cr/lf and dots
s_buf = buf[0]
while buflen > 0 and (ord(s_buf[buflen - 1]) <= ord(' ') or
s_buf[buflen - 1] == u'.'):
buflen -= 1
if buflen <= 0:
result = u'Windows Error %d' % (code,)
else:
result = rffi.wcharpsize2unicode(s_buf, buflen)
finally:
LocalFree(rffi.cast(rffi.VOIDP, buf[0]))
lltype.free(buf, flavor='raw')
return result
def PyUnicode_EncodeDecimal(space, s, length, output, llerrors):
"""Takes a Unicode string holding a decimal value and writes it
into an output buffer using standard ASCII digit codes.
The output buffer has to provide at least length+1 bytes of
storage area. The output string is 0-terminated.
The encoder converts whitespace to ' ', decimal characters to
their corresponding ASCII digit and all other Latin-1 characters
except \0 as-is. Characters outside this range (Unicode ordinals
1-256) are treated as errors. This includes embedded NULL bytes.
Returns 0 on success, -1 on failure.
"""
u = rffi.wcharpsize2unicode(s, length)
if llerrors:
errors = rffi.charp2str(llerrors)
else:
errors = None
state = space.fromcache(CodecState)
result = runicode.unicode_encode_decimal(u, length, errors,
state.encode_error_handler)
i = len(result)
output[i] = '\0'
i -= 1
while i >= 0:
output[i] = result[i]
i -= 1
return 0
def PyLong_FromUnicode(space, u, length, base):
"""Convert a sequence of Unicode digits to a Python long integer value.
The first parameter, u, points to the first character of the Unicode
string, length gives the number of characters, and base is the radix
for the conversion. The radix must be in the range [2, 36]; if it is
out of range, ValueError will be raised."""
w_value = space.newunicode(rffi.wcharpsize2unicode(u, length))
return PyLong_FromUnicodeObject(space, w_value, base)
def test_copy(self, space, api):
w_x = space.wrap(u"abcd\u0660")
count1 = space.int_w(space.len(w_x))
target_chunk = lltype.malloc(rffi.CWCHARP.TO, count1, flavor='raw')
x_chunk = api.PyUnicode_AS_UNICODE(w_x)
api.Py_UNICODE_COPY(target_chunk, x_chunk, 4)
w_y = space.wrap(rffi.wcharpsize2unicode(target_chunk, 4))
assert space.eq_w(w_y, space.wrap(u"abcd"))
size = api.PyUnicode_GET_SIZE(w_x)
api.Py_UNICODE_COPY(target_chunk, x_chunk, size)
w_y = space.wrap(rffi.wcharpsize2unicode(target_chunk, size))
assert space.eq_w(w_y, w_x)
lltype.free(target_chunk, flavor='raw')
def PyLong_FromUnicode(space, u, length, base):
"""Convert a sequence of Unicode digits to a Python long integer value.
The first parameter, u, points to the first character of the Unicode
string, length gives the number of characters, and base is the radix
for the conversion. The radix must be in the range [2, 36]; if it is
out of range, ValueError will be raised."""
w_value = space.wrap(rffi.wcharpsize2unicode(u, length))
w_base = space.wrap(rffi.cast(lltype.Signed, base))
return space.call_function(space.w_int, w_value, w_base)
def test_copy(self, space, api):
w_x = space.wrap(u"abcd\u0660")
target_chunk, _ = rffi.alloc_unicodebuffer(space.int_w(space.len(w_x)))
#lltype.malloc(Py_UNICODE, space.int_w(space.len(w_x)), flavor='raw')
x_chunk = api.PyUnicode_AS_UNICODE(w_x)
api.Py_UNICODE_COPY(target_chunk, x_chunk, 4)
w_y = space.wrap(rffi.wcharpsize2unicode(target_chunk, 4))
assert space.eq_w(w_y, space.wrap(u"abcd"))
size = api.PyUnicode_GET_SIZE(w_x)
api.Py_UNICODE_COPY(target_chunk, x_chunk, size)
w_y = space.wrap(rffi.wcharpsize2unicode(target_chunk, size))
assert space.eq_w(w_y, w_x)
lltype.free(target_chunk, flavor='raw')
def test_copy(self, space):
w_x = space.wrap(u"abcd\u0660")
count1 = space.int_w(space.len(w_x))
target_chunk = lltype.malloc(rffi.CWCHARP.TO, count1, flavor='raw')
x_chunk = PyUnicode_AsUnicode(space, w_x)
Py_UNICODE_COPY(space, target_chunk, x_chunk, 4)
w_y = space.wrap(rffi.wcharpsize2unicode(target_chunk, 4))
assert space.eq_w(w_y, space.wrap(u"abcd"))
size = get_wsize(as_pyobj(space, w_x))
Py_UNICODE_COPY(space, target_chunk, x_chunk, size)
w_y = space.wrap(rffi.wcharpsize2unicode(target_chunk, size))
assert space.eq_w(w_y, w_x)
lltype.free(target_chunk, flavor='raw')
def PyLong_FromUnicode(space, u, length, base):
"""Convert a sequence of Unicode digits to a Python long integer value.
The first parameter, u, points to the first character of the Unicode
string, length gives the number of characters, and base is the radix
for the conversion. The radix must be in the range [2, 36]; if it is
out of range, ValueError will be raised."""
w_value = space.wrap(rffi.wcharpsize2unicode(u, length))
w_base = space.wrap(rffi.cast(lltype.Signed, base))
return space.call_function(space.w_long, w_value, w_base)
def PyUnicode_EncodeXXX(space, s, size, errors):
"""Encode the Py_UNICODE buffer of the given size and return a
Python string object. Return NULL if an exception was raised
by the codec."""
w_u = space.wrap(rffi.wcharpsize2unicode(s, size))
if errors:
w_errors = space.wrap(rffi.charp2str(errors))
else:
w_errors = None
return space.call_method(w_u, 'encode', space.wrap(encoding), w_errors)
def unicode_realize(space, py_obj):
"""
Creates the unicode in the interpreter. The PyUnicodeObject buffer must not
be modified after this call.
"""
py_uni = rffi.cast(PyUnicodeObject, py_obj)
s = rffi.wcharpsize2unicode(py_uni.c_buffer, py_uni.c_size)
w_obj = space.wrap(s)
track_reference(space, py_obj, w_obj)
return w_obj
def unicode_realize(space, py_obj):
"""
Creates the unicode in the interpreter. The PyUnicodeObject buffer must not
be modified after this call.
"""
py_uni = rffi.cast(PyUnicodeObject, py_obj)
s = rffi.wcharpsize2unicode(py_uni.c_buffer, py_uni.c_size)
w_obj = space.wrap(s)
track_reference(space, py_obj, w_obj)
return w_obj
def PyUnicode_EncodeXXX(space, s, size, errors):
"""Encode the Py_UNICODE buffer of the given size and return a
Python string object. Return NULL if an exception was raised
by the codec."""
w_u = space.newunicode(rffi.wcharpsize2unicode(s, size))
if errors:
w_errors = space.newtext(rffi.charp2str(errors))
else:
w_errors = None
return space.call_method(w_u, 'encode', space.newtext(encoding), w_errors)
def unicode_realize(space, py_obj):
"""
Creates the unicode in the interpreter. The PyUnicodeObject buffer must not
be modified after this call.
"""
s = rffi.wcharpsize2unicode(get_wbuffer(py_obj), get_wsize(py_obj))
w_type = from_ref(space, rffi.cast(PyObject, py_obj.c_ob_type))
w_obj = space.allocate_instance(unicodeobject.W_UnicodeObject, w_type)
w_obj.__init__(s)
track_reference(space, py_obj, w_obj)
return w_obj
def PyUnicode_FromUnicode(space, wchar_p, length):
"""Create a Unicode Object from the Py_UNICODE buffer u of the given size. u
may be NULL which causes the contents to be undefined. It is the user's
responsibility to fill in the needed data. The buffer is copied into the new
object. If the buffer is not NULL, the return value might be a shared object.
Therefore, modification of the resulting Unicode object is only allowed when u
is NULL."""
if wchar_p:
s = rffi.wcharpsize2unicode(wchar_p, length)
return make_ref(space, space.wrap(s))
else:
return rffi.cast(PyObject, new_empty_unicode(space, length))
def PyUnicode_FromUnicode(space, wchar_p, length):
"""Create a Unicode Object from the Py_UNICODE buffer u of the given size. u
may be NULL which causes the contents to be undefined. It is the user's
responsibility to fill in the needed data. The buffer is copied into the new
object. If the buffer is not NULL, the return value might be a shared object.
Therefore, modification of the resulting Unicode object is only allowed when u
is NULL."""
if wchar_p:
s = rffi.wcharpsize2unicode(wchar_p, length)
return make_ref(space, space.newunicode(s))
else:
return rffi.cast(PyObject, new_empty_unicode(space, length))
def unicode_realize(space, py_obj):
"""
Creates the unicode in the interpreter. The PyUnicodeObject buffer must not
be modified after this call.
"""
py_uni = rffi.cast(PyUnicodeObject, py_obj)
s = rffi.wcharpsize2unicode(py_uni.c_str, py_uni.c_length)
w_type = from_ref(space, rffi.cast(PyObject, py_obj.c_ob_type))
w_obj = space.allocate_instance(unicodeobject.W_UnicodeObject, w_type)
w_obj.__init__(s)
py_uni.c_hash = space.hash_w(space.newunicode(s))
track_reference(space, py_obj, w_obj)
return w_obj
def unicode_realize(space, py_obj):
"""
Creates the unicode in the interpreter. The PyUnicodeObject buffer must not
be modified after this call.
"""
py_uni = rffi.cast(PyUnicodeObject, py_obj)
s = rffi.wcharpsize2unicode(py_uni.c_str, py_uni.c_length)
w_type = from_ref(space, rffi.cast(PyObject, py_obj.c_ob_type))
w_obj = space.allocate_instance(unicodeobject.W_UnicodeObject, w_type)
w_obj.__init__(s)
py_uni.c_hash = space.hash_w(w_obj)
track_reference(space, py_obj, w_obj)
return w_obj
def decodeex(decodebuf, stringdata, errors="strict", errorcb=None, namecb=None, ignore_error=0):
inleft = len(stringdata)
with rffi.scoped_nonmovingbuffer(stringdata) as inbuf:
if pypy_cjk_dec_init(decodebuf, inbuf, inleft) < 0:
raise MemoryError
while True:
r = pypy_cjk_dec_chunk(decodebuf)
if r == 0 or r == ignore_error:
break
multibytecodec_decerror(decodebuf, r, errors, errorcb, namecb, stringdata)
src = pypy_cjk_dec_outbuf(decodebuf)
length = pypy_cjk_dec_outlen(decodebuf)
return rffi.wcharpsize2unicode(src, length)
def descr_tounicode(self, space):
""" tounicode() -> unicode
Convert the array to a unicode string. The array must be
a type 'u' array; otherwise a ValueError is raised. Use
array.tostring().decode() to obtain a unicode string from
an array of some other type.
"""
if self.typecode == 'u':
buf = rffi.cast(UNICODE_ARRAY, self._buffer_as_unsigned())
return space.wrap(rffi.wcharpsize2unicode(buf, self.len))
else:
msg = "tounicode() may only be called on type 'u' arrays"
raise OperationError(space.w_ValueError, space.wrap(msg))
def descr_tounicode(self, space):
""" tounicode() -> unicode
Convert the array to a unicode string. The array must be
a type 'u' array; otherwise a ValueError is raised. Use
array.tostring().decode() to obtain a unicode string from
an array of some other type.
"""
if self.typecode == 'u':
buf = rffi.cast(UNICODE_ARRAY, self._buffer_as_unsigned())
return space.wrap(rffi.wcharpsize2unicode(buf, self.len))
else:
msg = "tounicode() may only be called on type 'u' arrays"
raise OperationError(space.w_ValueError, space.wrap(msg))
def decodeex(decodebuf,
stringdata,
errors="strict",
errorcb=None,
namecb=None,
ignore_error=0):
inleft = len(stringdata)
with rffi.scoped_nonmovingbuffer(stringdata) as inbuf:
if pypy_cjk_dec_init(decodebuf, inbuf, inleft) < 0:
raise MemoryError
while True:
r = pypy_cjk_dec_chunk(decodebuf)
if r == 0 or r == ignore_error:
break
multibytecodec_decerror(decodebuf, r, errors, errorcb, namecb,
stringdata)
src = pypy_cjk_dec_outbuf(decodebuf)
length = pypy_cjk_dec_outlen(decodebuf)
return rffi.wcharpsize2unicode(src, length)
def convert_to_object(self, cdata):
unichardata = rffi.cast(rffi.CWCHARP, cdata)
s = rffi.wcharpsize2unicode(unichardata, 1)
return self.space.wrap(s)
def _unicode_from_wchar(ptr, length):
return rffi.wcharpsize2unicode(rffi.cast(rffi.CWCHARP, ptr), length)
def unpack_ptr(self, w_ctypeptr, ptr, length):
u = rffi.wcharpsize2unicode(rffi.cast(rffi.CWCHARP, ptr), length)
return self.space.wrap(u)
def wcharp2rawunicode(space, address, maxlength=-1):
if maxlength == -1:
return wcharp2unicode(space, address)
s = rffi.wcharpsize2unicode(rffi.cast(rffi.CWCHARP, address), maxlength)
return space.wrap(s)
def wcharp2rawunicode(space, address, maxlength=-1):
if maxlength == -1:
return wcharp2unicode(space, address)
s = rffi.wcharpsize2unicode(rffi.cast(rffi.CWCHARP, address), maxlength)
return space.newunicode(s)
def unpack_ptr(self, w_ctypeptr, ptr, length):
u = rffi.wcharpsize2unicode(rffi.cast(rffi.CWCHARP, ptr), length)
return self.space.newunicode(u)
