def float_bytes_to_real(bytes, signed):
# XXX Currently does not make use of the signed parameter
if len(bytes) not in (4, 8):
raise SchemeException(
"floating-point-bytes->real: byte string must have length 2, 4, or 8")
try:
if objectmodel.we_are_translated():
val = rarithmetic.r_int64(0)
for i, v in enumerate(bytes):
val += rarithmetic.r_int64(ord(v)) << (i * 8)
return values.W_Flonum(longlong2float.longlong2float(val))
else:
# use unsigned to avoid rlib bug
val = rarithmetic.r_uint64(0)
for i, v in enumerate(bytes):
val += rarithmetic.r_uint64(ord(v)) << (i * 8)
return values.W_Flonum(pycket_longlong2float(val))
except OverflowError, e:
# Uncomment the check below to run Pycket on the
# interpreter with compiled (zo) files
# (fasl makes a call that blows the longlong2float on rpython)
# if val == 18442240474082181120L:
# return values.W_Flonum.NEGINF
raise SchemeException("RPython overflow : %s" % e)
def float_bytes_to_real(bytes, signed):
# XXX Currently does not make use of the signed parameter
if len(bytes) not in (4, 8):
raise SchemeException(
"floating-point-bytes->real: byte string must have length 2, 4, or 8")
try:
if objectmodel.we_are_translated():
val = rarithmetic.r_int64(0)
for i, v in enumerate(bytes):
val += rarithmetic.r_int64(ord(v)) << (i * 8)
return values.W_Flonum(longlong2float.longlong2float(val))
else:
# use unsigned to avoid rlib bug
val = rarithmetic.r_uint64(0)
for i, v in enumerate(bytes):
val += rarithmetic.r_uint64(ord(v)) << (i * 8)
return values.W_Flonum(pycket_longlong2float(val))
except OverflowError, e:
# Uncomment the check below to run Pycket on the
# interpreter with compiled (zo) files
# (fasl makes a call that blows the longlong2float on rpython)
# if val == 18442240474082181120L:
# return values.W_Flonum.NEGINF
raise SchemeException("RPython overflow : %s" % e)
def choosen_seed(new_k0, new_k1, test_misaligned_path=False):
"""For tests."""
global misaligned_is_fine
old = seed.k0l, seed.k1l, misaligned_is_fine
seed.k0l = _le64toh(r_uint64(new_k0))
seed.k1l = _le64toh(r_uint64(new_k1))
if test_misaligned_path:
misaligned_is_fine = False
yield
seed.k0l, seed.k1l, misaligned_is_fine = old
def choosen_seed(new_k0, new_k1, test_misaligned_path=False,
test_prebuilt=False):
"""For tests."""
global misaligned_is_fine, seed
old = seed, misaligned_is_fine
seed = Seed()
seed.k0l = r_uint64(new_k0)
seed.k1l = r_uint64(new_k1)
if test_prebuilt:
_update_prebuilt_hashes()
else:
seed.bound_prebuilt_size = 0
if test_misaligned_path:
misaligned_is_fine = False
yield
seed, misaligned_is_fine = old
def _decode64(s):
return (r_uint64(ord(s[0])) |
r_uint64(ord(s[1])) << 8 |
r_uint64(ord(s[2])) << 16 |
r_uint64(ord(s[3])) << 24 |
r_uint64(ord(s[4])) << 32 |
r_uint64(ord(s[5])) << 40 |
r_uint64(ord(s[6])) << 48 |
r_uint64(ord(s[7])) << 56)
def test_sign_when_casting_uint_to_larger_int():
from rpython.rtyper.lltypesystem import rffi
from rpython.rlib.rarithmetic import r_uint32, r_uint64
#
value = 0xAAAABBBB
assert cast(lltype.SignedLongLong, r_uint32(value)) == value
if hasattr(rffi, '__INT128_T'):
value = 0xAAAABBBBCCCCDDDD
assert cast(rffi.__INT128_T, r_uint64(value)) == value
def test_sign_when_casting_uint_to_larger_int():
from rpython.rtyper.lltypesystem import rffi
from rpython.rlib.rarithmetic import r_uint32, r_uint64
#
value = 0xAAAABBBB
assert cast(lltype.SignedLongLong, r_uint32(value)) == value
if hasattr(rffi, '__INT128_T'):
value = 0xAAAABBBBCCCCDDDD
assert cast(rffi.__INT128_T, r_uint64(value)) == value
def _update_prebuilt_hashes():
seed.bound_prebuilt_size = 0
with lltype.scoped_alloc(rffi.CCHARP.TO, 1) as p:
addr = llmemory.cast_ptr_to_adr(p)
seed.hash_single = [r_uint64(0)] * 256
for i in range(256):
p[0] = chr(i)
seed.hash_single[i] = _siphash24(addr, 1)
seed.hash_empty = _siphash24(addr, 0)
seed.bound_prebuilt_size = 2
def source_hash(space, magic, content):
from rpython.rlib.rsiphash import siphash24_with_key
from rpython.rlib.rarithmetic import r_uint64
h = siphash24_with_key(content, r_uint64(magic))
res = [b"x"] * 8
for i in range(8):
res[i] = chr(h & 0xff)
h >>= 8
assert not h
return space.newbytes(b"".join(res))
builder.append(chr((0xf0 | (code >> 18))))
builder.append(chr((0x80 | ((code >> 12) & 0x3f))))
builder.append(chr((0x80 | ((code >> 6) & 0x3f))))
builder.append(chr((0x80 | (code & 0x3f))))
return
raise OutOfRange(code)
# note - table lookups are really slow. Measured on various elements of obama
# chinese wikipedia, they're anywhere between 10% and 30% slower.
# In extreme cases (small, only chinese text), they're 40% slower
# The following was found by hand to be more optimal than both,
# on x86-64...
_is_64bit = sys.maxint > 2**32
_constant_ncp = rarithmetic.r_uint64(0xffff0000ffffffff)
@always_inline
def next_codepoint_pos(code, pos):
"""Gives the position of the next codepoint after pos.
Assumes valid utf8. 'pos' must be before the end of the string.
"""
assert pos >= 0
chr1 = ord(code[pos])
if chr1 <= 0x7F:
return pos + 1
if _is_64bit and not jit.we_are_jitted():
# optimized for Intel x86-64 by hand
res = pos + 1 + (((chr1 > 0xDF) << 1) + rarithmetic.intmask(
(_constant_ncp >> (chr1 & 0x3F)) & 1))
class Seed:
k0l = k1l = r_uint64(0)
def _siphash24(addr_in, size, SZ=1):
"""Takes an address pointer and a size. Returns the hash as a r_uint64,
which can then be casted to the expected type."""
if BIG_ENDIAN:
index = SZ - 1
else:
index = 0
if size < seed.bound_prebuilt_size:
if size <= 0:
return seed.hash_empty
else:
t = rarithmetic.intmask(llop.raw_load(rffi.UCHAR, addr_in, index))
return seed.hash_single[t]
k0 = seed.k0l
k1 = seed.k1l
b = r_uint64(size) << 56
v0 = k0 ^ magic0
v1 = k1 ^ magic1
v2 = k0 ^ magic2
v3 = k1 ^ magic3
direct = (SZ == 1) and (misaligned_is_fine or
(rffi.cast(lltype.Signed, addr_in) & 7) == 0)
if direct:
assert SZ == 1
while size >= 8:
mi = llop.raw_load(rffi.ULONGLONG, addr_in, index)
mi = _le64toh(mi)
size -= 8
index += 8
v3 ^= mi
v0, v1, v2, v3 = _double_round(v0, v1, v2, v3)
v0 ^= mi
else:
while size >= 8:
mi = (
r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index)) | r_uint64(
llop.raw_load(rffi.UCHAR, addr_in, index + 1 * SZ)) << 8
| r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 2 * SZ))
<< 16 | r_uint64(
llop.raw_load(rffi.UCHAR, addr_in, index + 3 * SZ)) << 24
| r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 4 * SZ))
<< 32 | r_uint64(
llop.raw_load(rffi.UCHAR, addr_in, index + 5 * SZ)) << 40
| r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 6 * SZ))
<< 48 | r_uint64(
llop.raw_load(rffi.UCHAR, addr_in, index + 7 * SZ)) << 56)
size -= 8
index += 8 * SZ
v3 ^= mi
v0, v1, v2, v3 = _double_round(v0, v1, v2, v3)
v0 ^= mi
t = r_uint64(0)
if size == 7:
t = r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 6 * SZ)) << 48
size = 6
if size == 6:
t |= r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 5 * SZ)) << 40
size = 5
if size == 5:
t |= r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 4 * SZ)) << 32
size = 4
if size == 4:
if direct:
v = _le32toh(r_uint32(llop.raw_load(rffi.UINT, addr_in, index)))
t |= r_uint64(v)
size = 0
else:
t |= r_uint64(llop.raw_load(rffi.UCHAR, addr_in,
index + 3 * SZ)) << 24
size = 3
if size == 3:
t |= r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 2 * SZ)) << 16
size = 2
if size == 2:
t |= r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 1 * SZ)) << 8
size = 1
if size == 1:
t |= r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index))
size = 0
assert size == 0
b |= t
v3 ^= b
v0, v1, v2, v3 = _double_round(v0, v1, v2, v3)
v0 ^= b
v2 ^= 0xff
v0, v1, v2, v3 = _double_round(v0, v1, v2, v3)
v0, v1, v2, v3 = _double_round(v0, v1, v2, v3)
return (v0 ^ v1) ^ (v2 ^ v3)
global misaligned_is_fine, seed
old = seed, misaligned_is_fine
seed = Seed()
seed.k0l = r_uint64(new_k0)
seed.k1l = r_uint64(new_k1)
if test_prebuilt:
_update_prebuilt_hashes()
else:
seed.bound_prebuilt_size = 0
if test_misaligned_path:
misaligned_is_fine = False
yield
seed, misaligned_is_fine = old
magic0 = r_uint64(0x736f6d6570736575)
magic1 = r_uint64(0x646f72616e646f6d)
magic2 = r_uint64(0x6c7967656e657261)
magic3 = r_uint64(0x7465646279746573)
@always_inline
def _rotate(x, b):
return (x << b) | (x >> (64 - b))
@always_inline
def _half_round(a, b, c, d, s, t):
a += b
c += d
b = _rotate(b, s) ^ a
def _siphash24(addr_in, size):
"""Takes an address pointer and a size. Returns the hash as a r_uint64,
which can then be casted to the expected type."""
k0 = seed.k0l
k1 = seed.k1l
b = r_uint64(size) << 56
v0 = k0 ^ magic0
v1 = k1 ^ magic1
v2 = k0 ^ magic2
v3 = k1 ^ magic3
direct = (misaligned_is_fine
or (rffi.cast(lltype.Signed, addr_in) & 7) == 0)
index = 0
if direct:
while size >= 8:
mi = llop.raw_load(rffi.ULONGLONG, addr_in, index)
mi = _le64toh(mi)
size -= 8
index += 8
v3 ^= mi
v0, v1, v2, v3 = _double_round(v0, v1, v2, v3)
v0 ^= mi
else:
while size >= 8:
mi = (
r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index))
| r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 1)) << 8
| r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 2)) << 16
| r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 3)) << 24
| r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 4)) << 32
| r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 5)) << 40
| r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 6)) << 48
|
r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 7)) << 56)
size -= 8
index += 8
v3 ^= mi
v0, v1, v2, v3 = _double_round(v0, v1, v2, v3)
v0 ^= mi
t = r_uint64(0)
if size == 7:
t = r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 6)) << 48
size = 6
if size == 6:
t |= r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 5)) << 40
size = 5
if size == 5:
t |= r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 4)) << 32
size = 4
if size == 4:
if direct:
t |= r_uint64(llop.raw_load(rffi.UINT, addr_in, index))
size = 0
else:
t |= r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 3)) << 24
size = 3
if size == 3:
t |= r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 2)) << 16
size = 2
if size == 2:
t |= r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 1)) << 8
size = 1
if size == 1:
t |= r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index))
size = 0
assert size == 0
b |= _le64toh(t)
v3 ^= b
v0, v1, v2, v3 = _double_round(v0, v1, v2, v3)
v0 ^= b
v2 ^= 0xff
v0, v1, v2, v3 = _double_round(v0, v1, v2, v3)
v0, v1, v2, v3 = _double_round(v0, v1, v2, v3)
return (v0 ^ v1) ^ (v2 ^ v3)
def test_siphash24_with_key():
from rpython.rlib.rarithmetic import r_uint64
assert siphash24_with_key(b"abcdef", r_uint64(1)) == r_uint64(7956077396882317016L)
def _siphash24_with_key(addr_in, size, k0, k1, SZ=1):
if BIG_ENDIAN:
index = SZ - 1
else:
index = 0
b = r_uint64(size) << 56
v0 = k0 ^ magic0
v1 = k1 ^ magic1
v2 = k0 ^ magic2
v3 = k1 ^ magic3
direct = (SZ == 1) and (misaligned_is_fine or
(rffi.cast(lltype.Signed, addr_in) & 7) == 0)
if direct:
assert SZ == 1
while size >= 8:
mi = llop.raw_load(rffi.ULONGLONG, addr_in, index)
mi = _le64toh(mi)
size -= 8
index += 8
v3 ^= mi
v0, v1, v2, v3 = _double_round(v0, v1, v2, v3)
v0 ^= mi
else:
while size >= 8:
mi = (
r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index)) | r_uint64(
llop.raw_load(rffi.UCHAR, addr_in, index + 1 * SZ)) << 8
| r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 2 * SZ))
<< 16 | r_uint64(
llop.raw_load(rffi.UCHAR, addr_in, index + 3 * SZ)) << 24
| r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 4 * SZ))
<< 32 | r_uint64(
llop.raw_load(rffi.UCHAR, addr_in, index + 5 * SZ)) << 40
| r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 6 * SZ))
<< 48 | r_uint64(
llop.raw_load(rffi.UCHAR, addr_in, index + 7 * SZ)) << 56)
size -= 8
index += 8 * SZ
v3 ^= mi
v0, v1, v2, v3 = _double_round(v0, v1, v2, v3)
v0 ^= mi
t = r_uint64(0)
if size == 7:
t = r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 6 * SZ)) << 48
size = 6
if size == 6:
t |= r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 5 * SZ)) << 40
size = 5
if size == 5:
t |= r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 4 * SZ)) << 32
size = 4
if size == 4:
if direct:
v = _le32toh(r_uint32(llop.raw_load(rffi.UINT, addr_in, index)))
t |= r_uint64(v)
size = 0
else:
t |= r_uint64(llop.raw_load(rffi.UCHAR, addr_in,
index + 3 * SZ)) << 24
size = 3
if size == 3:
t |= r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 2 * SZ)) << 16
size = 2
if size == 2:
t |= r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index + 1 * SZ)) << 8
size = 1
if size == 1:
t |= r_uint64(llop.raw_load(rffi.UCHAR, addr_in, index))
size = 0
assert size == 0
b |= t
v3 ^= b
v0, v1, v2, v3 = _double_round(v0, v1, v2, v3)
v0 ^= b
v2 ^= 0xff
v0, v1, v2, v3 = _double_round(v0, v1, v2, v3)
v0, v1, v2, v3 = _double_round(v0, v1, v2, v3)
return (v0 ^ v1) ^ (v2 ^ v3)
