def test_list_of_float(self):
def f(x: ListOf(float), y: ListOf(float)) -> float:
j = 0
res = 0.0
i = 0
while j < len(y):
i = 0
while i < len(x):
res = res + x[i] * y[j]
i = i + 1
j = j + 1
return res
aListOfFloat = ListOf(float)(list(range(1000)))
aListOfFloat2 = ListOf(float)(list(range(1000)))
self.assertEqual(_types.refcount(aListOfFloat), 1)
t_py, t_fast = self.checkFunction(f, [(aListOfFloat, aListOfFloat2)])
self.assertEqual(_types.refcount(aListOfFloat), 1)
# I get around 150x
self.assertTrue(t_py / t_fast > 50.0)
print(t_py / t_fast, " speedup")
def f(x: ListOf(float), y: ListOf(float)) -> float:
j = 0
res = 0.0
i = 0
while j < len(y):
i = 0
while i < len(x):
res = res + x[i] * y[j]
i = i + 1
j = j + 1
return res
def test_serialize_large_lists(self):
x = SerializationContext({})
lst = ListOf(ListOf(int))()
lst.resize(100)
for sublist in lst:
sublist.resize(1000000)
t0 = time.time()
l2 = x.deserialize(x.serialize(lst))
print(time.time() - t0, " to roundtrip")
self.assertEqual(lst, l2)
def test_list_of(self):
L = ListOf(int)
self.assertEqual(L.__qualname__, "ListOf(Int64)")
l = L([1,2,3,4])
self.assertEqual(l[0], 1)
self.assertEqual(l[-1], 4)
l[0] = 10
self.assertEqual(l[0], 10)
l[-1] = 11
self.assertEqual(l[3], 11)
with self.assertRaisesRegex(IndexError, "index out of range"):
l[100] = 20
l2 = L((10,2,3,11))
self.assertEqual(l,l2)
self.assertNotEqual(l,(10,2,3,11))
self.assertEqual(l,[10,2,3,11])
self.assertEqual(str(l),str([10,2,3,11]))
l3 = l + l2
self.assertEqual(l3, [10,2,3,11,10,2,3,11])
l3.append(23)
self.assertEqual(l3, [10,2,3,11,10,2,3,11, 23])
def test_list_short_circut_if(self):
@Compiled
def chkList(x: ListOf(int)):
return len(x) > 0 and x[0]
x = ListOf(int)([])
self.assertFalse(chkList(x))
def test_serialize_is_parallel(self):
if os.environ.get('TRAVIS_CI', None):
return
T = ListOf(int)
x = T()
x.resize(1000000)
sc = SerializationContext({}).withoutCompression()
def f():
for i in range(10):
sc.deserialize(sc.serialize(x, T), T)
ratios = []
for i in range(10):
t0 = time.time()
thread_apply(f, [()])
t1 = time.time()
thread_apply(f, [(), ()])
t2 = time.time()
first = t1 - t0
second = t2 - t1
ratios.append(second/first)
ratios = sorted(ratios)
ratio = ratios[5]
# expect the ratio to be close to 1, but have some error margin
self.assertTrue(ratio >= .8 and ratio < 1.2, ratios)
def test_pointer_operations(self):
T = ListOf(int)
def testfun(x: T):
pointer = x.pointerUnsafe(0)
pointer.set(20)
(pointer+1).set(20)
(pointer+2).set((pointer+1).get()+1)
(pointer+3).initialize((pointer+2).get())
(pointer+4).cast(float).set(1.0)
return pointer[3]
compiledFun = Compiled(testfun)
l1 = T(list(range(10)))
l2 = T(list(range(10)))
self.assertEqual(testfun(l1), l1[3])
self.assertEqual(compiledFun(l2), l2[3])
self.assertEqual(l1, l2)
self.assertEqual(l1[0], 20)
self.assertEqual(l1[1], 20)
self.assertEqual(l1[2], 21)
self.assertEqual(l1[3], 21)
self.assertEqual(l1[4], 0x3ff0000000000000) # hex representation of 64 bit float 1.0
def repeat(array: ListOf(int), times: int):
"""Concatenate a 'array' to itself 'times' times."""
out = ListOf(int)()
out.resize(len(array) * times)
i = 0
for o in range(times * len(array)):
out[o] = array[i]
i += 1
if i >= len(array):
i = 0
return out
def test_range_repeat(self):
@Compiled
def repeat(array: ListOf(int), times: int):
"""Concatenate a 'array' to itself 'times' times."""
out = ListOf(int)()
out.resize(len(array) * times)
i = 0
for o in range(times * len(array)):
out[o] = array[i]
i += 1
if i >= len(array):
i = 0
return out
aList = ListOf(int)([1, 2, 3])
self.assertEqual(repeat(aList, 0), type(aList)())
self.assertEqual(repeat(aList, 1), aList)
self.assertEqual(repeat(aList, 2), aList + aList)
self.assertEqual(repeat(aList, 3), aList + aList + aList)
def test_class_in_forward(self):
class C(Class):
x = Member(int)
Fwd = Forward("Forward")
Fwd = Fwd.define(OneOf(None, C, TupleOf(Fwd), ListOf(Fwd), ConstDict(str, Fwd)))
Fwd(C())
def test_list_duplicate_operation(self):
@Compiled
def dupList(x: ListOf(int)):
return ListOf(int)(x)
x = ListOf(int)([1, 2, 3])
y = dupList(x)
x[0] = 100
self.assertEqual(y[0], 1)
def test_class_dispatch_on_tuple_vs_list(self):
class WithTwoFunctions(Class):
def f(self, x: TupleOf(int)):
return "Tuple"
def f(self, x: ListOf(int)): # noqa: F811
return "List"
c = WithTwoFunctions()
self.assertEqual(c.f(TupleOf(int)((1, 2, 3))), "Tuple")
self.assertEqual(c.f(ListOf(int)((1, 2, 3))), "Tuple")
def test_serialize_listof(self):
T = ListOf(int)
aList = T()
aPopulatedList = T([1, 2, 3])
self.assertEqual(aList, deserialize(T, serialize(T, aList)))
self.assertEqual(refcount(deserialize(T, serialize(T, aList))), 1)
self.assertEqual(aPopulatedList, deserialize(T, serialize(T, aPopulatedList)))
self.assertEqual(refcount(deserialize(T, serialize(T, aPopulatedList))), 1)
def test_list_assign(self):
@Compiled
def f(x: ListOf(int)) -> ListOf(int):
y = x
return y
t = ListOf(int)((1, 2, 3))
self.assertEqual(f(t), t)
self.assertEqual(_types.refcount(t), 1)
def test_pointer_subtraction(self):
T = ListOf(int)
def testfun(x: T):
pointer = x.pointerUnsafe(0)
return (pointer + 1) - pointer
compiledFun = Compiled(testfun)
self.assertEqual(testfun(T()), 1)
self.assertEqual(compiledFun(T()), 1)
def test_list_reserve(self):
T = ListOf(TupleOf(int))
@Compiled
def f(x: T):
x.reserve(x.reserved() + 10)
aList = T()
aList.resize(10)
oldReserved = aList.reserved()
f(aList)
self.assertEqual(oldReserved + 10, aList.reserved())
def test_list_of_list_refcounting(self):
T = ListOf(int)
TT = ListOf(T)
@Compiled
def f(x: TT) -> TT:
return x + x + x
t1 = T((1, 2, 3))
t2 = T((4, 5, 5))
aTT = TT((t1, t2))
fRes = f(aTT)
self.assertEqual(fRes, aTT + aTT + aTT)
self.assertEqual(_types.refcount(aTT), 1)
self.assertEqual(_types.refcount(fRes), 1)
fRes = None
aTT = None
self.assertEqual(_types.refcount(t1), 1)
def test_list_of_oneOf(self):
T = ListOf(OneOf(None, float))
@Compiled
def f():
x = T()
x.resize(2)
x[0] = 10.0
x[1] = None
x.append(10.0)
x.append(None)
return x
self.assertEqual(f(), [10.0, None, 10.0, None])
def test_list_pop(self):
T = ListOf(int)
@Compiled
def f(x: T):
i = 0
while i < len(x):
x.pop(i)
i = i + 1
aList = T([1, 2, 3, 4])
f(aList)
self.assertEqual(aList, [2, 4])
def test_list_resize(self):
T = ListOf(TupleOf(int))
aTup = TupleOf(int)((1, 2, 3))
@Compiled
def f(x: T, y: TupleOf(int)):
x.resize(len(x) + 10, y)
aList = T()
aList.resize(10)
f(aList, aTup)
self.assertEqual(_types.refcount(aTup), 11)
def test_list_append(self):
T = ListOf(int)
@Compiled
def f(x: T):
i = 0
ct = len(x)
while i < ct:
x.append(x[i])
i = i + 1
aList = T([1, 2, 3, 4])
f(aList)
self.assertEqual(aList, [1, 2, 3, 4, 1, 2, 3, 4])
def test_serialize_stream_complex(self):
T = OneOf(None, float, str, int, ListOf(int))
for items in [
(1, 2),
("hi", None, 10, [1, 2, 3, 4]),
()]:
self.assertEqual(
serializeStream(T, items),
b"".join([serialize(T, x) for x in items])
)
self.assertEqual(
deserializeStream(T, serializeStream(T, items)),
TupleOf(T)(items)
)
def test_bytecount(self):
def testfun(x):
return _types.bytecount(type(x))
self.assertEqual(testfun(0), 8)
def check(x):
self.assertEqual(
testfun(x),
Runtime.singleton().compile(testfun, {'x': type(x)})(x)
)
check(0)
check(0.0)
check(ListOf(int)([10]))
check(Tuple(int, int, int)((10, 10, 10)))
def test_refcounts_of_objects_across_boundary(self):
class Object:
pass
_ = Object()
A = Alternative("A", X={'x': int}, Y={'y': int})
for instance in [
TupleOf(int)((1, 2, 3)),
ListOf(int)((1, 2, 3)),
# Dict(int,int)({1:2,3:4}),
ConstDict(int, int)({1: 2, 3: 4}),
AClass(),
# anObject,
A.X(x=10)
]:
self.refcountsTest(instance)
def test_list_creation_doesnt_leak(self):
T = ListOf(int)
@Compiled
def f(x: T, y: T) -> T:
return x + y
t1 = T(tuple(range(10000)))
initMem = psutil.Process().memory_info().rss / 1024**2
for i in range(10000):
f(t1, t1)
finalMem = psutil.Process().memory_info().rss / 1024**2
self.assertTrue(finalMem < initMem + 5)
def test_list_of_adding(self):
T = ListOf(int)
@Compiled
def f(x: T, y: T) -> T:
return x + y
t1 = T((1, 2, 3))
t2 = T((3, 4))
res = f(t1, t2)
self.assertEqual(_types.refcount(res), 1)
self.assertEqual(_types.refcount(t1), 1)
self.assertEqual(_types.refcount(t2), 1)
self.assertEqual(res, t1 + t2)
def test_recursive_list(self):
L = Forward("L")
L = L.define(ListOf(OneOf(int, L)))
listInst = L()
listInst.append(10)
listInst.append(listInst)
self.assertEqual(
serialize(L, listInst),
BEGIN_COMPOUND(0) + (
VARINT(0) + unsignedVarint(0) + # the ID
VARINT(0) + unsignedVarint(2) + # the size
SINGLE(0) + VARINT(0) + signedVarint(10) + SINGLE(0) +
SINGLE(1) +
( # a compound object encoding the second element of the one-of
VARINT(0) + unsignedVarint(0) # the ID and nothing else
)) + END_COMPOUND())
def test_list_refcounting(self):
@Function
def f(x: ListOf(int), y: ListOf(int)) -> ListOf(int):
return x
for compileIt in [False, True]:
if compileIt:
Runtime.singleton().compile(f)
intTup = ListOf(int)(list(range(1000)))
self.assertEqual(_types.refcount(intTup), 1)
res = f(intTup, intTup)
self.assertEqual(_types.refcount(intTup), 2)
res = None # noqa: F841
self.assertEqual(_types.refcount(intTup), 1)
def f(x: ListOf(int)) -> ListOf(int):
y = x
return y
def f(x: ListOf(int)) -> int:
return len(x)