def testUnmanagedThreadState(self, mapper, _):
mapper.ReleaseGIL()
# current thread state should be null if nobody has the GIL
self.assertEquals(CPyMarshal.ReadPtr(mapper._PyThreadState_Current),
IntPtr.Zero)
mapper.EnsureGIL()
mapper.LastException = NameError("Harold")
ts = CPyMarshal.ReadPtr(mapper._PyThreadState_Current)
curexc_type = CPyMarshal.ReadPtrField(ts, PyThreadState, "curexc_type")
curexc_value = CPyMarshal.ReadPtrField(ts, PyThreadState,
"curexc_value")
self.assertEquals(mapper.Retrieve(curexc_type), NameError)
self.assertEquals(mapper.Retrieve(curexc_value), "Harold")
mapper.ReleaseGIL()
def CheckOtherThread():
mapper.EnsureGIL()
ts2 = CPyMarshal.ReadPtr(mapper._PyThreadState_Current)
self.assertNotEquals(ts2, ts)
curexc_type = CPyMarshal.ReadPtrField(ts2, PyThreadState,
"curexc_type")
curexc_value = CPyMarshal.ReadPtrField(ts2, PyThreadState,
"curexc_value")
self.assertEquals(curexc_type, IntPtr.Zero)
self.assertEquals(curexc_value, IntPtr.Zero)
mapper.ReleaseGIL()
thread = Thread(ThreadStart(CheckOtherThread))
thread.Start()
thread.Join()
mapper.EnsureGIL()
def testReadPtr(self):
data = Marshal.AllocHGlobal(CPyMarshal.PtrSize)
Marshal.WriteIntPtr(data, IntPtr(0))
self.assertEquals(CPyMarshal.ReadPtr(data), IntPtr(0), "wrong")
Marshal.WriteIntPtr(data, IntPtr(100001))
self.assertEquals(CPyMarshal.ReadPtr(data), IntPtr(100001), "wrong")
Marshal.FreeHGlobal(data)
def testStoreList(self, mapper, _):
list_ = [1, 2, 3]
listPtr = mapper.Store(list_)
self.assertEquals(id(mapper.Retrieve(listPtr)), id(list_))
typePtr = CPyMarshal.ReadPtrField(listPtr, PyObject, "ob_type")
self.assertEquals(typePtr, mapper.PyList_Type, "wrong type")
dataStore = CPyMarshal.ReadPtrField(listPtr, PyListObject, "ob_item")
for i in range(1, 4):
self.assertEquals(mapper.Retrieve(CPyMarshal.ReadPtr(dataStore)), i, "contents not stored")
self.assertEquals(mapper.RefCount(CPyMarshal.ReadPtr(dataStore)), 1, "bad refcount for items")
dataStore = OffsetPtr(dataStore, CPyMarshal.PtrSize)
def testCanChangeValuesDuringIteration(self, mapper, addDealloc):
posPtr = Marshal.AllocHGlobal(CPyMarshal.PtrSize * 3)
keyPtrPtr = CPyMarshal.Offset(posPtr, CPyMarshal.PtrSize)
valuePtrPtr = CPyMarshal.Offset(keyPtrPtr, CPyMarshal.PtrSize)
addDealloc(lambda: Marshal.FreeHGlobal(posPtr))
CPyMarshal.WriteInt(posPtr, 0)
d = dict(a=1, b=2, c=3)
dPtr = mapper.Store(d)
while mapper.PyDict_Next(dPtr, posPtr, keyPtrPtr, valuePtrPtr) != 0:
key = mapper.Retrieve(CPyMarshal.ReadPtr(keyPtrPtr))
value = mapper.Retrieve(CPyMarshal.ReadPtr(valuePtrPtr))
d[key] = value * 10
self.assertEquals(d, dict(a=10, b=20, c=30))
def testPyList_DeallocDecRefsItemsAndCallsCorrectFreeFunction(self):
frees = []
mapper = PythonMapper(GetAllocatingTestAllocator([], frees))
deallocTypes = CreateTypes(mapper)
calls = []
def CustomFree(ptr):
calls.append(ptr)
mapper.PyObject_Free(listPtr)
self.freeDgt = dgt_void_ptr(CustomFree)
CPyMarshal.WriteFunctionPtrField(mapper.PyList_Type, PyTypeObject, "tp_free", self.freeDgt)
listPtr = mapper.Store([1, 2, 3])
itemPtrs = []
dataStore = CPyMarshal.ReadPtrField(listPtr, PyListObject, "ob_item")
for _ in range(3):
itemPtrs.append(CPyMarshal.ReadPtr(dataStore))
dataStore = OffsetPtr(dataStore, CPyMarshal.PtrSize)
mapper.IC_PyList_Dealloc(listPtr)
for itemPtr in itemPtrs:
self.assertEquals(itemPtr in frees, True, "did not decref item")
self.assertEquals(calls, [listPtr], "did not call type's free function")
mapper.Dispose()
deallocTypes()
def CreateInstance(mapper, calls):
listPtr = mapper.Store([1, 2, 3])
dataStore = CPyMarshal.ReadPtrField(listPtr, PyListObject, "ob_item")
for _ in range(3):
itemPtrs.append(CPyMarshal.ReadPtr(dataStore))
dataStore = OffsetPtr(dataStore, CPyMarshal.PtrSize)
return listPtr
def testIteratesSuccessfully(self, mapper, addDealloc):
posPtr = Marshal.AllocHGlobal(CPyMarshal.PtrSize * 3)
keyPtrPtr = CPyMarshal.Offset(posPtr, CPyMarshal.PtrSize)
valuePtrPtr = CPyMarshal.Offset(keyPtrPtr, CPyMarshal.PtrSize)
addDealloc(lambda: Marshal.FreeHGlobal(posPtr))
CPyMarshal.WriteInt(posPtr, 0)
d = dict(a=1, b=2, c=3)
dPtr = mapper.Store(d)
result = {}
while mapper.PyDict_Next(dPtr, posPtr, keyPtrPtr, valuePtrPtr) != 0:
key = mapper.Retrieve(CPyMarshal.ReadPtr(keyPtrPtr))
value = mapper.Retrieve(CPyMarshal.ReadPtr(valuePtrPtr))
result[key] = value
self.assertEquals(result, d)
def testStoreTupleCreatesTupleType(self):
allocs = []
mapper = PythonMapper(GetAllocatingTestAllocator(allocs, []))
typeBlock = Marshal.AllocHGlobal(Marshal.SizeOf(PyTypeObject()))
mapper.RegisterData("PyTuple_Type", typeBlock)
theTuple = (0, 1, 2)
tuplePtr = mapper.Store(theTuple)
self.assertEquals(
CPyMarshal.ReadPtrField(tuplePtr, PyTupleObject, "ob_type"),
typeBlock, "wrong type")
dataPtr = OffsetPtr(tuplePtr, Marshal.OffsetOf(PyTupleObject,
"ob_item"))
for i in range(3):
item = mapper.Retrieve(CPyMarshal.ReadPtr(dataPtr))
self.assertEquals(item, i, "did not store data")
dataPtr = OffsetPtr(dataPtr, CPyMarshal.PtrSize)
tuplePtr2 = mapper.Store(theTuple)
self.assertEquals(tuplePtr2, tuplePtr,
"didn't realise already had this tuple")
self.assertEquals(mapper.RefCount(tuplePtr), 2, "didn't incref")
mapper.Dispose()
Marshal.FreeHGlobal(typeBlock)
def testInternExisting(self, mapper, addToCleanUp):
testString = "mars needs women" + self.getStringWithValues(1, 256)
bytes = self.byteArrayFromString(testString)
testData = self.ptrFromByteArray(bytes)
sp1 = mapper.PyString_FromString(testData)
addToCleanUp(lambda: Marshal.FreeHGlobal(sp1p))
sp2 = mapper.PyString_InternFromString(testData)
addToCleanUp(lambda: Marshal.FreeHGlobal(testData))
self.assertNotEquals(sp1, sp2)
self.assertFalse(mapper.Retrieve(sp1) is mapper.Retrieve(sp2))
self.assertEquals(mapper.RefCount(sp1), 1)
self.assertEquals(
mapper.RefCount(sp2), 2,
'failed to grab extra reference to induce immortality')
mapper.IncRef(sp1)
sp1p = Marshal.AllocHGlobal(Marshal.SizeOf(IntPtr))
CPyMarshal.WritePtr(sp1p, sp1)
mapper.PyString_InternInPlace(sp1p)
sp1i = CPyMarshal.ReadPtr(sp1p)
self.assertEquals(sp1i, sp2, 'failed to intern')
self.assertTrue(mapper.Retrieve(sp1i) is mapper.Retrieve(sp2))
self.assertEquals(mapper.RefCount(sp1), 1,
'failed to decref old string')
self.assertEquals(mapper.RefCount(sp2), 3,
'failed to incref interned string')
def assertPyTuple_New_Works(self, length):
allocs = []
mapper = PythonMapper(GetAllocatingTestAllocator(allocs, []))
typeBlock = Marshal.AllocHGlobal(Marshal.SizeOf(PyTypeObject))
mapper.RegisterData("PyTuple_Type", typeBlock)
tuplePtr = mapper.PyTuple_New(length)
expectedSize = Marshal.SizeOf(PyTupleObject) + (CPyMarshal.PtrSize * (length - 1))
self.assertEquals(allocs, [(tuplePtr, expectedSize)], "bad alloc")
tupleStruct = Marshal.PtrToStructure(tuplePtr, PyTupleObject)
self.assertEquals(tupleStruct.ob_refcnt, 1, "bad refcount")
self.assertEquals(tupleStruct.ob_type, mapper.PyTuple_Type, "bad type")
self.assertEquals(tupleStruct.ob_size, length, "bad size")
self.assertEquals(mapper.PyTuple_Size(tuplePtr), length, "should still work with uninitialised tuple imo")
dataPtr = OffsetPtr(tuplePtr, Marshal.OffsetOf(PyTupleObject, "ob_item"))
itemPtrs = []
for i in range(length):
self.assertEquals(CPyMarshal.ReadPtr(dataPtr), IntPtr.Zero, "item memory not zeroed")
itemPtr = mapper.Store(i + 100)
CPyMarshal.WritePtr(dataPtr, itemPtr)
itemPtrs.append(itemPtr)
dataPtr = OffsetPtr(dataPtr, CPyMarshal.PtrSize)
immutableTuple = mapper.Retrieve(tuplePtr)
self.assertEquals(immutableTuple, tuple(i + 100 for i in range(length)), "broken")
tuplePtr2 = mapper.Store(immutableTuple)
self.assertEquals(tuplePtr2, tuplePtr, "didn't realise already had this object stored")
self.assertEquals(mapper.RefCount(tuplePtr), 2, "didn't incref")
mapper.Dispose()
Marshal.FreeHGlobal(typeBlock)
def testBufferProtocol(self, mapper, later):
# should all be implemented in C really, but weaving cpy string type into
# our code feels too much like hard work for now
strPtr = mapper.PyString_Type
bufPtr = CPyMarshal.ReadPtrField(strPtr, PyTypeObject, 'tp_as_buffer')
self.assertNotEquals(bufPtr, IntPtr.Zero)
getreadbuffer = CPyMarshal.ReadFunctionPtrField(
bufPtr, PyBufferProcs, 'bf_getreadbuffer', dgt_int_ptrintptr)
getwritebuffer = CPyMarshal.ReadFunctionPtrField(
bufPtr, PyBufferProcs, 'bf_getwritebuffer', dgt_int_ptrintptr)
getcharbuffer = CPyMarshal.ReadFunctionPtrField(
bufPtr, PyBufferProcs, 'bf_getcharbuffer', dgt_int_ptrintptr)
getsegcount = CPyMarshal.ReadFunctionPtrField(bufPtr, PyBufferProcs,
'bf_getsegcount',
dgt_int_ptrptr)
ptrptr = Marshal.AllocHGlobal(Marshal.SizeOf(IntPtr))
later(lambda: Marshal.FreeHGlobal(ptrptr))
strptr = mapper.Store("hullo")
for getter in (getreadbuffer, getcharbuffer):
self.assertEquals(getter(strptr, 0, ptrptr), 5)
self.assertEquals(
CPyMarshal.ReadPtr(ptrptr),
CPyMarshal.GetField(strptr, PyStringObject, 'ob_sval'))
self.assertEquals(getter(strptr, 1, ptrptr), -1)
self.assertMapperHasError(mapper, SystemError)
self.assertEquals(getwritebuffer(strptr, 0, ptrptr), -1)
self.assertMapperHasError(mapper, SystemError)
self.assertEquals(getsegcount(strptr, ptrptr), 1)
self.assertEquals(CPyMarshal.ReadInt(ptrptr), 5)
self.assertEquals(getsegcount(strptr, IntPtr.Zero), 1)
def testPyList_New_NonZeroLength(self):
allocs = []
mapper = PythonMapper(GetAllocatingTestAllocator(allocs, []))
deallocTypes = CreateTypes(mapper)
del allocs[:]
SIZE = 27
listPtr = mapper.PyList_New(SIZE)
listStruct = Marshal.PtrToStructure(listPtr, PyListObject)
self.assertEquals(listStruct.ob_refcnt, 1, "bad refcount")
self.assertEquals(listStruct.ob_type, mapper.PyList_Type, "bad type")
self.assertEquals(listStruct.ob_size, SIZE, "bad ob_size")
self.assertEquals(listStruct.allocated, SIZE, "bad allocated")
dataPtr = listStruct.ob_item
self.assertNotEquals(dataPtr, IntPtr.Zero, "failed to allocate space for data")
expectedAllocs = [(dataPtr, (SIZE * CPyMarshal.PtrSize)), (listPtr, Marshal.SizeOf(PyListObject))]
self.assertEquals(set(allocs), set(expectedAllocs), "allocated wrong")
for _ in range(SIZE):
self.assertEquals(CPyMarshal.ReadPtr(dataPtr), IntPtr.Zero, "failed to zero memory")
dataPtr = OffsetPtr(dataPtr, CPyMarshal.PtrSize)
mapper.Dispose()
deallocTypes()
def test_PyTuple_Resize_TooBig(self, mapper, addDealloc):
tuplePtrPtr = Marshal.AllocHGlobal(CPyMarshal.PtrSize)
addDealloc(lambda: Marshal.FreeHGlobal(tuplePtrPtr))
tuplePtr = mapper.PyTuple_New(1)
CPyMarshal.WritePtr(tuplePtrPtr, tuplePtr)
self.assertEquals(mapper._PyTuple_Resize(tuplePtrPtr, 2000000000), -1)
self.assertEquals(CPyMarshal.ReadPtr(tuplePtrPtr), IntPtr.Zero)
def testPyList_Append(self):
allocs = []
deallocs = []
mapper = PythonMapper(GetAllocatingTestAllocator(allocs, deallocs))
deallocTypes = CreateTypes(mapper)
del allocs[:]
listPtr = mapper.PyList_New(0)
self.assertEquals(allocs, [(listPtr, Marshal.SizeOf(PyListObject))], "bad alloc")
item1 = object()
item2 = object()
itemPtr1 = mapper.Store(item1)
itemPtr2 = mapper.Store(item2)
self.assertEquals(mapper.PyList_Append(listPtr, itemPtr1), 0, "failed to report success")
self.assertEquals(len(allocs), 4, "didn't allocate memory for data store (list; item1; item2; data store comes 4th)")
dataPtrAfterFirstAppend = CPyMarshal.ReadPtrField(listPtr, PyListObject, "ob_item")
self.assertEquals(allocs[3], (dataPtrAfterFirstAppend, CPyMarshal.PtrSize), "allocated wrong amount of memory")
self.assertEquals(CPyMarshal.ReadPtr(dataPtrAfterFirstAppend), itemPtr1, "failed to fill memory")
self.assertEquals(mapper.RefCount(itemPtr1), 2, "failed to incref new contents")
self.assertEquals(mapper.Retrieve(listPtr), [item1], "retrieved wrong list")
# make refcount 1, to prove that references are not lost when reallocing data
mapper.DecRef(itemPtr1)
self.assertEquals(mapper.PyList_Append(listPtr, itemPtr2), 0, "failed to report success")
self.assertEquals(len(allocs), 5, "didn't allocate memory for new, larger data store")
self.assertEquals(deallocs, [dataPtrAfterFirstAppend])
dataPtrAfterSecondAppend = CPyMarshal.ReadPtrField(listPtr, PyListObject, "ob_item")
self.assertEquals(allocs[4], (dataPtrAfterSecondAppend, (CPyMarshal.PtrSize * 2)),
"allocated wrong amount of memory")
self.assertEquals(CPyMarshal.ReadPtr(dataPtrAfterSecondAppend), itemPtr1,
"failed to keep reference to first item")
self.assertEquals(CPyMarshal.ReadPtr(OffsetPtr(dataPtrAfterSecondAppend, CPyMarshal.PtrSize)), itemPtr2,
"failed to keep reference to first item")
self.assertEquals(mapper.RefCount(itemPtr1), 1, "wrong refcount for item existing only in list")
self.assertEquals(mapper.RefCount(itemPtr2), 2, "wrong refcount newly-added item")
self.assertEquals(mapper.Retrieve(listPtr), [item1, item2], "retrieved wrong list")
mapper.Dispose()
deallocTypes()
def CheckOtherThread():
mapper.EnsureGIL()
ts2 = CPyMarshal.ReadPtr(mapper._PyThreadState_Current)
self.assertNotEquals(ts2, ts)
curexc_type = CPyMarshal.ReadPtrField(ts2, PyThreadState,
"curexc_type")
curexc_value = CPyMarshal.ReadPtrField(ts2, PyThreadState,
"curexc_value")
self.assertEquals(curexc_type, IntPtr.Zero)
self.assertEquals(curexc_value, IntPtr.Zero)
mapper.ReleaseGIL()
def testWorksWithoutEmbeddedNulls(self, mapper, addDealloc):
dataPtrPtr = Marshal.AllocHGlobal(CPyMarshal.PtrSize * 2)
sizePtr = CPyMarshal.Offset(dataPtrPtr, CPyMarshal.PtrSize)
addDealloc(lambda: Marshal.FreeHGlobal(dataPtrPtr))
testStr = "You're fighting Ed the Undying." + self.getStringWithValues(
1, 256)
strPtr = mapper.Store(testStr)
dataPtr = self.dataPtrFromStrPtr(strPtr)
self.assertEquals(
mapper.PyString_AsStringAndSize(strPtr, dataPtrPtr, sizePtr), 0)
self.assertEquals(CPyMarshal.ReadPtr(dataPtrPtr), dataPtr)
self.assertEquals(CPyMarshal.ReadInt(sizePtr), len(testStr))
self.assertMapperHasError(mapper, None)
CPyMarshal.Zero(dataPtrPtr, CPyMarshal.PtrSize * 2)
self.assertEquals(
mapper.PyString_AsStringAndSize(strPtr, dataPtrPtr, IntPtr.Zero),
0)
self.assertEquals(CPyMarshal.ReadPtr(dataPtrPtr), dataPtr)
self.assertMapperHasError(mapper, None)
def testReferencesAreBorrowed(self, mapper, addDealloc):
posPtr = Marshal.AllocHGlobal(CPyMarshal.PtrSize * 3)
keyPtrPtr = CPyMarshal.Offset(posPtr, CPyMarshal.PtrSize)
valuePtrPtr = CPyMarshal.Offset(keyPtrPtr, CPyMarshal.PtrSize)
addDealloc(lambda: Marshal.FreeHGlobal(posPtr))
CPyMarshal.WriteInt(posPtr, 0)
d = dict(a=1)
dPtr = mapper.Store(d)
mapper.EnsureGIL()
mapper.PyDict_Next(dPtr, posPtr, keyPtrPtr, valuePtrPtr)
keyPtr = CPyMarshal.ReadPtr(keyPtrPtr)
valuePtr = CPyMarshal.ReadPtr(valuePtrPtr)
# grab extra references to retard spoilage
mapper.IncRef(keyPtr)
mapper.IncRef(valuePtr)
mapper.ReleaseGIL()
# check refcount has dropped back to 1
self.assertEquals(mapper.RefCount(keyPtr), 1)
self.assertEquals(mapper.RefCount(valuePtr), 1)
def testPySequence_SetItem_List(self, mapper, _):
seq = [1, 2, 3]
seqPtr = mapper.Store(seq)
seqData = CPyMarshal.ReadPtrField(seqPtr, PyListObject, "ob_item")
for i in range(3):
itemPtr = mapper.Store(i)
self.assertEquals(mapper.PySequence_SetItem(seqPtr, i, itemPtr), 0)
self.assertEquals(mapper.RefCount(itemPtr), 2)
index = i
if index < 0:
index += 3
valData = CPyMarshal.Offset(seqData, index * CPyMarshal.PtrSize)
self.assertEquals(CPyMarshal.ReadPtr(valData), itemPtr)
self.assertEquals(seq[i], i)
for i in (5, 66):
mapper.LastException = None
self.assertEquals(mapper.PySequence_SetItem(seqPtr, i, mapper.Store(i)), -1)
self.assertMapperHasError(mapper, IndexError)
def test_PyTuple_Resize(self):
allocs = []
mapper = PythonMapper(GetAllocatingTestAllocator(allocs, []))
tuplePtrPtr = Marshal.AllocHGlobal(CPyMarshal.PtrSize)
oldTuplePtr = mapper.PyTuple_New(1)
del allocs[:]
CPyMarshal.WritePtr(tuplePtrPtr, oldTuplePtr)
self.assertEquals(mapper._PyTuple_Resize(tuplePtrPtr, 100), 0)
newTuplePtr = CPyMarshal.ReadPtr(tuplePtrPtr)
expectedSize = Marshal.SizeOf(PyTupleObject) + (CPyMarshal.PtrSize * (99))
self.assertEquals(allocs, [(newTuplePtr, expectedSize)])
tupleStruct = Marshal.PtrToStructure(newTuplePtr, PyTupleObject)
self.assertEquals(tupleStruct.ob_refcnt, 1)
self.assertEquals(tupleStruct.ob_type, mapper.PyTuple_Type)
self.assertEquals(tupleStruct.ob_size, 100)
mapper.Dispose()
Marshal.FreeHGlobal(tuplePtrPtr)
