def assertPyTuple_New_Works(self, length):
allocs = []
mapper = PythonMapper(GetAllocatingTestAllocator(allocs, []))
typeBlock = Marshal.AllocHGlobal(Marshal.SizeOf(PyTypeObject))
mapper.RegisterData("PyTuple_Type", typeBlock)
tuplePtr = mapper.PyTuple_New(length)
expectedSize = Marshal.SizeOf(PyTupleObject) + (CPyMarshal.PtrSize * (length - 1))
self.assertEquals(allocs, [(tuplePtr, expectedSize)], "bad alloc")
tupleStruct = Marshal.PtrToStructure(tuplePtr, PyTupleObject)
self.assertEquals(tupleStruct.ob_refcnt, 1, "bad refcount")
self.assertEquals(tupleStruct.ob_type, mapper.PyTuple_Type, "bad type")
self.assertEquals(tupleStruct.ob_size, length, "bad size")
self.assertEquals(mapper.PyTuple_Size(tuplePtr), length, "should still work with uninitialised tuple imo")
dataPtr = OffsetPtr(tuplePtr, Marshal.OffsetOf(PyTupleObject, "ob_item"))
itemPtrs = []
for i in range(length):
self.assertEquals(CPyMarshal.ReadPtr(dataPtr), IntPtr.Zero, "item memory not zeroed")
itemPtr = mapper.Store(i + 100)
CPyMarshal.WritePtr(dataPtr, itemPtr)
itemPtrs.append(itemPtr)
dataPtr = OffsetPtr(dataPtr, CPyMarshal.PtrSize)
immutableTuple = mapper.Retrieve(tuplePtr)
self.assertEquals(immutableTuple, tuple(i + 100 for i in range(length)), "broken")
tuplePtr2 = mapper.Store(immutableTuple)
self.assertEquals(tuplePtr2, tuplePtr, "didn't realise already had this object stored")
self.assertEquals(mapper.RefCount(tuplePtr), 2, "didn't incref")
mapper.Dispose()
Marshal.FreeHGlobal(typeBlock)
def testStoreTupleCreatesTupleType(self):
allocs = []
mapper = PythonMapper(GetAllocatingTestAllocator(allocs, []))
typeBlock = Marshal.AllocHGlobal(Marshal.SizeOf(PyTypeObject()))
mapper.RegisterData("PyTuple_Type", typeBlock)
theTuple = (0, 1, 2)
tuplePtr = mapper.Store(theTuple)
self.assertEquals(
CPyMarshal.ReadPtrField(tuplePtr, PyTupleObject, "ob_type"),
typeBlock, "wrong type")
dataPtr = OffsetPtr(tuplePtr, Marshal.OffsetOf(PyTupleObject,
"ob_item"))
for i in range(3):
item = mapper.Retrieve(CPyMarshal.ReadPtr(dataPtr))
self.assertEquals(item, i, "did not store data")
dataPtr = OffsetPtr(dataPtr, CPyMarshal.PtrSize)
tuplePtr2 = mapper.Store(theTuple)
self.assertEquals(tuplePtr2, tuplePtr,
"didn't realise already had this tuple")
self.assertEquals(mapper.RefCount(tuplePtr), 2, "didn't incref")
mapper.Dispose()
Marshal.FreeHGlobal(typeBlock)
def testSomeItems(self):
allocs = []
mapper = PythonMapper(GetAllocatingTestAllocator(allocs, []))
deallocTypes = CreateTypes(mapper)
typeSpec = {
"tp_basicsize": 32,
"tp_itemsize": 64,
}
typePtr, deallocType = MakeTypePtr(mapper, typeSpec)
del allocs[:]
result = mapper.PyType_GenericAlloc(typePtr, 3)
self.assertEquals(allocs, [(result, 224)], "allocated wrong")
refcount = CPyMarshal.ReadIntField(result, PyObject, "ob_refcnt")
self.assertEquals(refcount, 1, "bad initialisation")
instanceType = CPyMarshal.ReadPtrField(result, PyObject, "ob_type")
self.assertEquals(instanceType, typePtr, "bad type ptr")
size = CPyMarshal.ReadIntField(result, PyVarObject, "ob_size")
self.assertEquals(size, 3, "bad ob_size")
headerSize = Marshal.SizeOf(PyVarObject)
zerosPtr = OffsetPtr(result, headerSize)
for i in range(224 - headerSize):
self.assertEquals(CPyMarshal.ReadByte(zerosPtr), 0, "not zeroed")
zerosPtr = OffsetPtr(zerosPtr, 1)
mapper.Dispose()
deallocTypes()
deallocType()
def CreateInstance(mapper, calls):
tuplePtr = mapper.PyTuple_New(len(model))
dataPtr = OffsetPtr(tuplePtr, Marshal.OffsetOf(PyTupleObject, "ob_item"))
for i in range(len(model)):
itemPtr = mapper.Store(model[i])
itemPtrs.append(itemPtr)
CPyMarshal.WritePtr(dataPtr, itemPtr)
dataPtr = OffsetPtr(dataPtr, CPyMarshal.PtrSize)
return tuplePtr
def CreateInstance(mapper, calls):
listPtr = mapper.Store([1, 2, 3])
dataStore = CPyMarshal.ReadPtrField(listPtr, PyListObject, "ob_item")
for _ in range(3):
itemPtrs.append(CPyMarshal.ReadPtr(dataStore))
dataStore = OffsetPtr(dataStore, CPyMarshal.PtrSize)
return listPtr
def testPyList_New_NonZeroLength(self):
allocs = []
mapper = PythonMapper(GetAllocatingTestAllocator(allocs, []))
deallocTypes = CreateTypes(mapper)
del allocs[:]
SIZE = 27
listPtr = mapper.PyList_New(SIZE)
listStruct = Marshal.PtrToStructure(listPtr, PyListObject)
self.assertEquals(listStruct.ob_refcnt, 1, "bad refcount")
self.assertEquals(listStruct.ob_type, mapper.PyList_Type, "bad type")
self.assertEquals(listStruct.ob_size, SIZE, "bad ob_size")
self.assertEquals(listStruct.allocated, SIZE, "bad allocated")
dataPtr = listStruct.ob_item
self.assertNotEquals(dataPtr, IntPtr.Zero, "failed to allocate space for data")
expectedAllocs = [(dataPtr, (SIZE * CPyMarshal.PtrSize)), (listPtr, Marshal.SizeOf(PyListObject))]
self.assertEquals(set(allocs), set(expectedAllocs), "allocated wrong")
for _ in range(SIZE):
self.assertEquals(CPyMarshal.ReadPtr(dataPtr), IntPtr.Zero, "failed to zero memory")
dataPtr = OffsetPtr(dataPtr, CPyMarshal.PtrSize)
mapper.Dispose()
deallocTypes()
def testPyList_DeallocDecRefsItemsAndCallsCorrectFreeFunction(self):
frees = []
mapper = PythonMapper(GetAllocatingTestAllocator([], frees))
deallocTypes = CreateTypes(mapper)
calls = []
def CustomFree(ptr):
calls.append(ptr)
mapper.PyObject_Free(listPtr)
self.freeDgt = dgt_void_ptr(CustomFree)
CPyMarshal.WriteFunctionPtrField(mapper.PyList_Type, PyTypeObject, "tp_free", self.freeDgt)
listPtr = mapper.Store([1, 2, 3])
itemPtrs = []
dataStore = CPyMarshal.ReadPtrField(listPtr, PyListObject, "ob_item")
for _ in range(3):
itemPtrs.append(CPyMarshal.ReadPtr(dataStore))
dataStore = OffsetPtr(dataStore, CPyMarshal.PtrSize)
mapper.IC_PyList_Dealloc(listPtr)
for itemPtr in itemPtrs:
self.assertEquals(itemPtr in frees, True, "did not decref item")
self.assertEquals(calls, [listPtr], "did not call type's free function")
mapper.Dispose()
deallocTypes()
def testConvertPyFileToDescriptor(self, mapper, _):
testDir = tempfile.mkdtemp()
outFile = os.path.join(testDir, "test")
pyFile = file(outFile, 'w')
pyFile.write("I love streams ")
pyFile.flush()
fd = mapper.ConvertPyFileToDescriptor(pyFile)
FILE = Unmanaged._fdopen(fd, "w")
testStr = "and file descriptors!"
testStrPtr = mapper.Store(testStr)
testDataPtr = OffsetPtr(testStrPtr,
Marshal.OffsetOf(PyStringObject, "ob_sval"))
self.assertTrue(
Unmanaged.fwrite(testDataPtr, 1, len(testStr), FILE) > 0,
"writing failed")
Unmanaged.fflush(FILE)
pyFile.close()
stream = file(outFile, 'r')
output = stream.read()
stream.close()
self.assertEquals(output, "I love streams and file descriptors!")
def testIC_PyFile_AsFile_Write(self, mapper, addToCleanUp, _):
testDir = tempfile.mkdtemp()
addToCleanUp(lambda: shutil.rmtree(testDir))
path = os.path.join(testDir, "test")
testStr = "meh, string data"
testLength = len(testStr)
testStrPtr = mapper.Store(testStr)
testDataPtr = OffsetPtr(testStrPtr, Marshal.OffsetOf(PyStringObject, "ob_sval"))
filePtr = mapper.Store(open(path, 'w'))
f = mapper.IC_PyFile_AsFile(filePtr)
self.assertEquals(Unmanaged.fwrite(testDataPtr, 1, testLength, f), testLength, "didn't work")
# nasty test: patch out PyObject_Free
# the memory will not be deallocated, but the FILE handle should be
calls = []
def Free(ptr):
calls.append(ptr)
freeDgt = dgt_void_ptr(Free)
CPyMarshal.WriteFunctionPtrField(mapper.PyFile_Type, PyTypeObject, 'tp_free', freeDgt)
mapper.DecRef(filePtr)
self.assertEquals(calls, [filePtr], 'failed to call tp_free function')
mgdF = open(path)
result = mgdF.read()
self.assertEquals(result, testStr, "failed to write (got >>%s<<) -- deallocing filePtr did not close FILE" % result)
mgdF.close()
def assertStringObjectHasLength(self, strPtr, length):
stringObject = Marshal.PtrToStructure(strPtr, PyStringObject)
self.assertEquals(stringObject.ob_refcnt, 1, "unexpected refcount")
self.assertEquals(stringObject.ob_size, length, "unexpected ob_size")
self.assertEquals(stringObject.ob_shash, -1,
"unexpected currently-useless-field")
self.assertEquals(stringObject.ob_sstate, 0,
"unexpected currently-useless-field")
strDataPtr = self.dataPtrFromStrPtr(strPtr)
terminatorPtr = OffsetPtr(strDataPtr, length)
self.assertEquals(Marshal.ReadByte(terminatorPtr), 0,
"string not terminated")
def testZero(self):
bufferlen = 200
zerolen = 173
data = Marshal.AllocHGlobal(bufferlen)
this = data
for _ in xrange(bufferlen):
CPyMarshal.WriteByte(this, 255)
this = OffsetPtr(this, 1)
CPyMarshal.Zero(data, zerolen)
this = data
for i in xrange(bufferlen):
actual = CPyMarshal.ReadByte(this)
expected = (255, 0)[i < zerolen]
self.assertEquals(
actual, expected,
"wrong value at %d (%d, %d)" % (i, actual, expected))
this = OffsetPtr(this, 1)
Marshal.FreeHGlobal(data)
def testStoreList(self, mapper, _):
list_ = [1, 2, 3]
listPtr = mapper.Store(list_)
self.assertEquals(id(mapper.Retrieve(listPtr)), id(list_))
typePtr = CPyMarshal.ReadPtrField(listPtr, PyObject, "ob_type")
self.assertEquals(typePtr, mapper.PyList_Type, "wrong type")
dataStore = CPyMarshal.ReadPtrField(listPtr, PyListObject, "ob_item")
for i in range(1, 4):
self.assertEquals(mapper.Retrieve(CPyMarshal.ReadPtr(dataStore)), i, "contents not stored")
self.assertEquals(mapper.RefCount(CPyMarshal.ReadPtr(dataStore)), 1, "bad refcount for items")
dataStore = OffsetPtr(dataStore, CPyMarshal.PtrSize)
def testIC_PyFile_AsFile(self, mapper, addToCleanUp, stderr_writes):
buflen = len(TEST_TEXT) + 10
buf = Marshal.AllocHGlobal(buflen)
filePtr = mapper.Store(open(*READ_ARGS))
f = mapper.IC_PyFile_AsFile(filePtr)
self.assertEquals(stderr_writes, [('Warning: creating unmanaged FILE* from managed stream. Please use ironclad.open with this extension.',), ('\n',)])
self.assertEquals(Unmanaged.fread(buf, 1, buflen, f), len(TEST_TEXT), "didn't get a real FILE")
ptr = buf
for c in TEST_TEXT:
self.assertEquals(Marshal.ReadByte(ptr), ord(c), "got bad data from FILE")
ptr = OffsetPtr(ptr, 1)
Marshal.FreeHGlobal(buf)
def MakeItemsTablePtr(items):
if not items:
return IntPtr.Zero, lambda: None
itemtype = items[0].__class__
typesize = Marshal.SizeOf(itemtype())
size = typesize * (len(items) + 1)
tablePtr = Marshal.AllocHGlobal(size)
CPyMarshal.Zero(tablePtr, size)
for i, item in enumerate(items):
Marshal.StructureToPtr(item, OffsetPtr(tablePtr, typesize * i), False)
def dealloc():
Marshal.DestroyStructure(tablePtr, itemtype)
Marshal.FreeHGlobal(tablePtr)
return tablePtr, dealloc
def testPyList_Append(self):
allocs = []
deallocs = []
mapper = PythonMapper(GetAllocatingTestAllocator(allocs, deallocs))
deallocTypes = CreateTypes(mapper)
del allocs[:]
listPtr = mapper.PyList_New(0)
self.assertEquals(allocs, [(listPtr, Marshal.SizeOf(PyListObject))], "bad alloc")
item1 = object()
item2 = object()
itemPtr1 = mapper.Store(item1)
itemPtr2 = mapper.Store(item2)
self.assertEquals(mapper.PyList_Append(listPtr, itemPtr1), 0, "failed to report success")
self.assertEquals(len(allocs), 4, "didn't allocate memory for data store (list; item1; item2; data store comes 4th)")
dataPtrAfterFirstAppend = CPyMarshal.ReadPtrField(listPtr, PyListObject, "ob_item")
self.assertEquals(allocs[3], (dataPtrAfterFirstAppend, CPyMarshal.PtrSize), "allocated wrong amount of memory")
self.assertEquals(CPyMarshal.ReadPtr(dataPtrAfterFirstAppend), itemPtr1, "failed to fill memory")
self.assertEquals(mapper.RefCount(itemPtr1), 2, "failed to incref new contents")
self.assertEquals(mapper.Retrieve(listPtr), [item1], "retrieved wrong list")
# make refcount 1, to prove that references are not lost when reallocing data
mapper.DecRef(itemPtr1)
self.assertEquals(mapper.PyList_Append(listPtr, itemPtr2), 0, "failed to report success")
self.assertEquals(len(allocs), 5, "didn't allocate memory for new, larger data store")
self.assertEquals(deallocs, [dataPtrAfterFirstAppend])
dataPtrAfterSecondAppend = CPyMarshal.ReadPtrField(listPtr, PyListObject, "ob_item")
self.assertEquals(allocs[4], (dataPtrAfterSecondAppend, (CPyMarshal.PtrSize * 2)),
"allocated wrong amount of memory")
self.assertEquals(CPyMarshal.ReadPtr(dataPtrAfterSecondAppend), itemPtr1,
"failed to keep reference to first item")
self.assertEquals(CPyMarshal.ReadPtr(OffsetPtr(dataPtrAfterSecondAppend, CPyMarshal.PtrSize)), itemPtr2,
"failed to keep reference to first item")
self.assertEquals(mapper.RefCount(itemPtr1), 1, "wrong refcount for item existing only in list")
self.assertEquals(mapper.RefCount(itemPtr2), 2, "wrong refcount newly-added item")
self.assertEquals(mapper.Retrieve(listPtr), [item1, item2], "retrieved wrong list")
mapper.Dispose()
deallocTypes()
def dataPtrFromStrPtr(self, strPtr):
return OffsetPtr(strPtr, Marshal.OffsetOf(PyStringObject, "ob_sval"))
def ptrFromByteArray(self, bytes):
testData = Marshal.AllocHGlobal(bytes.Length + 1)
Marshal.Copy(bytes, 0, testData, bytes.Length)
Marshal.WriteByte(OffsetPtr(testData, bytes.Length), 0)
return testData
def WriteBytes(address):
for a in range(intCount):
ptr = OffsetPtr(address, a * CPyMarshal.IntSize)
Marshal.WriteInt32(ptr, TEST_NUMBER)
def TestWroteBytes(address):
for a in range(intCount):
ptr = OffsetPtr(address, a * CPyMarshal.IntSize)
data = Marshal.ReadInt32(ptr)
if data != TEST_NUMBER:
raise AssertionError("write failed")
