def test_fixedsizearray(self): py.test.skip("fails because of the interior structure changes") A = lltype.FixedSizeArray(lltype.Signed, 3) S = lltype.GcStruct('S', ('a', A)) def fn(n1, n2): s = lltype.malloc(S) a = s.a a[0] = n1 a[2] = n2 return a[0] - a[2] self.check(fn, [int, int], [100, 42], 58)
def test_getarraysubstruct(self): py.test.skip("llptr support not really useful any more") U = lltype.Struct('U', ('n', lltype.Signed)) for length in [1, 2]: S = lltype.GcStruct('S', ('a', lltype.FixedSizeArray(U, length))) for index in range(length): def fn(): s = lltype.malloc(S) s.a[index].n = 12 return s.a[index].n self.check(fn, [], [], 12)
def test_getarraysubstruct(self): py.test.skip("fails because of the interior structure changes") U = lltype.Struct('U', ('n', lltype.Signed)) for length in [1, 2]: S = lltype.GcStruct('S', ('a', lltype.FixedSizeArray(U, length))) for index in range(length): def fn(): s = lltype.malloc(S) s.a[index].n = 12 return s.a[index].n self.check(fn, [], [], 12)
def test_fixedsizearray(self): A = lltype.FixedSizeArray(lltype.Signed, 3) S = lltype.GcStruct('S', ('a', A)) def fn(n1, n2): s = lltype.malloc(S) a = s.a a[0] = n1 a[2] = n2 return a[0] - a[2] self.check(fn, [int, int], [100, 42], 58, expected_mallocs=1) # no support for interior arrays
def register_time_clock(self): if sys.platform == 'win32': # hacking to avoid LARGE_INTEGER which is a union... A = lltype.FixedSizeArray(lltype.SignedLongLong, 1) QueryPerformanceCounter = self.llexternal( 'QueryPerformanceCounter', [lltype.Ptr(A)], lltype.Void, threadsafe=False) QueryPerformanceFrequency = self.llexternal( 'QueryPerformanceFrequency', [lltype.Ptr(A)], rffi.INT, threadsafe=False) class State(object): pass state = State() state.divisor = 0.0 state.counter_start = 0 def time_clock_llimpl(): a = lltype.malloc(A, flavor='raw') if state.divisor == 0.0: QueryPerformanceCounter(a) state.counter_start = a[0] QueryPerformanceFrequency(a) state.divisor = float(a[0]) QueryPerformanceCounter(a) diff = a[0] - state.counter_start lltype.free(a, flavor='raw') return float(diff) / state.divisor else: RUSAGE = self.RUSAGE RUSAGE_SELF = self.RUSAGE_SELF or 0 c_getrusage = self.llexternal( 'getrusage', [rffi.INT, lltype.Ptr(RUSAGE)], lltype.Void, threadsafe=False) def time_clock_llimpl(): a = lltype.malloc(RUSAGE, flavor='raw') c_getrusage(RUSAGE_SELF, a) result = (decode_timeval(a.c_ru_utime) + decode_timeval(a.c_ru_stime)) lltype.free(a, flavor='raw') return result return extdef([], float, llimpl=time_clock_llimpl, export_name='ll_time.ll_time_clock')
def _raw_malloc(self, rest, zero): assert not rest if (isinstance(self.TYPE, lltype.ContainerType) and self.TYPE._gckind == 'gc'): assert self.repeat == 1 p = lltype.malloc(self.TYPE, flavor='raw', zero=zero, track_allocation=False) return cast_ptr_to_adr(p) else: T = lltype.FixedSizeArray(self.TYPE, self.repeat) p = lltype.malloc(T, flavor='raw', zero=zero, track_allocation=False) array_adr = cast_ptr_to_adr(p) return array_adr + ArrayItemsOffset(T)
class CConfig: _compilation_info_ = ExternalCompilationInfo( includes=['windows.h', 'winbase.h', 'sys/stat.h'], ) WIN32_FIND_DATA = platform.Struct( 'struct _WIN32_FIND_DATA' + suffix, # Only interesting fields [('dwFileAttributes', rwin32.DWORD), ('nFileSizeHigh', rwin32.DWORD), ('nFileSizeLow', rwin32.DWORD), ('ftCreationTime', rwin32.FILETIME), ('ftLastAccessTime', rwin32.FILETIME), ('ftLastWriteTime', rwin32.FILETIME), ('cFileName', lltype.FixedSizeArray(traits.CHAR, 250))]) ERROR_FILE_NOT_FOUND = platform.ConstantInteger('ERROR_FILE_NOT_FOUND') ERROR_NO_MORE_FILES = platform.ConstantInteger('ERROR_NO_MORE_FILES') GetFileExInfoStandard = platform.ConstantInteger( 'GetFileExInfoStandard') FILE_ATTRIBUTE_DIRECTORY = platform.ConstantInteger( 'FILE_ATTRIBUTE_DIRECTORY') FILE_ATTRIBUTE_READONLY = platform.ConstantInteger( 'FILE_ATTRIBUTE_READONLY') INVALID_FILE_ATTRIBUTES = platform.ConstantInteger( 'INVALID_FILE_ATTRIBUTES') ERROR_SHARING_VIOLATION = platform.ConstantInteger( 'ERROR_SHARING_VIOLATION') _S_IFDIR = platform.ConstantInteger('_S_IFDIR') _S_IFREG = platform.ConstantInteger('_S_IFREG') _S_IFCHR = platform.ConstantInteger('_S_IFCHR') _S_IFIFO = platform.ConstantInteger('_S_IFIFO') FILE_TYPE_UNKNOWN = platform.ConstantInteger('FILE_TYPE_UNKNOWN') FILE_TYPE_CHAR = platform.ConstantInteger('FILE_TYPE_CHAR') FILE_TYPE_PIPE = platform.ConstantInteger('FILE_TYPE_PIPE') WIN32_FILE_ATTRIBUTE_DATA = platform.Struct( 'WIN32_FILE_ATTRIBUTE_DATA', [('dwFileAttributes', rwin32.DWORD), ('nFileSizeHigh', rwin32.DWORD), ('nFileSizeLow', rwin32.DWORD), ('ftCreationTime', rwin32.FILETIME), ('ftLastAccessTime', rwin32.FILETIME), ('ftLastWriteTime', rwin32.FILETIME)]) BY_HANDLE_FILE_INFORMATION = platform.Struct( 'BY_HANDLE_FILE_INFORMATION', [('dwFileAttributes', rwin32.DWORD), ('nFileSizeHigh', rwin32.DWORD), ('nFileSizeLow', rwin32.DWORD), ('nNumberOfLinks', rwin32.DWORD), ('nFileIndexHigh', rwin32.DWORD), ('nFileIndexLow', rwin32.DWORD), ('ftCreationTime', rwin32.FILETIME), ('ftLastAccessTime', rwin32.FILETIME), ('ftLastWriteTime', rwin32.FILETIME)])
def test_frexp(self): if sys.platform != 'win32': eci = ExternalCompilationInfo(includes=['math.h'], libraries=['m']) else: eci = ExternalCompilationInfo(includes=['math.h']) A = lltype.FixedSizeArray(rffi.INT, 1) frexp = rffi.llexternal('frexp', [rffi.DOUBLE, lltype.Ptr(A)], rffi.DOUBLE, compilation_info=eci) p = lltype.malloc(A, flavor='raw') res = frexp(2.5, p) assert res == 0.625 assert p[0] == 2 lltype.free(p, flavor='raw') assert not ALLOCATED # detects memory leaks in the test
def test_force_cast(self): from pypy.rpython.annlowlevel import llstr from pypy.rpython.lltypesystem.rstr import STR from pypy.rpython.lltypesystem import rffi, llmemory, lltype P = lltype.Ptr(lltype.FixedSizeArray(lltype.Char, 1)) def f(): a = llstr("xyz") b = (llmemory.cast_ptr_to_adr(a) + llmemory.offsetof(STR, 'chars') + llmemory.itemoffsetof(STR.chars, 0)) buf = rffi.cast(rffi.VOIDP, b) return buf[2] fn = self.getcompiled(f, []) res = fn() assert res == 'z'
def test_adr_cast(self): from pypy.rpython.annlowlevel import llstr from pypy.rpython.lltypesystem.rstr import STR P = lltype.Ptr(lltype.FixedSizeArray(lltype.Char, 1)) def f(): a = llstr("xyz") b = (llmemory.cast_ptr_to_adr(a) + llmemory.offsetof(STR, 'chars') + llmemory.itemoffsetof(STR.chars, 0)) buf = rffi.cast(rffi.VOIDP, b) return buf[2] assert f() == 'z' res = interpret(f, []) assert res == 'z'
def register_os_pipe(self): # we need a different approach on Windows and on Posix if sys.platform.startswith('win'): HANDLE = rffi.ULONG HANDLEP = lltype.Ptr(lltype.FixedSizeArray(HANDLE, 1)) CreatePipe = self.llexternal('CreatePipe', [HANDLEP, HANDLEP, rffi.VOIDP, rffi.ULONG], rffi.INT) _open_osfhandle = self.llexternal('_open_osfhandle', [rffi.ULONG, rffi.INT], rffi.INT) null = lltype.nullptr(rffi.VOIDP.TO) def os_pipe_llimpl(): pread = lltype.malloc(HANDLEP.TO, flavor='raw') pwrite = lltype.malloc(HANDLEP.TO, flavor='raw') ok = CreatePipe(pread, pwrite, null, 0) hread = pread[0] hwrite = pwrite[0] lltype.free(pwrite, flavor='raw') lltype.free(pread, flavor='raw') if not ok: # XXX guess the error, can't use GetLastError() raise OSError(errno.EMFILE, "os_pipe failed") fdread = _open_osfhandle(hread, 0) fdwrite = _open_osfhandle(hwrite, 1) return (fdread, fdwrite) else: INT_ARRAY_P = rffi.CArrayPtr(rffi.INT) os_pipe = self.llexternal('pipe', [INT_ARRAY_P], rffi.INT) def os_pipe_llimpl(): filedes = lltype.malloc(INT_ARRAY_P.TO, 2, flavor='raw') error = rffi.cast(lltype.Signed, os_pipe(filedes)) read_fd = filedes[0] write_fd = filedes[1] lltype.free(filedes, flavor='raw') if error != 0: raise OSError(rposix.get_errno(), "os_pipe failed") return (rffi.cast(lltype.Signed, read_fd), rffi.cast(lltype.Signed, write_fd)) return extdef([], (int, int), "ll_os.ll_os_pipe", llimpl=os_pipe_llimpl)
def raw_memcopy(self, srcadr, dstadr): repeat = self.repeat if repeat == 0: return if isinstance(self.TYPE, lltype.ContainerType): PTR = lltype.Ptr(self.TYPE) else: PTR = lltype.Ptr(lltype.FixedSizeArray(self.TYPE, 1)) while True: src = cast_adr_to_ptr(srcadr, PTR) dst = cast_adr_to_ptr(dstadr, PTR) _reccopy(src, dst) repeat -= 1 if repeat <= 0: break srcadr += ItemOffset(self.TYPE) dstadr += ItemOffset(self.TYPE)
def get_chunk_manager(chunk_size=DEFAULT_CHUNK_SIZE, cache={}): try: return cache[chunk_size] except KeyError: pass CHUNK = lltype.ForwardReference() CHUNK.become( lltype.Struct( 'AddressChunk', ('next', lltype.Ptr(CHUNK)), ('items', lltype.FixedSizeArray(llmemory.Address, chunk_size)))) null_chunk = lltype.nullptr(CHUNK) class FreeList(object): _alloc_flavor_ = "raw" def __init__(self): self.free_list = null_chunk def get(self): if not self.free_list: # we zero-initialize the chunks to make the translation # backends happy, but we don't need to do it at run-time. zero = not we_are_translated() return lltype.malloc(CHUNK, flavor="raw", zero=zero, track_allocation=False) result = self.free_list self.free_list = result.next return result def put(self, chunk): if we_are_translated(): chunk.next = self.free_list self.free_list = chunk else: # Don't cache the old chunks but free them immediately. # Helps debugging, and avoids that old chunks full of # addresses left behind by a test end up in genc... lltype.free(chunk, flavor="raw", track_allocation=False) unused_chunks = FreeList() cache[chunk_size] = unused_chunks, null_chunk return unused_chunks, null_chunk
def test_fakeaccessor(): S = lltype.GcStruct("S", ("x", lltype.Signed), ("y", lltype.Signed)) s = lltype.malloc(S) s.x = 123 s.y = 456 adr = cast_ptr_to_adr(s) adr += FieldOffset(S, "y") assert adr.signed[0] == 456 adr.signed[0] = 789 assert s.y == 789 A = lltype.GcArray(lltype.Signed) a = lltype.malloc(A, 5) a[3] = 123 adr = cast_ptr_to_adr(a) assert (adr + ArrayLengthOffset(A)).signed[0] == 5 assert (adr + ArrayItemsOffset(A)).signed[3] == 123 (adr + ArrayItemsOffset(A)).signed[3] = 456 assert a[3] == 456 adr1000 = (adr + ArrayItemsOffset(A) + ItemOffset(lltype.Signed, 1000)) assert adr1000.signed[-997] == 456 A = lltype.GcArray(lltype.Char) a = lltype.malloc(A, 5) a[3] = '*' adr = cast_ptr_to_adr(a) assert (adr + ArrayLengthOffset(A)).signed[0] == 5 assert (adr + ArrayItemsOffset(A)).char[3] == '*' (adr + ArrayItemsOffset(A)).char[3] = '+' assert a[3] == '+' adr1000 = (adr + ArrayItemsOffset(A) + ItemOffset(lltype.Char, 1000)) assert adr1000.char[-997] == '+' T = lltype.FixedSizeArray(lltype.Char, 10) S = lltype.GcStruct('S', ('z', lltype.Ptr(T))) s = lltype.malloc(S) s.z = lltype.malloc(T, immortal=True) adr = cast_ptr_to_adr(s) assert (adr + offsetof(S, 'z')).address[0] == cast_ptr_to_adr(s.z) (adr + offsetof(S, 'z')).address[0] = NULL assert s.z == lltype.nullptr(T) t = lltype.malloc(T, immortal=True) (adr + offsetof(S, 'z')).address[0] = cast_ptr_to_adr(t) assert s.z == t
def test_itemoffsetof_fixedsizearray(): ARRAY = lltype.FixedSizeArray(lltype.Signed, 5) itemoffsets = [llmemory.itemoffsetof(ARRAY, i) for i in range(5)] a = lltype.malloc(ARRAY, immortal=True) def f(): adr = llmemory.cast_ptr_to_adr(a) result = 0 for i in range(5): a[i] = i + 1 for i in range(5): result = result * 10 + (adr + itemoffsets[i]).signed[0] for i in range(5): (adr + itemoffsets[i]).signed[0] = i for i in range(5): result = 10 * result + a[i] return result fn, t = getcompiled(f, []) res = fn() assert res == 1234501234
def test_raw_memcopy_nonrec(): T = lltype.GcStruct('T', ('x', lltype.Signed)) A = lltype.FixedSizeArray(lltype.Ptr(T), 1) t1 = lltype.malloc(T) t2 = lltype.malloc(T) t1.x = 1 t2.x = 2 at1 = raw_malloc(sizeof(A)) at2 = raw_malloc(sizeof(A)) p1 = cast_adr_to_ptr(at1, lltype.Ptr(A)) p2 = cast_adr_to_ptr(at2, lltype.Ptr(A)) p1[0] = t1 p2[0] = t2 raw_memcopy(at1, at2, sizeof(A)) assert p1[0] == t1 assert p2[0] == t1 assert t1.x == 1 # not assert t2.x == 2 # modified
def register_time_clock(self): c_clock = self.llexternal('clock', [], self.CLOCK_T, threadsafe=False) if sys.platform == 'win32': # hacking to avoid LARGE_INTEGER which is a union... A = lltype.FixedSizeArray(lltype.SignedLongLong, 1) QueryPerformanceCounter = self.llexternal( 'QueryPerformanceCounter', [lltype.Ptr(A)], lltype.Void, threadsafe=False) QueryPerformanceFrequency = self.llexternal( 'QueryPerformanceFrequency', [lltype.Ptr(A)], rffi.INT, threadsafe=False) class State(object): pass state = State() state.divisor = 0.0 state.counter_start = 0 def time_clock_llimpl(): a = lltype.malloc(A, flavor='raw') if state.divisor == 0.0: QueryPerformanceCounter(a) state.counter_start = a[0] QueryPerformanceFrequency(a) state.divisor = float(a[0]) QueryPerformanceCounter(a) diff = a[0] - state.counter_start lltype.free(a, flavor='raw') return float(diff) / state.divisor else: def time_clock_llimpl(): result = c_clock() return float(result) / self.CLOCKS_PER_SEC return extdef([], float, llimpl=time_clock_llimpl, export_name='ll_time.ll_time_clock')
def test_keep_all_keepalives(self): SIZE = llmemory.sizeof(lltype.Signed) PARRAY = lltype.Ptr(lltype.FixedSizeArray(lltype.Signed, 1)) class A: def __init__(self): self.addr = llmemory.raw_malloc(SIZE) def __del__(self): llmemory.raw_free(self.addr) class B: pass def myfunc(): b = B() b.keep = A() b.data = llmemory.cast_adr_to_ptr(b.keep.addr, PARRAY) b.data[0] = 42 ptr = b.data # normally 'b' could go away as early as here, which would free # the memory held by the instance of A in b.keep... res = ptr[0] # ...so we explicitly keep 'b' alive until here objectmodel.keepalive_until_here(b) return res graph = self.check(myfunc, [], [], 42, must_be_removed=False) # 'A' instance left # there is a getarrayitem near the end of the graph of myfunc. # However, the memory it accesses must still be protected by the # following keepalive, even after malloc removal entrymap = mkentrymap(graph) [link] = entrymap[graph.returnblock] assert link.prevblock.operations[-1].opname == 'keepalive'
def define_tree_cloning(cls): import os # this makes a tree of calls. Each leaf stores its path (a linked # list) in 'result'. Paths are mutated in-place but the leaves don't # see each other's mutations because of x_clone. STUFF = lltype.FixedSizeArray(lltype.Signed, 21) NODE = lltype.GcForwardReference() NODE.become(lltype.GcStruct('node', ('index', lltype.Signed), ('counter', lltype.Signed), ('next', lltype.Ptr(NODE)), ('use_some_space', STUFF))) PATHARRAY = lltype.GcArray(lltype.Ptr(NODE)) clonedata = lltype.malloc(X_CLONE) def clone(node): # that's for testing if the test is correct... if not node: return node newnode = lltype.malloc(NODE) newnode.index = node.index newnode.counter = node.counter newnode.next = clone(node.next) return newnode def do_call(result, path, index, remaining_depth): # clone the while path clonedata.gcobjectptr = lltype.cast_opaque_ptr(llmemory.GCREF, path) clonedata.pool = lltype.nullptr(X_POOL) llop.gc_x_clone(lltype.Void, clonedata) # install the new pool as the current one parentpool = llop.gc_x_swap_pool(X_POOL_PTR, clonedata.pool) path = lltype.cast_opaque_ptr(lltype.Ptr(NODE), clonedata.gcobjectptr) # The above should have the same effect as: # path = clone(path) # bump all the path node counters by one p = path while p: p.counter += 1 p = p.next if remaining_depth == 0: llop.debug_print(lltype.Void, "setting", index, "with", path) result[index] = path # leaf else: node = lltype.malloc(NODE) node.index = index * 2 node.counter = 0 node.next = path do_call(result, node, index * 2, remaining_depth - 1) node.index += 1 # mutation! do_call(result, node, index * 2 + 1, remaining_depth - 1) # restore the parent pool llop.gc_x_swap_pool(X_POOL_PTR, parentpool) def check(path, index, level, depth): if level == depth: assert index == 0 assert not path else: assert path.index == index assert path.counter == level + 1 check(path.next, index >> 1, level + 1, depth) def func(depth, dummy): result = lltype.malloc(PATHARRAY, 1 << depth) os.write(2, 'building tree... ') do_call(result, lltype.nullptr(NODE), 0, depth) os.write(2, 'checking tree... ') #from pypy.rpython.lltypesystem.lloperation import llop #llop.debug_view(lltype.Void, result, # llop.gc_x_size_header(lltype.Signed)) for i in range(1 << depth): check(result[i], i, 0, depth) os.write(2, 'ok\n') return 1 return func
if stacklessgc: t.config.translation.gcrootfinder = "stackless" t.config.set(**extraconfigopts) ann = t.buildannotator(policy=annpolicy.StrictAnnotatorPolicy()) ann.build_types(func, inputtypes) if specialize: t.buildrtyper().specialize() if backendopt: from pypy.translator.backendopt.all import backend_optimizations backend_optimizations(t) if conftest.option.view: t.viewcg() return t ARGS = lltype.FixedSizeArray(lltype.Signed, 3) class GCTest(object): gcpolicy = None stacklessgc = False GC_CAN_MOVE = False GC_CANNOT_MALLOC_NONMOVABLE = False taggedpointers = False def setup_class(cls): funcs0 = [] funcs2 = [] cleanups = [] name_to_func = {} mixlevelstuff = [] for fullname in dir(cls):
def setup(): INSPECT = { 'b': 'signed char', 'h': 'signed short', 'i': 'signed int', 'l': 'signed long', 'q': 'signed long long', 'B': 'unsigned char', 'H': 'unsigned short', 'I': 'unsigned int', 'L': 'unsigned long', 'Q': 'unsigned long long', 'P': 'char *', 'f': 'float', 'd': 'double', '?': '_Bool', } pre_include_bits = [ """ #ifdef _MSC_VER #define _Bool char #endif""" ] field_names = dict.fromkeys(INSPECT) for fmtchar, ctype in INSPECT.iteritems(): field_name = ctype.replace(" ", "_").replace("*", "star") field_names[fmtchar] = field_name pre_include_bits.append(""" struct about_%s { char pad; %s field; }; """ % (field_name, ctype)) class CConfig: _compilation_info_ = ExternalCompilationInfo( pre_include_bits=pre_include_bits) for fmtchar, ctype in INSPECT.items(): setattr( CConfig, field_names[fmtchar], rffi_platform.Struct( "struct about_%s" % (field_names[fmtchar], ), [('field', lltype.FixedSizeArray(rffi.CHAR, 1))])) cConfig = rffi_platform.configure(CConfig) for fmtchar, ctype in INSPECT.items(): S = cConfig[field_names[fmtchar]] alignment = rffi.offsetof(S, 'c_field') size = rffi.sizeof(S.c_field) signed = 'a' <= fmtchar <= 'z' if fmtchar == 'f': pack = pack_float unpack = unpack_float elif fmtchar == 'd': pack = pack_double unpack = unpack_double elif fmtchar == '?': pack = std.pack_bool unpack = std.unpack_bool else: pack = std.make_int_packer(size, signed, True) unpack = std.make_int_unpacker(size, signed) native_fmttable[fmtchar] = { 'size': size, 'alignment': alignment, 'pack': pack, 'unpack': unpack }
def CFixedArray(tp, size): return lltype.FixedSizeArray(tp, size)
hop.genop("direct_call", [self.identityhash_ptr, v_adr], resultvar=hop.spaceop.result) def gct_gc_id(self, hop): # this is the logic from the HIDE_POINTER macro in <gc/gc.h> v_int = hop.genop('cast_ptr_to_int', [hop.spaceop.args[0]], resulttype=lltype.Signed) hop.genop('int_invert', [v_int], resultvar=hop.spaceop.result) ########## weakrefs ########## # Boehm: weakref objects are small structures containing only a Boehm # disappearing link. We don't have to hide the link's value with # HIDE_POINTER(), because we explicitly use GC_MALLOC_ATOMIC(). WEAKLINK = lltype.FixedSizeArray(llmemory.Address, 1) sizeof_weakreflink = llmemory.sizeof(WEAKLINK) empty_weaklink = lltype.malloc(WEAKLINK, immortal=True) empty_weaklink[0] = llmemory.NULL def ll_weakref_create(targetaddr): link = llop.boehm_malloc_atomic(llmemory.Address, sizeof_weakreflink) if not link: raise MemoryError plink = llmemory.cast_adr_to_ptr(link, lltype.Ptr(WEAKLINK)) plink[0] = targetaddr llop.boehm_disappearing_link(lltype.Void, link, targetaddr) return llmemory.cast_ptr_to_weakrefptr(plink)
def test_raw_memclear_on_empty_array(): py.test.skip("Fails") A = lltype.FixedSizeArray(lltype.Signed, 0) a = lltype.malloc(A, flavor='raw') src = cast_ptr_to_adr(a) + itemoffsetof(A, 0) raw_memclear(src, sizeof(lltype.Signed) * 0)
class CConfig: _compilation_info_ = eci NCCS = rffi_platform.DefinedConstantInteger('NCCS') NCCS = rffi_platform.configure(CConfig)['NCCS'] TCFLAG_T = rffi.UINT CC_T = rffi.UCHAR SPEED_T = rffi.UINT INT = rffi.INT TERMIOSP = rffi.CStructPtr('termios', ('c_iflag', TCFLAG_T), ('c_oflag', TCFLAG_T), ('c_cflag', TCFLAG_T), ('c_lflag', TCFLAG_T), ('c_cc', lltype.FixedSizeArray(CC_T, NCCS))) def c_external(name, args, result): return rffi.llexternal(name, args, result, compilation_info=eci) c_tcsetattr = c_external('tcsetattr', [INT, INT, TERMIOSP], INT) c_cfgetispeed = c_external('cfgetispeed', [TERMIOSP], SPEED_T) c_cfgetospeed = c_external('cfgetospeed', [TERMIOSP], SPEED_T) c_cfsetispeed = c_external('cfsetispeed', [TERMIOSP, SPEED_T], INT) c_cfsetospeed = c_external('cfsetospeed', [TERMIOSP, SPEED_T], INT) c_tcsendbreak = c_external('tcsendbreak', [INT, INT], INT) c_tcdrain = c_external('tcdrain', [INT], INT) c_tcflush = c_external('tcflush', [INT, INT], INT) c_tcflow = c_external('tcflow', [INT, INT], INT)
# - the value that %esi had when the current function started # - the value that %edi had when the current function started # - the value that %ebp had when the current function started # - frame address (actually the addr of the retaddr of the current function; # that's the last word of the frame in memory) # CALLEE_SAVED_REGS = 4 # there are 4 callee-saved registers INDEX_OF_EBP = 3 FRAME_PTR = CALLEE_SAVED_REGS # the frame is at index 4 in the array ASM_CALLBACK_PTR = lltype.Ptr(lltype.FuncType([], lltype.Void)) # used internally by walk_stack_from() WALKFRAME = lltype.Struct('WALKFRAME', ('regs_stored_at', # address of where the registers have been saved lltype.FixedSizeArray(llmemory.Address, CALLEE_SAVED_REGS)), ('frame_address', llmemory.Address), ) pypy_asm_stackwalk = rffi.llexternal('pypy_asm_stackwalk', [ASM_CALLBACK_PTR], lltype.Signed, sandboxsafe=True, _nowrapper=True) c_asm_stackwalk = Constant(pypy_asm_stackwalk, lltype.typeOf(pypy_asm_stackwalk)) pypy_asm_gcroot = rffi.llexternal('pypy_asm_gcroot', [llmemory.Address], llmemory.Address,
class CConfig: _compilation_info_ = compilation_info DIRENT = platform.Struct('struct dirent', [('d_name', lltype.FixedSizeArray(rffi.CHAR, 1))])
class BlockBuilderMixin(object): _mixin_ = True # A base class to generate assembler. It is equivalent to just a list # of chars, but it is potentially more efficient for that usage. # It works by allocating the assembler SUBBLOCK_SIZE bytes at a time. # Ideally, this number should be a power of two that fits the GC's most # compact allocation scheme (which is so far 35 * WORD for minimark.py). WORD = LONG_BIT // 8 SUBBLOCK_SIZE = 32 * WORD SUBBLOCK_PTR = lltype.Ptr(lltype.GcForwardReference()) SUBBLOCK = lltype.GcStruct( 'SUBBLOCK', ('prev', SUBBLOCK_PTR), ('data', lltype.FixedSizeArray(lltype.Char, SUBBLOCK_SIZE))) SUBBLOCK_PTR.TO.become(SUBBLOCK) gcroot_markers = None def __init__(self, translated=None): if translated is None: translated = we_are_translated() if translated: self.init_block_builder() else: self._become_a_plain_block_builder() def init_block_builder(self): self._cursubblock = lltype.nullptr(self.SUBBLOCK) self._baserelpos = -self.SUBBLOCK_SIZE self._make_new_subblock() def _make_new_subblock(self): nextsubblock = lltype.malloc(self.SUBBLOCK) nextsubblock.prev = self._cursubblock self._cursubblock = nextsubblock self._cursubindex = 0 self._baserelpos += self.SUBBLOCK_SIZE _make_new_subblock._dont_inline_ = True def writechar(self, char): index = self._cursubindex if index == self.SUBBLOCK_SIZE: self._make_new_subblock() index = 0 self._cursubblock.data[index] = char self._cursubindex = index + 1 def overwrite(self, index, char): assert 0 <= index < self.get_relative_pos() block = self._cursubblock index -= self._baserelpos while index < 0: block = block.prev index += self.SUBBLOCK_SIZE block.data[index] = char def get_relative_pos(self): return self._baserelpos + self._cursubindex def copy_to_raw_memory(self, addr): # indirection for _become_a_plain_block_builder() and for subclasses self._copy_to_raw_memory(addr) def _copy_to_raw_memory(self, addr): block = self._cursubblock blocksize = self._cursubindex targetindex = self._baserelpos while targetindex >= 0: dst = rffi.cast(rffi.CCHARP, addr + targetindex) for j in range(blocksize): dst[j] = block.data[j] block = block.prev blocksize = self.SUBBLOCK_SIZE targetindex -= self.SUBBLOCK_SIZE assert not block def _dump(self, addr, logname, backend=None): debug_start(logname) if have_debug_prints(): # if backend is not None: debug_print('BACKEND', backend) # from pypy.jit.backend.hlinfo import highleveljitinfo if highleveljitinfo.sys_executable: debug_print('SYS_EXECUTABLE', highleveljitinfo.sys_executable) # HEX = '0123456789ABCDEF' dump = [] src = rffi.cast(rffi.CCHARP, addr) for p in range(self.get_relative_pos()): o = ord(src[p]) dump.append(HEX[o >> 4]) dump.append(HEX[o & 15]) debug_print( 'CODE_DUMP', '@%x' % addr, '+0 ', # backwards compatibility ''.join(dump)) # debug_stop(logname) def materialize(self, asmmemmgr, allblocks, gcrootmap=None): size = self.get_relative_pos() malloced = asmmemmgr.malloc(size, size) allblocks.append(malloced) rawstart = malloced[0] self.copy_to_raw_memory(rawstart) if self.gcroot_markers is not None: assert gcrootmap is not None for pos, mark in self.gcroot_markers: gcrootmap.put(rawstart + pos, mark) return rawstart def _become_a_plain_block_builder(self): # hack purely for speed of tests self._data = [] self.writechar = self._data.append self.overwrite = self._data.__setitem__ self.get_relative_pos = self._data.__len__ def plain_copy_to_raw_memory(addr): dst = rffi.cast(rffi.CCHARP, addr) for i, c in enumerate(self._data): dst[i] = c self._copy_to_raw_memory = plain_copy_to_raw_memory def insert_gcroot_marker(self, mark): if self.gcroot_markers is None: self.gcroot_markers = [] self.gcroot_markers.append((self.get_relative_pos(), mark))
def handle_call_with_close_stack(self, hop): fnptr = hop.spaceop.args[0].value # We cannot easily pass variable amount of arguments of the call # across the call to the pypy_asm_stackwalk helper. So we store # them away and restore them. We need to make a new graph # that starts with restoring the arguments. if self._asmgcc_save_restore_arguments is None: self._asmgcc_save_restore_arguments = {} sradict = self._asmgcc_save_restore_arguments sra = [] # list of pointers to raw-malloced containers for args seen = {} FUNC1 = lltype.typeOf(fnptr).TO for TYPE in FUNC1.ARGS: if isinstance(TYPE, lltype.Ptr): TYPE = llmemory.Address num = seen.get(TYPE, 0) seen[TYPE] = num + 1 key = (TYPE, num) if key not in sradict: CONTAINER = lltype.FixedSizeArray(TYPE, 1) p = lltype.malloc(CONTAINER, flavor='raw', zero=True) sradict[key] = Constant(p, lltype.Ptr(CONTAINER)) sra.append(sradict[key]) # # store the value of the arguments livevars = self.push_roots(hop) c_item0 = Constant('item0', lltype.Void) for v_arg, c_p in zip(hop.spaceop.args[1:], sra): if isinstance(v_arg.concretetype, lltype.Ptr): v_arg = hop.genop("cast_ptr_to_adr", [v_arg], resulttype=llmemory.Address) hop.genop("bare_setfield", [c_p, c_item0, v_arg]) # # make a copy of the graph that will reload the values graph2 = copygraph(fnptr._obj.graph) block2 = graph2.startblock block2.isstartblock = False block1 = Block([]) reloadedvars = [] for v, c_p in zip(block2.inputargs, sra): v = copyvar(None, v) if isinstance(v.concretetype, lltype.Ptr): w = Variable('tmp') w.concretetype = llmemory.Address else: w = v block1.operations.append(SpaceOperation('getfield', [c_p, c_item0], w)) if w is not v: block1.operations.append(SpaceOperation('cast_adr_to_ptr', [w], v)) reloadedvars.append(v) block1.closeblock(Link(reloadedvars, block2)) block1.isstartblock = True graph2.startblock = block1 FUNC2 = lltype.FuncType([], FUNC1.RESULT) fnptr2 = lltype.functionptr(FUNC2, fnptr._obj._name + '_reload', graph=graph2) c_fnptr2 = Constant(fnptr2, lltype.Ptr(FUNC2)) HELPERFUNC = lltype.FuncType([lltype.Ptr(FUNC2)], FUNC1.RESULT) # v_asm_stackwalk = hop.genop("cast_pointer", [c_asm_stackwalk], resulttype=lltype.Ptr(HELPERFUNC)) hop.genop("indirect_call", [v_asm_stackwalk, c_fnptr2, Constant(None, lltype.Void)], resultvar=hop.spaceop.result) self.pop_roots(hop, livevars)
def __init__(self, translator): from pypy.rpython.memory.gc.base import choose_gc_from_config super(FrameworkGCTransformer, self).__init__(translator, inline=True) if hasattr(self, 'GC_PARAMS'): # for tests: the GC choice can be specified as class attributes from pypy.rpython.memory.gc.marksweep import MarkSweepGC GCClass = getattr(self, 'GCClass', MarkSweepGC) GC_PARAMS = self.GC_PARAMS else: # for regular translation: pick the GC from the config GCClass, GC_PARAMS = choose_gc_from_config(translator.config) self.layoutbuilder = TransformerLayoutBuilder(self) self.get_type_id = self.layoutbuilder.get_type_id # set up dummy a table, to be overwritten with the real one in finish() type_info_table = lltype._ptr( lltype.Ptr(gctypelayout.GCData.TYPE_INFO_TABLE), "delayed!type_info_table", solid=True) gcdata = gctypelayout.GCData(type_info_table) # initialize the following two fields with a random non-NULL address, # to make the annotator happy. The fields are patched in finish() # to point to a real array. foo = lltype.malloc(lltype.FixedSizeArray(llmemory.Address, 1), immortal=True, zero=True) a_random_address = llmemory.cast_ptr_to_adr(foo) gcdata.static_root_start = a_random_address # patched in finish() gcdata.static_root_nongcend = a_random_address # patched in finish() gcdata.static_root_end = a_random_address # patched in finish() self.gcdata = gcdata self.malloc_fnptr_cache = {} gcdata.gc = GCClass(**GC_PARAMS) root_walker = self.build_root_walker() gcdata.set_query_functions(gcdata.gc) gcdata.gc.set_root_walker(root_walker) self.num_pushs = 0 self.write_barrier_calls = 0 def frameworkgc_setup(): # run-time initialization code root_walker.setup_root_walker() gcdata.gc.setup() bk = self.translator.annotator.bookkeeper # the point of this little dance is to not annotate # self.gcdata.static_root_xyz as constants. XXX is it still needed?? data_classdef = bk.getuniqueclassdef(gctypelayout.GCData) data_classdef.generalize_attr( 'static_root_start', annmodel.SomeAddress()) data_classdef.generalize_attr( 'static_root_nongcend', annmodel.SomeAddress()) data_classdef.generalize_attr( 'static_root_end', annmodel.SomeAddress()) annhelper = annlowlevel.MixLevelHelperAnnotator(self.translator.rtyper) def getfn(ll_function, args_s, s_result, inline=False, minimal_transform=True): graph = annhelper.getgraph(ll_function, args_s, s_result) if minimal_transform: self.need_minimal_transform(graph) if inline: self.graphs_to_inline[graph] = True return annhelper.graph2const(graph) self.frameworkgc_setup_ptr = getfn(frameworkgc_setup, [], annmodel.s_None) if root_walker.need_root_stack: self.incr_stack_ptr = getfn(root_walker.incr_stack, [annmodel.SomeInteger()], annmodel.SomeAddress(), inline = True) self.decr_stack_ptr = getfn(root_walker.decr_stack, [annmodel.SomeInteger()], annmodel.SomeAddress(), inline = True) else: self.incr_stack_ptr = None self.decr_stack_ptr = None self.weakref_deref_ptr = self.inittime_helper( ll_weakref_deref, [llmemory.WeakRefPtr], llmemory.Address) classdef = bk.getuniqueclassdef(GCClass) s_gc = annmodel.SomeInstance(classdef) s_gcref = annmodel.SomePtr(llmemory.GCREF) malloc_fixedsize_clear_meth = GCClass.malloc_fixedsize_clear.im_func self.malloc_fixedsize_clear_ptr = getfn( malloc_fixedsize_clear_meth, [s_gc, annmodel.SomeInteger(nonneg=True), annmodel.SomeInteger(nonneg=True), annmodel.SomeBool(), annmodel.SomeBool(), annmodel.SomeBool()], s_gcref, inline = False) if hasattr(GCClass, 'malloc_fixedsize'): malloc_fixedsize_meth = GCClass.malloc_fixedsize.im_func self.malloc_fixedsize_ptr = getfn( malloc_fixedsize_meth, [s_gc, annmodel.SomeInteger(nonneg=True), annmodel.SomeInteger(nonneg=True), annmodel.SomeBool(), annmodel.SomeBool(), annmodel.SomeBool()], s_gcref, inline = False) else: malloc_fixedsize_meth = None self.malloc_fixedsize_ptr = self.malloc_fixedsize_clear_ptr ## self.malloc_varsize_ptr = getfn( ## GCClass.malloc_varsize.im_func, ## [s_gc] + [annmodel.SomeInteger(nonneg=True) for i in range(5)] ## + [annmodel.SomeBool(), annmodel.SomeBool()], s_gcref) self.malloc_varsize_clear_ptr = getfn( GCClass.malloc_varsize_clear.im_func, [s_gc] + [annmodel.SomeInteger(nonneg=True) for i in range(5)] + [annmodel.SomeBool(), annmodel.SomeBool()], s_gcref) self.collect_ptr = getfn(GCClass.collect.im_func, [s_gc], annmodel.s_None) self.can_move_ptr = getfn(GCClass.can_move.im_func, [s_gc, annmodel.SomeAddress()], annmodel.SomeBool()) # in some GCs we can inline the common case of # malloc_fixedsize(typeid, size, True, False, False) if getattr(GCClass, 'inline_simple_malloc', False): # make a copy of this function so that it gets annotated # independently and the constants are folded inside if malloc_fixedsize_meth is None: malloc_fast_meth = malloc_fixedsize_clear_meth self.malloc_fast_is_clearing = True else: malloc_fast_meth = malloc_fixedsize_meth self.malloc_fast_is_clearing = False malloc_fast = func_with_new_name( malloc_fast_meth, "malloc_fast") s_False = annmodel.SomeBool(); s_False.const = False s_True = annmodel.SomeBool(); s_True .const = True self.malloc_fast_ptr = getfn( malloc_fast, [s_gc, annmodel.SomeInteger(nonneg=True), annmodel.SomeInteger(nonneg=True), s_True, s_False, s_False], s_gcref, inline = True) else: self.malloc_fast_ptr = None # in some GCs we can also inline the common case of # malloc_varsize(typeid, length, (3 constant sizes), True, False) if getattr(GCClass, 'inline_simple_malloc_varsize', False): # make a copy of this function so that it gets annotated # independently and the constants are folded inside malloc_varsize_clear_fast = func_with_new_name( GCClass.malloc_varsize_clear.im_func, "malloc_varsize_clear_fast") s_False = annmodel.SomeBool(); s_False.const = False s_True = annmodel.SomeBool(); s_True .const = True self.malloc_varsize_clear_fast_ptr = getfn( malloc_varsize_clear_fast, [s_gc, annmodel.SomeInteger(nonneg=True), annmodel.SomeInteger(nonneg=True), annmodel.SomeInteger(nonneg=True), annmodel.SomeInteger(nonneg=True), annmodel.SomeInteger(nonneg=True), s_True, s_False], s_gcref, inline = True) else: self.malloc_varsize_clear_fast_ptr = None if getattr(GCClass, 'malloc_varsize_nonmovable', False): malloc_nonmovable = func_with_new_name( GCClass.malloc_varsize_nonmovable.im_func, "malloc_varsize_nonmovable") self.malloc_varsize_nonmovable_ptr = getfn( malloc_nonmovable, [s_gc, annmodel.SomeInteger(nonneg=True), annmodel.SomeInteger(nonneg=True)], s_gcref) else: self.malloc_varsize_nonmovable_ptr = None if getattr(GCClass, 'malloc_varsize_resizable', False): malloc_resizable = func_with_new_name( GCClass.malloc_varsize_resizable.im_func, "malloc_varsize_resizable") self.malloc_varsize_resizable_ptr = getfn( malloc_resizable, [s_gc, annmodel.SomeInteger(nonneg=True), annmodel.SomeInteger(nonneg=True)], s_gcref) else: self.malloc_varsize_resizable_ptr = None if getattr(GCClass, 'realloc', False): self.realloc_ptr = getfn( GCClass.realloc.im_func, [s_gc, s_gcref] + [annmodel.SomeInteger(nonneg=True)] * 4 + [annmodel.SomeBool()], s_gcref) if GCClass.moving_gc: self.id_ptr = getfn(GCClass.id.im_func, [s_gc, s_gcref], annmodel.SomeInteger(), inline = False, minimal_transform = False) else: self.id_ptr = None self.set_max_heap_size_ptr = getfn(GCClass.set_max_heap_size.im_func, [s_gc, annmodel.SomeInteger(nonneg=True)], annmodel.s_None) if GCClass.needs_write_barrier: self.write_barrier_ptr = getfn(GCClass.write_barrier.im_func, [s_gc, annmodel.SomeAddress(), annmodel.SomeAddress()], annmodel.s_None, inline=True) else: self.write_barrier_ptr = None self.statistics_ptr = getfn(GCClass.statistics.im_func, [s_gc, annmodel.SomeInteger()], annmodel.SomeInteger()) # experimental gc_x_* operations s_x_pool = annmodel.SomePtr(marksweep.X_POOL_PTR) s_x_clone = annmodel.SomePtr(marksweep.X_CLONE_PTR) # the x_*() methods use some regular mallocs that must be # transformed in the normal way self.x_swap_pool_ptr = getfn(GCClass.x_swap_pool.im_func, [s_gc, s_x_pool], s_x_pool, minimal_transform = False) self.x_clone_ptr = getfn(GCClass.x_clone.im_func, [s_gc, s_x_clone], annmodel.s_None, minimal_transform = False) # thread support if translator.config.translation.thread: if not hasattr(root_walker, "need_thread_support"): raise Exception("%s does not support threads" % ( root_walker.__class__.__name__,)) root_walker.need_thread_support() self.thread_prepare_ptr = getfn(root_walker.thread_prepare, [], annmodel.s_None) self.thread_run_ptr = getfn(root_walker.thread_run, [], annmodel.s_None, inline=True) self.thread_die_ptr = getfn(root_walker.thread_die, [], annmodel.s_None) annhelper.finish() # at this point, annotate all mix-level helpers annhelper.backend_optimize() self.collect_analyzer = CollectAnalyzer(self.translator) self.collect_analyzer.analyze_all() s_gc = self.translator.annotator.bookkeeper.valueoftype(GCClass) r_gc = self.translator.rtyper.getrepr(s_gc) self.c_const_gc = rmodel.inputconst(r_gc, self.gcdata.gc) self.malloc_zero_filled = GCClass.malloc_zero_filled HDR = self._gc_HDR = self.gcdata.gc.gcheaderbuilder.HDR self._gc_fields = fields = [] for fldname in HDR._names: FLDTYPE = getattr(HDR, fldname) fields.append(('_' + fldname, FLDTYPE))