def get_L2cache_linux2(filename="/proc/cpuinfo"): debug_start("gc-hardware") L2cache = sys.maxint try: fd = os.open(filename, os.O_RDONLY, 0644) try: data = [] while True: buf = os.read(fd, 4096) if not buf: break data.append(buf) finally: os.close(fd) except OSError: pass else: data = ''.join(data) linepos = 0 while True: start = _findend(data, '\ncache size', linepos) if start < 0: break # done linepos = _findend(data, '\n', start) if linepos < 0: break # no end-of-line?? # *** data[start:linepos] == " : 2048 KB\n" start = _skipspace(data, start) if data[start] != ':': continue # *** data[start:linepos] == ": 2048 KB\n" start = _skipspace(data, start + 1) # *** data[start:linepos] == "2048 KB\n" end = start while '0' <= data[end] <= '9': end += 1 # *** data[start:end] == "2048" if start == end: continue number = int(data[start:end]) # *** data[end:linepos] == " KB\n" end = _skipspace(data, end) if data[end] not in ('K', 'k'): # assume kilobytes for now continue number = number * 1024 # for now we look for the smallest of the L2 caches of the CPUs if number < L2cache: L2cache = number debug_print("L2cache =", L2cache) debug_stop("gc-hardware") if L2cache < sys.maxint: return L2cache else: # Print a top-level warning even in non-debug builds llop.debug_print( lltype.Void, "Warning: cannot find your CPU L2 cache size in /proc/cpuinfo") return -1
def collect_nursery(self): if self.nursery_size > self.top_of_space - self.free: # the semispace is running out, do a full collect self.obtain_free_space(self.nursery_size) ll_assert(self.nursery_size <= self.top_of_space - self.free, "obtain_free_space failed to do its job") if self.nursery: if DEBUG_PRINT: llop.debug_print(lltype.Void, "minor collect") # a nursery-only collection scan = beginning = self.free self.collect_oldrefs_to_nursery() self.collect_roots_in_nursery() scan = self.scan_objects_just_copied_out_of_nursery(scan) # at this point, all static and old objects have got their # GCFLAG_NO_YOUNG_PTRS set again by trace_and_drag_out_of_nursery if self.young_objects_with_weakrefs.non_empty(): self.invalidate_young_weakrefs() self.notify_objects_just_moved() # mark the nursery as free and fill it with zeroes again llarena.arena_reset(self.nursery, self.nursery_size, True) if DEBUG_PRINT: llop.debug_print(lltype.Void, "percent survived:", float(scan - beginning) / self.nursery_size) else: # no nursery - this occurs after a full collect, triggered either # just above or by some previous non-nursery-based allocation. # Grab a piece of the current space for the nursery. self.nursery = self.free self.nursery_top = self.nursery + self.nursery_size self.free = self.nursery_top self.nursery_free = self.nursery return self.nursery_free
def debug_collect_start(self): if self.config.gcconfig.debugprint: llop.debug_print(lltype.Void) llop.debug_print(lltype.Void, ".----------- Full collection ------------------") start_time = time.time() return start_time
def estimate_best_nursery_size(debugprint=False): """Try to estimate the best nursery size at run-time, depending on the machine we are running on. """ L2cache = 0 l2cache_p = lltype.malloc(rffi.LONGLONGP.TO, 1, flavor='raw') try: len_p = lltype.malloc(rffi.SIZE_TP.TO, 1, flavor='raw') try: size = rffi.sizeof(rffi.LONGLONG) l2cache_p[0] = rffi.cast(rffi.LONGLONG, 0) len_p[0] = rffi.cast(rffi.SIZE_T, size) # XXX a hack for llhelper not being robust-enough result = sysctlbyname("hw.l2cachesize", rffi.cast(rffi.VOIDP, l2cache_p), len_p, lltype.nullptr(rffi.VOIDP.TO), rffi.cast(rffi.SIZE_T, 0)) if (rffi.cast(lltype.Signed, result) == 0 and rffi.cast(lltype.Signed, len_p[0]) == size): L2cache = rffi.cast(lltype.Signed, l2cache_p[0]) if rffi.cast(rffi.LONGLONG, L2cache) != l2cache_p[0]: L2cache = 0 # overflow! finally: lltype.free(len_p, flavor='raw') finally: lltype.free(l2cache_p, flavor='raw') if L2cache > 0: return best_nursery_size_for_L2cache(L2cache, debugprint) else: # Print a warning even in non-debug builds llop.debug_print( lltype.Void, "Warning: cannot find your CPU L2 cache size with sysctl()") return -1
def do_call(result, path, index, remaining_depth): # clone the while path clonedata.gcobjectptr = lltype.cast_opaque_ptr(llmemory.GCREF, path) clonedata.pool = lltype.nullptr(X_POOL) llop.gc_x_clone(lltype.Void, clonedata) # install the new pool as the current one parentpool = llop.gc_x_swap_pool(X_POOL_PTR, clonedata.pool) path = lltype.cast_opaque_ptr(lltype.Ptr(NODE), clonedata.gcobjectptr) # The above should have the same effect as: # path = clone(path) # bump all the path node counters by one p = path while p: p.counter += 1 p = p.next if remaining_depth == 0: llop.debug_print(lltype.Void, "setting", index, "with", path) result[index] = path # leaf else: node = lltype.malloc(NODE) node.index = index * 2 node.counter = 0 node.next = path do_call(result, node, index * 2, remaining_depth - 1) node.index += 1 # mutation! do_call(result, node, index * 2 + 1, remaining_depth - 1) # restore the parent pool llop.gc_x_swap_pool(X_POOL_PTR, parentpool)
def debug_collect_start(self): if self.config.gcconfig.debugprint: llop.debug_print(lltype.Void) llop.debug_print( lltype.Void, ".----------- Full collection ------------------") start_time = time.time() return start_time
def estimate_best_nursery_size(): """Try to estimate the best nursery size at run-time, depending on the machine we are running on. """ L2cache = 0 l2cache_p = lltype.malloc(rffi.LONGLONGP.TO, 1, flavor='raw') try: len_p = lltype.malloc(rffi.SIZE_TP.TO, 1, flavor='raw') try: size = rffi.sizeof(rffi.LONGLONG) l2cache_p[0] = rffi.cast(rffi.LONGLONG, 0) len_p[0] = rffi.cast(rffi.SIZE_T, size) result = sysctlbyname("hw.l2cachesize", rffi.cast(rffi.VOIDP, l2cache_p), len_p, lltype.nullptr(rffi.VOIDP.TO), rffi.cast(rffi.SIZE_T, 0)) if (rffi.cast(lltype.Signed, result) == 0 and rffi.cast(lltype.Signed, len_p[0]) == size): L2cache = rffi.cast(lltype.Signed, l2cache_p[0]) if rffi.cast(rffi.LONGLONG, L2cache) != l2cache_p[0]: L2cache = 0 # overflow! finally: lltype.free(len_p, flavor='raw') finally: lltype.free(l2cache_p, flavor='raw') if L2cache > 0: return best_nursery_size_for_L2cache(L2cache) else: # Print a warning even in non-debug builds llop.debug_print(lltype.Void, "Warning: cannot find your CPU L2 cache size with sysctl()") return -1
def get_L2cache_linux2(filename="/proc/cpuinfo"): debug_start("gc-hardware") L2cache = sys.maxint try: fd = os.open(filename, os.O_RDONLY, 0644) try: data = [] while True: buf = os.read(fd, 4096) if not buf: break data.append(buf) finally: os.close(fd) except OSError: pass else: data = ''.join(data) linepos = 0 while True: start = _findend(data, '\ncache size', linepos) if start < 0: break # done linepos = _findend(data, '\n', start) if linepos < 0: break # no end-of-line?? # *** data[start:linepos] == " : 2048 KB\n" start = _skipspace(data, start) if data[start] != ':': continue # *** data[start:linepos] == ": 2048 KB\n" start = _skipspace(data, start + 1) # *** data[start:linepos] == "2048 KB\n" end = start while '0' <= data[end] <= '9': end += 1 # *** data[start:end] == "2048" if start == end: continue number = int(data[start:end]) # *** data[end:linepos] == " KB\n" end = _skipspace(data, end) if data[end] not in ('K', 'k'): # assume kilobytes for now continue number = number * 1024 # for now we look for the smallest of the L2 caches of the CPUs if number < L2cache: L2cache = number debug_print("L2cache =", L2cache) debug_stop("gc-hardware") if L2cache < sys.maxint: return L2cache else: # Print a top-level warning even in non-debug builds llop.debug_print(lltype.Void, "Warning: cannot find your CPU L2 cache size in /proc/cpuinfo") return -1
def estimate_best_nursery_size(debugprint=False): """Try to estimate the best nursery size at run-time, depending on the machine we are running on. """ L2cache = sys.maxint try: fd = os.open('/proc/cpuinfo', os.O_RDONLY, 0644) try: data = [] while True: buf = os.read(fd, 4096) if not buf: break data.append(buf) finally: os.close(fd) except OSError: pass else: data = ''.join(data) linepos = 0 while True: start = findend(data, '\ncache size', linepos) if start < 0: break # done linepos = findend(data, '\n', start) if linepos < 0: break # no end-of-line?? # *** data[start:linepos] == " : 2048 KB\n" start = skipspace(data, start) if data[start] != ':': continue # *** data[start:linepos] == ": 2048 KB\n" start = skipspace(data, start + 1) # *** data[start:linepos] == "2048 KB\n" end = start while '0' <= data[end] <= '9': end += 1 # *** data[start:end] == "2048" if start == end: continue number = int(data[start:end]) # *** data[end:linepos] == " KB\n" end = skipspace(data, end) if data[end] not in ('K', 'k'): # assume kilobytes for now continue number = number * 1024 # for now we look for the smallest of the L2 caches of the CPUs if number < L2cache: L2cache = number if L2cache < sys.maxint: return best_nursery_size_for_L2cache(L2cache, debugprint) else: # Print a warning even in non-debug builds llop.debug_print( lltype.Void, "Warning: cannot find your CPU L2 cache size in /proc/cpuinfo") return -1
def estimate_best_nursery_size(): """Try to estimate the best nursery size at run-time, depending on the machine we are running on. """ L2cache = sys.maxint try: fd = os.open('/proc/cpuinfo', os.O_RDONLY, 0644) try: data = [] while True: buf = os.read(fd, 4096) if not buf: break data.append(buf) finally: os.close(fd) except OSError: pass else: data = ''.join(data) linepos = 0 while True: start = findend(data, '\ncache size', linepos) if start < 0: break # done linepos = findend(data, '\n', start) if linepos < 0: break # no end-of-line?? # *** data[start:linepos] == " : 2048 KB\n" start = skipspace(data, start) if data[start] != ':': continue # *** data[start:linepos] == ": 2048 KB\n" start = skipspace(data, start + 1) # *** data[start:linepos] == "2048 KB\n" end = start while '0' <= data[end] <= '9': end += 1 # *** data[start:end] == "2048" if start == end: continue number = int(data[start:end]) # *** data[end:linepos] == " KB\n" end = skipspace(data, end) if data[end] not in ('K', 'k'): # assume kilobytes for now continue number = number * 1024 # for now we look for the smallest of the L2 caches of the CPUs if number < L2cache: L2cache = number if L2cache < sys.maxint: return best_nursery_size_for_L2cache(L2cache) else: # Print a warning even in non-debug builds llop.debug_print(lltype.Void, "Warning: cannot find your CPU L2 cache size in /proc/cpuinfo") return -1
def _check_rawsize_alloced(self, size_estimate, can_collect=True): self.large_objects_collect_trigger -= size_estimate if can_collect and self.large_objects_collect_trigger < 0: if self.config.gcconfig.debugprint: llop.debug_print(lltype.Void, "allocated", self._initial_trigger - self.large_objects_collect_trigger, "bytes, triggering full collection") self.semispace_collect()
def _check_rawsize_alloced(self, size_estimate, can_collect=True): self.large_objects_collect_trigger -= size_estimate if can_collect and self.large_objects_collect_trigger < 0: if self.config.gcconfig.debugprint: llop.debug_print( lltype.Void, "allocated", self._initial_trigger - self.large_objects_collect_trigger, "bytes, triggering full collection") self.semispace_collect()
def semispace_collect(self, size_changing=False): self.reset_young_gcflags() # we are doing a full collection anyway self.weakrefs_grow_older() self.reset_nursery() if DEBUG_PRINT: llop.debug_print(lltype.Void, "major collect, size changing", size_changing) SemiSpaceGC.semispace_collect(self, size_changing) if DEBUG_PRINT and not size_changing: llop.debug_print(lltype.Void, "percent survived", float(self.free - self.tospace) / self.space_size)
def collect_oldrefs_to_nursery(self): # Follow the old_objects_pointing_to_young list and move the # young objects they point to out of the nursery. count = 0 oldlist = self.old_objects_pointing_to_young while oldlist.non_empty(): count += 1 obj = oldlist.pop() self.trace_and_drag_out_of_nursery(obj) if DEBUG_PRINT: llop.debug_print(lltype.Void, "collect_oldrefs_to_nursery", count)
def collect_oldrefs_to_nursery(self): # Follow the old_objects_pointing_to_young list and move the # young objects they point to out of the nursery. count = 0 oldlist = self.old_objects_pointing_to_young while oldlist.non_empty(): count += 1 obj = oldlist.pop() hdr = self.header(obj) hdr.tid |= GCFLAG_NO_YOUNG_PTRS self.trace_and_drag_out_of_nursery(obj) if self.config.gcconfig.debugprint: llop.debug_print(lltype.Void, "collect_oldrefs_to_nursery", count)
def set_nursery_size(self, newsize): if newsize < self.min_nursery_size: newsize = self.min_nursery_size if newsize > self.space_size // 2: newsize = self.space_size // 2 # Compute the new bounds for how large young objects can be # (larger objects are allocated directly old). XXX adjust self.nursery_size = newsize self.largest_young_fixedsize = self.get_young_fixedsize(newsize) self.largest_young_var_basesize = self.get_young_var_basesize(newsize) scale = 0 while (self.min_nursery_size << (scale+1)) <= newsize: scale += 1 self.nursery_scale = scale if DEBUG_PRINT: llop.debug_print(lltype.Void, "SSS nursery_size =", newsize) llop.debug_print(lltype.Void, "SSS largest_young_fixedsize =", self.largest_young_fixedsize) llop.debug_print(lltype.Void, "SSS largest_young_var_basesize =", self.largest_young_var_basesize) llop.debug_print(lltype.Void, "SSS nursery_scale =", scale) # we get the following invariant: assert self.nursery_size >= (self.min_nursery_size << scale) # Force a full collect to remove the current nursery whose size # no longer matches the bounds that we just computed. This must # be done after changing the bounds, because it might re-create # a new nursery (e.g. if it invokes finalizers). self.semispace_collect()
def set_nursery_size(self, newsize): if newsize < self.min_nursery_size: newsize = self.min_nursery_size if newsize > self.space_size // 2: newsize = self.space_size // 2 # Compute the new bounds for how large young objects can be # (larger objects are allocated directly old). XXX adjust self.nursery_size = newsize self.largest_young_fixedsize = self.get_young_fixedsize(newsize) self.largest_young_var_basesize = self.get_young_var_basesize(newsize) scale = 0 while (self.min_nursery_size << (scale + 1)) <= newsize: scale += 1 self.nursery_scale = scale if self.config.gcconfig.debugprint: llop.debug_print(lltype.Void, "SSS nursery_size =", newsize) llop.debug_print(lltype.Void, "SSS largest_young_fixedsize =", self.largest_young_fixedsize) llop.debug_print(lltype.Void, "SSS largest_young_var_basesize =", self.largest_young_var_basesize) llop.debug_print(lltype.Void, "SSS nursery_scale =", scale) # we get the following invariant: assert self.nursery_size >= (self.min_nursery_size << scale) # Force a full collect to remove the current nursery whose size # no longer matches the bounds that we just computed. This must # be done after changing the bounds, because it might re-create # a new nursery (e.g. if it invokes finalizers). self.semispace_collect()
def f(x, y): persistent_a1 = A() persistent_a2 = A() i = 0 while i < x: i += 1 a = A() persistent_a3 = A() persistent_a4 = A() llop.gc__collect(lltype.Void) llop.gc__collect(lltype.Void) b.bla = persistent_a1.id + persistent_a2.id + persistent_a3.id + persistent_a4.id # NB print would create a static root! llop.debug_print(lltype.Void, b.num_deleted_c) return b.num_deleted
def get_L2cache_darwin(): """Try to estimate the best nursery size at run-time, depending on the machine we are running on. """ debug_start("gc-hardware") L2cache = get_darwin_sysctl_signed("hw.l2cachesize") L3cache = get_darwin_sysctl_signed("hw.l3cachesize") debug_print("L2cache =", L2cache) debug_print("L3cache =", L3cache) debug_stop("gc-hardware") mangled = L2cache + L3cache if mangled > 0: return mangled else: # Print a top-level warning even in non-debug builds llop.debug_print(lltype.Void, "Warning: cannot find your CPU L2 cache size with sysctl()") return -1
def finished_full_collect(self): ll_assert(not self.rawmalloced_objects_to_trace.non_empty(), "rawmalloced_objects_to_trace should be empty at end") if self.config.gcconfig.debugprint: llop.debug_print(lltype.Void, "| [hybrid] made nonmoving: ", self._nonmoving_copy_size, "bytes in", self._nonmoving_copy_count, "objs") # sweep the nonmarked rawmalloced objects if self.is_collecting_gen3(): self.sweep_rawmalloced_objects(generation=3) self.sweep_rawmalloced_objects(generation=2) self.sweep_rawmalloced_objects(generation=-2) # As we just collected, it's fine to raw_malloc'ate up to space_size # bytes again before we should force another collect. self.large_objects_collect_trigger = self.space_size if self.is_collecting_gen3(): self.count_semispaceonly_collects = 0 if self.config.gcconfig.debugprint: self._initial_trigger = self.large_objects_collect_trigger
def debug_collect_finish(self, start_time): if self.config.gcconfig.debugprint: end_time = time.time() elapsed_time = end_time - start_time self.total_collection_time += elapsed_time self.total_collection_count += 1 total_program_time = end_time - self.program_start_time ct = self.total_collection_time cc = self.total_collection_count llop.debug_print(lltype.Void, "| number of collections so far ", cc) llop.debug_print(lltype.Void, "| total collections per second: ", cc / total_program_time) llop.debug_print(lltype.Void, "| total time in markcompact-collect: ", ct, "seconds") llop.debug_print(lltype.Void, "| percentage collection<->total time:", ct * 100.0 / total_program_time, "%") llop.debug_print( lltype.Void, "`----------------------------------------------")
def debug_collect_finish(self, start_time): if self.config.gcconfig.debugprint: end_time = time.time() elapsed_time = end_time - start_time self.total_collection_time += elapsed_time self.total_collection_count += 1 total_program_time = end_time - self.program_start_time ct = self.total_collection_time cc = self.total_collection_count llop.debug_print(lltype.Void, "| number of collections so far ", cc) llop.debug_print(lltype.Void, "| total collections per second: ", cc / total_program_time) llop.debug_print(lltype.Void, "| total time in markcompact-collect: ", ct, "seconds") llop.debug_print(lltype.Void, "| percentage collection<->total time:", ct * 100.0 / total_program_time, "%") llop.debug_print(lltype.Void, "`----------------------------------------------")
def collect_nursery(self): if self.nursery_size > self.top_of_space - self.free: # the semispace is running out, do a full collect self.obtain_free_space(self.nursery_size) ll_assert(self.nursery_size <= self.top_of_space - self.free, "obtain_free_space failed to do its job") if self.nursery: if self.config.gcconfig.debugprint: llop.debug_print(lltype.Void, "--- minor collect ---") llop.debug_print(lltype.Void, "nursery:", self.nursery, "to", self.nursery_top) # a nursery-only collection scan = beginning = self.free self.collect_oldrefs_to_nursery() self.collect_roots_in_nursery() scan = self.scan_objects_just_copied_out_of_nursery(scan) # at this point, all static and old objects have got their # GCFLAG_NO_YOUNG_PTRS set again by trace_and_drag_out_of_nursery if self.young_objects_with_weakrefs.non_empty(): self.invalidate_young_weakrefs() if self.young_objects_with_id.length() > 0: self.update_young_objects_with_id() # mark the nursery as free and fill it with zeroes again llarena.arena_reset(self.nursery, self.nursery_size, 2) if self.config.gcconfig.debugprint: llop.debug_print(lltype.Void, "survived (fraction of the size):", float(scan - beginning) / self.nursery_size) #self.debug_check_consistency() # -- quite expensive else: # no nursery - this occurs after a full collect, triggered either # just above or by some previous non-nursery-based allocation. # Grab a piece of the current space for the nursery. self.nursery = self.free self.nursery_top = self.nursery + self.nursery_size self.free = self.nursery_top self.nursery_free = self.nursery return self.nursery_free
def sweep_rawmalloced_objects(self, generation): # free all the rawmalloced objects of the specified generation # that have not been marked if generation == 2: objects = self.gen2_rawmalloced_objects # generation 2 sweep: if A points to an object object B that # moves from gen2 to gen3, it's possible that A no longer points # to any gen2 object. In this case, A remains a bit too long in # last_generation_root_objects, but this will be fixed by the # next collect_last_generation_roots(). elif generation == 3: objects = self.gen3_rawmalloced_objects # generation 3 sweep: remove from last_generation_root_objects # all the objects that we are about to free gen3roots = self.last_generation_root_objects newgen3roots = self.AddressStack() while gen3roots.non_empty(): obj = gen3roots.pop() if not (self.header(obj).tid & GCFLAG_UNVISITED): newgen3roots.append(obj) gen3roots.delete() self.last_generation_root_objects = newgen3roots else: # mostly a hack: the generation number -2 is the part of the # generation 2 that lives in gen2_resizable_objects ll_assert(generation == -2, "bogus 'generation'") objects = self.gen2_resizable_objects surviving_objects = self.AddressStack() # Help the flow space alive_count = alive_size = dead_count = dead_size = 0 while objects.non_empty(): obj = objects.pop() tid = self.header(obj).tid if tid & GCFLAG_UNVISITED: if self.config.gcconfig.debugprint: dead_count+=1 dead_size+=raw_malloc_usage(self.get_size(obj)) addr = obj - self.gcheaderbuilder.size_gc_header llmemory.raw_free(addr) else: if self.config.gcconfig.debugprint: alive_count+=1 alive_size+=raw_malloc_usage(self.get_size(obj)) if generation == 3: surviving_objects.append(obj) elif generation == 2: ll_assert((tid & GCFLAG_AGE_MASK) < GCFLAG_AGE_MAX, "wrong age for generation 2 object") tid += GCFLAG_AGE_ONE if (tid & GCFLAG_AGE_MASK) == GCFLAG_AGE_MAX: # the object becomes part of generation 3 self.gen3_rawmalloced_objects.append(obj) # GCFLAG_NO_HEAP_PTRS not set yet, conservatively self.last_generation_root_objects.append(obj) else: # the object stays in generation 2 tid |= GCFLAG_UNVISITED surviving_objects.append(obj) self.header(obj).tid = tid elif generation == -2: # the object stays in generation -2 tid |= GCFLAG_UNVISITED surviving_objects.append(obj) self.header(obj).tid = tid objects.delete() if generation == 2: self.gen2_rawmalloced_objects = surviving_objects elif generation == 3: self.gen3_rawmalloced_objects = surviving_objects elif generation == -2: self.gen2_resizable_objects = surviving_objects if self.config.gcconfig.debugprint: llop.debug_print(lltype.Void, "| [hyb] gen", generation, "nonmoving now alive: ", alive_size, "bytes in", alive_count, "objs") llop.debug_print(lltype.Void, "| [hyb] gen", generation, "nonmoving freed: ", dead_size, "bytes in", dead_count, "objs")
def l(y, x, t): llop.debug_print(lltype.Void, y, x, t)
def externfn(node): llop.debug_print(lltype.Void, compute_unique_id(node), node.value, node.extra) return node.value * 2
def _teardown(self): llop.debug_print(lltype.Void, "Teardown") llarena.arena_free(self.fromspace) llarena.arena_free(self.tospace)
def semispace_collect(self, size_changing=False): if self.config.gcconfig.debugprint: llop.debug_print(lltype.Void) llop.debug_print( lltype.Void, ".----------- Full collection ------------------") start_usage = self.free - self.tospace llop.debug_print(lltype.Void, "| used before collection: ", start_usage, "bytes") start_time = time.time() else: start_time = 0 # Help the flow space start_usage = 0 # Help the flow space #llop.debug_print(lltype.Void, 'semispace_collect', int(size_changing)) # Switch the spaces. We copy everything over to the empty space # (self.fromspace at the beginning of the collection), and clear the old # one (self.tospace at the beginning). Their purposes will be reversed # for the next collection. tospace = self.fromspace fromspace = self.tospace self.fromspace = fromspace self.tospace = tospace self.top_of_space = tospace + self.space_size scan = self.free = tospace self.starting_full_collect() self.collect_roots() if self.run_finalizers.non_empty(): self.update_run_finalizers() scan = self.scan_copied(scan) if self.objects_with_finalizers.non_empty(): scan = self.deal_with_objects_with_finalizers(scan) if self.objects_with_weakrefs.non_empty(): self.invalidate_weakrefs() self.update_objects_with_id() self.finished_full_collect() self.debug_check_consistency() if not size_changing: llarena.arena_reset(fromspace, self.space_size, True) self.record_red_zone() self.execute_finalizers() #llop.debug_print(lltype.Void, 'collected', self.space_size, size_changing, self.top_of_space - self.free) if self.config.gcconfig.debugprint: end_time = time.time() elapsed_time = end_time - start_time self.total_collection_time += elapsed_time self.total_collection_count += 1 total_program_time = end_time - self.program_start_time end_usage = self.free - self.tospace llop.debug_print(lltype.Void, "| used after collection: ", end_usage, "bytes") llop.debug_print(lltype.Void, "| freed: ", start_usage - end_usage, "bytes") llop.debug_print(lltype.Void, "| size of each semispace: ", self.space_size, "bytes") llop.debug_print(lltype.Void, "| fraction of semispace now used: ", end_usage * 100.0 / self.space_size, "%") ct = self.total_collection_time cc = self.total_collection_count llop.debug_print(lltype.Void, "| number of semispace_collects: ", cc) llop.debug_print(lltype.Void, "| i.e.: ", cc / total_program_time, "per second") llop.debug_print(lltype.Void, "| total time in semispace_collect: ", ct, "seconds") llop.debug_print(lltype.Void, "| i.e.: ", ct * 100.0 / total_program_time, "%") llop.debug_print( lltype.Void, "`----------------------------------------------")
def write_free_statistics(self, typeid, result): llop.debug_print(lltype.Void, "free", typeid, " ", result)
def write_malloc_statistics(self, typeid, size, result, varsize): if varsize: what = "malloc_varsize" else: what = "malloc" llop.debug_print(lltype.Void, what, typeid, " ", size, " ", result)
def best_nursery_size_for_L2cache(L2cache, debugprint=False): if debugprint: llop.debug_print(lltype.Void, "CCC L2cache =", L2cache) # Heuristically, the best nursery size to choose is about half # of the L2 cache. XXX benchmark some more. return L2cache // 2
def fn(): llop.debug_print(lltype.Void, "hello world")
def semispace_collect(self, size_changing=False): if DEBUG_PRINT: import time llop.debug_print(lltype.Void) llop.debug_print( lltype.Void, ".----------- Full collection ------------------") start_usage = self.free - self.tospace llop.debug_print(lltype.Void, "| used before collection: ", start_usage, "bytes") start_time = time.time() #llop.debug_print(lltype.Void, 'semispace_collect', int(size_changing)) tospace = self.fromspace fromspace = self.tospace self.fromspace = fromspace self.tospace = tospace self.top_of_space = tospace + self.space_size scan = self.free = tospace self.starting_full_collect() self.collect_roots() if self.run_finalizers.non_empty(): self.update_run_finalizers() scan = self.scan_copied(scan) if self.objects_with_finalizers.non_empty(): scan = self.deal_with_objects_with_finalizers(scan) if self.objects_with_weakrefs.non_empty(): self.invalidate_weakrefs() self.update_objects_with_id() self.finished_full_collect() self.debug_check_consistency() if not size_changing: llarena.arena_reset(fromspace, self.space_size, True) self.record_red_zone() self.execute_finalizers() #llop.debug_print(lltype.Void, 'collected', self.space_size, size_changing, self.top_of_space - self.free) if DEBUG_PRINT: end_time = time.time() elapsed_time = end_time - start_time self.total_collection_time += elapsed_time self.total_collection_count += 1 total_program_time = end_time - self.program_start_time end_usage = self.free - self.tospace llop.debug_print(lltype.Void, "| used after collection: ", end_usage, "bytes") llop.debug_print(lltype.Void, "| freed: ", start_usage - end_usage, "bytes") llop.debug_print(lltype.Void, "| size of each semispace: ", self.space_size, "bytes") llop.debug_print(lltype.Void, "| fraction of semispace now used: ", end_usage * 100.0 / self.space_size, "%") ct = self.total_collection_time cc = self.total_collection_count llop.debug_print(lltype.Void, "| number of semispace_collects: ", cc) llop.debug_print(lltype.Void, "| i.e.: ", cc / total_program_time, "per second") llop.debug_print(lltype.Void, "| total time in semispace_collect: ", ct, "seconds") llop.debug_print(lltype.Void, "| i.e.: ", ct * 100.0 / total_program_time, "%") llop.debug_print( lltype.Void, "`----------------------------------------------")
def externfn(node): llop.debug_print(lltype.Void, node) return node.value * 2
def best_nursery_size_for_L2cache(L2cache): if DEBUG_PRINT: llop.debug_print(lltype.Void, "CCC L2cache =", L2cache) # Heuristically, the best nursery size to choose is about half # of the L2 cache. XXX benchmark some more. return L2cache // 2
def collect(self): # 1. mark from the roots, and also the objects that objects-with-del # point to (using the list of malloced_objects_with_finalizer) # 2. walk the list of objects-without-del and free the ones not marked # 3. walk the list of objects-with-del and for the ones not marked: # call __del__, move the object to the list of object-without-del import time from pypy.rpython.lltypesystem.lloperation import llop if DEBUG_PRINT: llop.debug_print(lltype.Void, 'collecting...') start_time = time.time() self.collect_in_progress = True size_gc_header = self.gcheaderbuilder.size_gc_header ## llop.debug_view(lltype.Void, self.malloced_objects, self.poolnodes, ## size_gc_header) # push the roots on the mark stack objects = self.AddressStack() # mark stack self._mark_stack = objects self.root_walker.walk_roots( MarkSweepGC._mark_root, # stack roots MarkSweepGC._mark_root, # static in prebuilt non-gc structures MarkSweepGC._mark_root) # static in prebuilt gc objects # from this point onwards, no more mallocs should be possible old_malloced = self.bytes_malloced self.bytes_malloced = 0 curr_heap_size = 0 freed_size = 0 # mark objects reachable by objects with a finalizer, but not those # themselves. add their size to curr_heap_size, since they always # survive the collection hdr = self.malloced_objects_with_finalizer while hdr: next = hdr.next typeid = hdr.typeid >> 1 gc_info = llmemory.cast_ptr_to_adr(hdr) obj = gc_info + size_gc_header if not hdr.typeid & 1: self.add_reachable_to_stack(obj, objects) addr = llmemory.cast_ptr_to_adr(hdr) size = self.fixed_size(typeid) if self.is_varsize(typeid): length = (obj + self.varsize_offset_to_length(typeid)).signed[0] size += self.varsize_item_sizes(typeid) * length estimate = raw_malloc_usage(size_gc_header + size) curr_heap_size += estimate hdr = next # mark thinks on the mark stack and put their descendants onto the # stack until the stack is empty while objects.non_empty(): #mark curr = objects.pop() gc_info = curr - size_gc_header hdr = llmemory.cast_adr_to_ptr(gc_info, self.HDRPTR) if hdr.typeid & 1: continue self.add_reachable_to_stack(curr, objects) hdr.typeid = hdr.typeid | 1 objects.delete() # also mark self.curpool if self.curpool: gc_info = llmemory.cast_ptr_to_adr(self.curpool) - size_gc_header hdr = llmemory.cast_adr_to_ptr(gc_info, self.HDRPTR) hdr.typeid = hdr.typeid | 1 # go through the list of objects containing weak pointers # and kill the links if they go to dead objects # if the object itself is not marked, free it hdr = self.objects_with_weak_pointers surviving = lltype.nullptr(self.HDR) while hdr: typeid = hdr.typeid >> 1 next = hdr.next addr = llmemory.cast_ptr_to_adr(hdr) size = self.fixed_size(typeid) estimate = raw_malloc_usage(size_gc_header + size) if hdr.typeid & 1: typeid = hdr.typeid >> 1 offset = self.weakpointer_offset(typeid) hdr.typeid = hdr.typeid & (~1) gc_info = llmemory.cast_ptr_to_adr(hdr) weakref_obj = gc_info + size_gc_header pointing_to = (weakref_obj + offset).address[0] if pointing_to: gc_info_pointing_to = pointing_to - size_gc_header hdr_pointing_to = llmemory.cast_adr_to_ptr( gc_info_pointing_to, self.HDRPTR) # pointed to object will die # XXX what to do if the object has a finalizer which resurrects # the object? if not hdr_pointing_to.typeid & 1: (weakref_obj + offset).address[0] = NULL hdr.next = surviving surviving = hdr curr_heap_size += estimate else: gc_info = llmemory.cast_ptr_to_adr(hdr) weakref_obj = gc_info + size_gc_header self.write_free_statistics(typeid, weakref_obj) freed_size += estimate raw_free(addr) hdr = next self.objects_with_weak_pointers = surviving # sweep: delete objects without del if they are not marked # unmark objects without del that are marked firstpoolnode = lltype.malloc(self.POOLNODE, flavor='raw') firstpoolnode.linkedlist = self.malloced_objects firstpoolnode.nextnode = self.poolnodes prevpoolnode = lltype.nullptr(self.POOLNODE) poolnode = firstpoolnode while poolnode: #sweep ppnext = llmemory.cast_ptr_to_adr(poolnode) ppnext += llmemory.offsetof(self.POOLNODE, 'linkedlist') hdr = poolnode.linkedlist while hdr: #sweep typeid = hdr.typeid >> 1 next = hdr.next addr = llmemory.cast_ptr_to_adr(hdr) size = self.fixed_size(typeid) if self.is_varsize(typeid): length = (addr + size_gc_header + self.varsize_offset_to_length(typeid)).signed[0] size += self.varsize_item_sizes(typeid) * length estimate = raw_malloc_usage(size_gc_header + size) if hdr.typeid & 1: hdr.typeid = hdr.typeid & (~1) ppnext.address[0] = addr ppnext = llmemory.cast_ptr_to_adr(hdr) ppnext += llmemory.offsetof(self.HDR, 'next') curr_heap_size += estimate else: gc_info = llmemory.cast_ptr_to_adr(hdr) obj = gc_info + size_gc_header self.write_free_statistics(typeid, obj) freed_size += estimate raw_free(addr) hdr = next ppnext.address[0] = llmemory.NULL next = poolnode.nextnode if not poolnode.linkedlist and prevpoolnode: # completely empty node prevpoolnode.nextnode = next lltype.free(poolnode, flavor='raw') else: prevpoolnode = poolnode poolnode = next self.malloced_objects = firstpoolnode.linkedlist self.poolnodes = firstpoolnode.nextnode lltype.free(firstpoolnode, flavor='raw') #llop.debug_view(lltype.Void, self.malloced_objects, self.malloced_objects_with_finalizer, size_gc_header) end_time = time.time() compute_time = start_time - self.prev_collect_end_time collect_time = end_time - start_time garbage_collected = old_malloced - (curr_heap_size - self.heap_usage) if (collect_time * curr_heap_size > 0.02 * garbage_collected * compute_time): self.bytes_malloced_threshold += self.bytes_malloced_threshold / 2 if (collect_time * curr_heap_size < 0.005 * garbage_collected * compute_time): self.bytes_malloced_threshold /= 2 # Use atleast as much memory as current live objects. if curr_heap_size > self.bytes_malloced_threshold: self.bytes_malloced_threshold = curr_heap_size # Cap at 1/4 GB self.bytes_malloced_threshold = min(self.bytes_malloced_threshold, 256 * 1024 * 1024) self.total_collection_time += collect_time self.prev_collect_end_time = end_time if DEBUG_PRINT: llop.debug_print(lltype.Void, " malloced since previous collection:", old_malloced, "bytes") llop.debug_print(lltype.Void, " heap usage at start of collection: ", self.heap_usage + old_malloced, "bytes") llop.debug_print(lltype.Void, " freed: ", freed_size, "bytes") llop.debug_print(lltype.Void, " new heap usage: ", curr_heap_size, "bytes") llop.debug_print(lltype.Void, " total time spent collecting: ", self.total_collection_time, "seconds") llop.debug_print(lltype.Void, " collecting time: ", collect_time) llop.debug_print(lltype.Void, " computing time: ", collect_time) llop.debug_print(lltype.Void, " new threshold: ", self.bytes_malloced_threshold) ## llop.debug_view(lltype.Void, self.malloced_objects, self.poolnodes, ## size_gc_header) assert self.heap_usage + old_malloced == curr_heap_size + freed_size self.heap_usage = curr_heap_size hdr = self.malloced_objects_with_finalizer self.malloced_objects_with_finalizer = lltype.nullptr(self.HDR) last = lltype.nullptr(self.HDR) while hdr: next = hdr.next if hdr.typeid & 1: hdr.next = lltype.nullptr(self.HDR) if not self.malloced_objects_with_finalizer: self.malloced_objects_with_finalizer = hdr else: last.next = hdr hdr.typeid = hdr.typeid & (~1) last = hdr else: obj = llmemory.cast_ptr_to_adr(hdr) + size_gc_header finalizer = self.getfinalizer(hdr.typeid >> 1) # make malloced_objects_with_finalizer consistent # for the sake of a possible collection caused by finalizer if not self.malloced_objects_with_finalizer: self.malloced_objects_with_finalizer = next else: last.next = next hdr.next = self.malloced_objects self.malloced_objects = hdr #llop.debug_view(lltype.Void, self.malloced_objects, self.malloced_objects_with_finalizer, size_gc_header) finalizer(obj) if not self.collect_in_progress: # another collection was caused? llop.debug_print(lltype.Void, "outer collect interrupted " "by recursive collect") return if not last: if self.malloced_objects_with_finalizer == next: self.malloced_objects_with_finalizer = lltype.nullptr(self.HDR) else: # now it gets annoying: finalizer caused a malloc of something # with a finalizer last = self.malloced_objects_with_finalizer while last.next != next: last = last.next last.next = lltype.nullptr(self.HDR) else: last.next = lltype.nullptr(self.HDR) hdr = next self.collect_in_progress = False
def sweep_rawmalloced_objects(self, generation): # free all the rawmalloced objects of the specified generation # that have not been marked if generation == 2: objects = self.gen2_rawmalloced_objects # generation 2 sweep: if A points to an object object B that # moves from gen2 to gen3, it's possible that A no longer points # to any gen2 object. In this case, A remains a bit too long in # last_generation_root_objects, but this will be fixed by the # next collect_last_generation_roots(). elif generation == 3: objects = self.gen3_rawmalloced_objects # generation 3 sweep: remove from last_generation_root_objects # all the objects that we are about to free gen3roots = self.last_generation_root_objects newgen3roots = self.AddressStack() while gen3roots.non_empty(): obj = gen3roots.pop() if not (self.header(obj).tid & GCFLAG_UNVISITED): newgen3roots.append(obj) gen3roots.delete() self.last_generation_root_objects = newgen3roots else: # mostly a hack: the generation number -2 is the part of the # generation 2 that lives in gen2_resizable_objects ll_assert(generation == -2, "bogus 'generation'") objects = self.gen2_resizable_objects surviving_objects = self.AddressStack() # Help the flow space alive_count = alive_size = dead_count = dead_size = 0 while objects.non_empty(): obj = objects.pop() tid = self.header(obj).tid if tid & GCFLAG_UNVISITED: if self.config.gcconfig.debugprint: dead_count += 1 dead_size += raw_malloc_usage(self.get_size_incl_hash(obj)) addr = obj - self.gcheaderbuilder.size_gc_header llmemory.raw_free(addr) else: if self.config.gcconfig.debugprint: alive_count += 1 alive_size += raw_malloc_usage( self.get_size_incl_hash(obj)) if generation == 3: surviving_objects.append(obj) elif generation == 2: ll_assert((tid & GCFLAG_AGE_MASK) < GCFLAG_AGE_MAX, "wrong age for generation 2 object") tid += GCFLAG_AGE_ONE if (tid & GCFLAG_AGE_MASK) == GCFLAG_AGE_MAX: # the object becomes part of generation 3 self.gen3_rawmalloced_objects.append(obj) # GCFLAG_NO_HEAP_PTRS not set yet, conservatively self.last_generation_root_objects.append(obj) else: # the object stays in generation 2 tid |= GCFLAG_UNVISITED surviving_objects.append(obj) self.header(obj).tid = tid elif generation == -2: # the object stays in generation -2 tid |= GCFLAG_UNVISITED surviving_objects.append(obj) self.header(obj).tid = tid objects.delete() if generation == 2: self.gen2_rawmalloced_objects = surviving_objects elif generation == 3: self.gen3_rawmalloced_objects = surviving_objects elif generation == -2: self.gen2_resizable_objects = surviving_objects if self.config.gcconfig.debugprint: llop.debug_print(lltype.Void, "| [hyb] gen", generation, "nonmoving now alive: ", alive_size, "bytes in", alive_count, "objs") llop.debug_print(lltype.Void, "| [hyb] gen", generation, "nonmoving freed: ", dead_size, "bytes in", dead_count, "objs")
def semispace_collect(self, size_changing=False): if self.config.gcconfig.debugprint: llop.debug_print(lltype.Void) llop.debug_print(lltype.Void, ".----------- Full collection ------------------") start_usage = self.free - self.tospace llop.debug_print(lltype.Void, "| used before collection: ", start_usage, "bytes") start_time = time.time() else: start_time = 0 # Help the flow space start_usage = 0 # Help the flow space #llop.debug_print(lltype.Void, 'semispace_collect', int(size_changing)) tospace = self.fromspace fromspace = self.tospace self.fromspace = fromspace self.tospace = tospace self.top_of_space = tospace + self.space_size scan = self.free = tospace self.starting_full_collect() self.collect_roots() if self.run_finalizers.non_empty(): self.update_run_finalizers() scan = self.scan_copied(scan) if self.objects_with_finalizers.non_empty(): scan = self.deal_with_objects_with_finalizers(scan) if self.objects_with_weakrefs.non_empty(): self.invalidate_weakrefs() self.update_objects_with_id() self.finished_full_collect() self.debug_check_consistency() if not size_changing: llarena.arena_reset(fromspace, self.space_size, True) self.record_red_zone() self.execute_finalizers() #llop.debug_print(lltype.Void, 'collected', self.space_size, size_changing, self.top_of_space - self.free) if self.config.gcconfig.debugprint: end_time = time.time() elapsed_time = end_time - start_time self.total_collection_time += elapsed_time self.total_collection_count += 1 total_program_time = end_time - self.program_start_time end_usage = self.free - self.tospace llop.debug_print(lltype.Void, "| used after collection: ", end_usage, "bytes") llop.debug_print(lltype.Void, "| freed: ", start_usage - end_usage, "bytes") llop.debug_print(lltype.Void, "| size of each semispace: ", self.space_size, "bytes") llop.debug_print(lltype.Void, "| fraction of semispace now used: ", end_usage * 100.0 / self.space_size, "%") ct = self.total_collection_time cc = self.total_collection_count llop.debug_print(lltype.Void, "| number of semispace_collects: ", cc) llop.debug_print(lltype.Void, "| i.e.: ", cc / total_program_time, "per second") llop.debug_print(lltype.Void, "| total time in semispace_collect: ", ct, "seconds") llop.debug_print(lltype.Void, "| i.e.: ", ct * 100.0 / total_program_time, "%") llop.debug_print(lltype.Void, "`----------------------------------------------")