def dump(self, memo): if have_debug_prints(): debug_start("jit-log-exported-state") debug_print("[" + ", ".join([x.repr_short(memo) for x in self.next_iteration_args]) + "]") for box in self.short_boxes: debug_print(" " + box.repr(memo)) debug_stop("jit-log-exported-state")
def send_loop_to_backend(greenkey, jitdriver_sd, metainterp_sd, loop, type, orig_inpargs, memo): forget_optimization_info(loop.operations) forget_optimization_info(loop.inputargs) vinfo = jitdriver_sd.virtualizable_info if vinfo is not None: vable = orig_inpargs[jitdriver_sd.index_of_virtualizable].getref_base() patch_new_loop_to_load_virtualizable_fields(loop, jitdriver_sd, vable) original_jitcell_token = loop.original_jitcell_token globaldata = metainterp_sd.globaldata original_jitcell_token.number = n = globaldata.loopnumbering globaldata.loopnumbering += 1 if not we_are_translated(): show_procedures(metainterp_sd, loop) loop.check_consistency() if metainterp_sd.warmrunnerdesc is not None: hooks = metainterp_sd.warmrunnerdesc.hooks debug_info = JitDebugInfo(jitdriver_sd, metainterp_sd.logger_ops, original_jitcell_token, loop.operations, type, greenkey) hooks.before_compile(debug_info) else: debug_info = None hooks = None operations = get_deep_immutable_oplist(loop.operations) metainterp_sd.profiler.start_backend() debug_start("jit-backend") try: loopname = jitdriver_sd.warmstate.get_location_str(greenkey) unique_id = jitdriver_sd.warmstate.get_unique_id(greenkey) asminfo = do_compile_loop(jitdriver_sd.index, unique_id, metainterp_sd, loop.inputargs, operations, original_jitcell_token, name=loopname, log=have_debug_prints(), memo=memo) finally: debug_stop("jit-backend") metainterp_sd.profiler.end_backend() if hooks is not None: debug_info.asminfo = asminfo hooks.after_compile(debug_info) metainterp_sd.stats.add_new_loop(loop) if not we_are_translated(): metainterp_sd.stats.compiled() metainterp_sd.log("compiled new " + type) # if asminfo is not None: ops_offset = asminfo.ops_offset else: ops_offset = None metainterp_sd.logger_ops.log_loop(loop.inputargs, loop.operations, n, type, ops_offset, name=loopname) # if metainterp_sd.warmrunnerdesc is not None: # for tests metainterp_sd.warmrunnerdesc.memory_manager.keep_loop_alive(original_jitcell_token)
def f(x): debug_start("mycat") debug_print("foo", 2, "bar", x) debug_stop("mycat") debug_flush() # does nothing debug_offset() # should not explode at least return have_debug_prints()
def set_nursery_size(self, newsize): debug_start("gc-set-nursery-size") if newsize < self.min_nursery_size: newsize = self.min_nursery_size if newsize > self.space_size // 2: newsize = self.space_size // 2 # Compute the new bounds for how large young objects can be # (larger objects are allocated directly old). XXX adjust self.nursery_size = newsize self.largest_young_fixedsize = self.get_young_fixedsize(newsize) self.largest_young_var_basesize = self.get_young_var_basesize(newsize) scale = 0 while (self.min_nursery_size << (scale+1)) <= newsize: scale += 1 self.nursery_scale = scale debug_print("nursery_size =", newsize) debug_print("largest_young_fixedsize =", self.largest_young_fixedsize) debug_print("largest_young_var_basesize =", self.largest_young_var_basesize) debug_print("nursery_scale =", scale) # we get the following invariant: assert self.nursery_size >= (self.min_nursery_size << scale) # Force a full collect to remove the current nursery whose size # no longer matches the bounds that we just computed. This must # be done after changing the bounds, because it might re-create # a new nursery (e.g. if it invokes finalizers). self.semispace_collect() debug_stop("gc-set-nursery-size")
def detect_arch_version(filename="/proc/cpuinfo"): fd = os.open(filename, os.O_RDONLY, 0644) n = 0 debug_start("jit-backend-arch") try: buf = os.read(fd, 2048) if not buf: n = 6 # we assume ARMv6 as base case debug_print("Could not detect ARM architecture " "version, assuming", "ARMv%d" % n) finally: os.close(fd) # "Processor : ARMv%d-compatible processor rev 7 (v6l)" i = buf.find('ARMv') if i == -1: n = 6 debug_print("Could not detect architecture version, " "falling back to", "ARMv%d" % n) else: n = int(buf[i + 4]) if n < 6: raise ValueError("Unsupported ARM architecture version") debug_print("Detected", "ARMv%d" % n) if n > 7: n = 7 debug_print("Architecture version not explicitly supported, " "falling back to", "ARMv%d" % n) debug_stop("jit-backend-arch") return n
def setup_once(self): # the address of the function called by 'new' gc_ll_descr = self.cpu.gc_ll_descr gc_ll_descr.initialize() if hasattr(gc_ll_descr, 'minimal_size_in_nursery'): self.gc_minimal_size_in_nursery = gc_ll_descr.minimal_size_in_nursery else: self.gc_minimal_size_in_nursery = 0 if hasattr(gc_ll_descr, 'gcheaderbuilder'): self.gc_size_of_header = gc_ll_descr.gcheaderbuilder.size_gc_header else: self.gc_size_of_header = WORD # for tests self.memcpy_addr = self.cpu.cast_ptr_to_int(memcpy_fn) self.memset_addr = self.cpu.cast_ptr_to_int(memset_fn) self._build_failure_recovery(False, withfloats=False) self._build_failure_recovery(True, withfloats=False) self._build_wb_slowpath(False) self._build_wb_slowpath(True) self._build_wb_slowpath(False, for_frame=True) # only one of those self.build_frame_realloc_slowpath() if self.cpu.supports_floats: self._build_failure_recovery(False, withfloats=True) self._build_failure_recovery(True, withfloats=True) self._build_wb_slowpath(False, withfloats=True) self._build_wb_slowpath(True, withfloats=True) self._build_propagate_exception_path() if gc_ll_descr.get_malloc_slowpath_addr is not None: # generate few slowpaths for various cases self.malloc_slowpath = self._build_malloc_slowpath(kind='fixed') self.malloc_slowpath_varsize = self._build_malloc_slowpath( kind='var') if hasattr(gc_ll_descr, 'malloc_str'): self.malloc_slowpath_str = self._build_malloc_slowpath(kind='str') else: self.malloc_slowpath_str = None if hasattr(gc_ll_descr, 'malloc_unicode'): self.malloc_slowpath_unicode = self._build_malloc_slowpath( kind='unicode') else: self.malloc_slowpath_unicode = None self.cond_call_slowpath = [self._build_cond_call_slowpath(False, False), self._build_cond_call_slowpath(False, True), self._build_cond_call_slowpath(True, False), self._build_cond_call_slowpath(True, True)] self._build_stack_check_slowpath() self._build_release_gil(gc_ll_descr.gcrootmap) if not self._debug: # if self._debug is already set it means that someone called # set_debug by hand before initializing the assembler. Leave it # as it is debug_start('jit-backend-counts') self.set_debug(have_debug_prints()) debug_stop('jit-backend-counts') # when finishing, we only have one value at [0], the rest dies self.gcmap_for_finish = lltype.malloc(jitframe.GCMAP, 1, flavor='raw', track_allocation=False) self.gcmap_for_finish[0] = r_uint(1)
def disable_noninlinable_function(self, greenkey): cell = self.JitCell.ensure_jit_cell_at_key(greenkey) cell.flags |= JC_DONT_TRACE_HERE debug_start("jit-disableinlining") loc = self.get_location_str(greenkey) debug_print("disabled inlining", loc) debug_stop("jit-disableinlining")
def disable_noninlinable_function(self, greenkey): cell = self.jit_cell_at_key(greenkey) cell.dont_trace_here = True debug_start("jit-disableinlining") loc = self.get_location_str(greenkey) debug_print("disabled inlining", loc) debug_stop("jit-disableinlining")
def get_L2cache_linux2_cpuinfo_s390x(filename="/proc/cpuinfo", label='cache3'): debug_start("gc-hardware") L2cache = sys.maxint try: fd = os.open(filename, os.O_RDONLY, 0644) try: data = [] while True: buf = os.read(fd, 4096) if not buf: break data.append(buf) finally: os.close(fd) except OSError: pass else: data = ''.join(data) linepos = 0 while True: start = _findend(data, '\n' + label, linepos) if start < 0: break # done linepos = _findend(data, '\n', start) if linepos < 0: break # no end-of-line?? # *** data[start:linepos] == " : level=2 type=Instruction scope=Private size=2048K ..." start = _skipspace(data, start) if data[start] != ':': continue # *** data[start:linepos] == ": level=2 type=Instruction scope=Private size=2048K ..." start = _skipspace(data, start + 1) # *** data[start:linepos] == "level=2 type=Instruction scope=Private size=2048K ..." start += 44 end = start while '0' <= data[end] <= '9': end += 1 # *** data[start:end] == "2048" if start == end: continue number = int(data[start:end]) # *** data[end:linepos] == " KB\n" end = _skipspace(data, end) if data[end] not in ('K', 'k'): # assume kilobytes for now continue number = number * 1024 # for now we look for the smallest of the L2 caches of the CPUs if number < L2cache: L2cache = number debug_print("L2cache =", L2cache) debug_stop("gc-hardware") if L2cache < sys.maxint: return L2cache else: # Print a top-level warning even in non-debug builds llop.debug_print(lltype.Void, "Warning: cannot find your CPU L2 cache size in /proc/cpuinfo") return -1
def get_L2cache_linux2_sparc(): debug_start("gc-hardware") cpu = 0 L2cache = sys.maxint while True: try: fd = os.open('/sys/devices/system/cpu/cpu' + assert_str0(str(cpu)) + '/l2_cache_size', os.O_RDONLY, 0644) try: line = os.read(fd, 4096) finally: os.close(fd) end = len(line) - 1 assert end > 0 number = int(line[:end]) except OSError: break if number < L2cache: L2cache = number cpu += 1 debug_print("L2cache =", L2cache) debug_stop("gc-hardware") if L2cache < sys.maxint: return L2cache else: # Print a top-level warning even in non-debug builds llop.debug_print(lltype.Void, "Warning: cannot find your CPU L2 cache size in " "/sys/devices/system/cpu/cpuX/l2_cache_size") return -1
def _do(self, goal, func, *args, **kwds): title = func.task_title if goal in self.done: self.log.info("already done: %s" % title) return else: self.log.info("%s..." % title) debug_start('translation-task') debug_print('starting', goal) self.timer.start_event(goal) try: instrument = False try: if goal in PROFILE: res = self._profile(goal, func) else: res = func() except Instrument: instrument = True if not func.task_idempotent: self.done[goal] = True if instrument: self.proceed('compile') assert False, 'we should not get here' finally: try: debug_stop('translation-task') self.timer.end_event(goal) except (KeyboardInterrupt, SystemExit): raise except: pass #import gc; gc.dump_rpy_heap('rpyheap-after-%s.dump' % goal) return res
def load_linklet_from_fasl(file_name, set_version=False): from pycket.fasl import Fasl from pycket.env import w_version from pycket.util import console_log from pycket.ast_vs_sexp import deserialize_loop debug_start("loading-linklet") debug_print("Loading linklet from fasl -- %s" % file_name) sexp = Fasl().to_sexp_from_file(file_name) version_sexp, linklet_sexp = W_String.make(""), None if set_version: version_sexp = sexp.car() linklet_sexp = sexp.cdr() else: linklet_sexp = sexp linklet = None if "zo" in file_name: linklet = deserialize_loop(linklet_sexp) else: console_log("Run pycket with --make-linklet-zos to make the compiled zo files for bootstrap linklets", 1) compile_linklet = get_primitive("compile-linklet") linklet = compile_linklet.call_interpret([linklet_sexp, W_Symbol.make("linkl"), w_false, w_false, w_false]) if set_version: ver = version_sexp.as_str_ascii() console_log("Setting the version to %s" % ver) w_version.set_version(ver) debug_stop("loading-linklet") return linklet, version_sexp
def get_total_memory_linux(filename): debug_start("gc-hardware") result = -1.0 try: fd = os.open(filename, os.O_RDONLY, 0644) try: buf = os.read(fd, 4096) finally: os.close(fd) except OSError: pass else: if buf.startswith('MemTotal:'): start = _skipspace(buf, len('MemTotal:')) stop = start while stop < len(buf) and buf[stop].isdigit(): stop += 1 if start < stop: result = float(buf[start:stop]) * 1024.0 # assume kB if result < 0.0: debug_print("get_total_memory() failed") result = addressable_size else: debug_print("memtotal =", result) if result > addressable_size: result = addressable_size debug_stop("gc-hardware") return result
def _dump(self, addr, logname, backend=None): debug_start(logname) if have_debug_prints(): # if backend is not None: debug_print('BACKEND', backend) # from rpython.jit.backend.hlinfo import highleveljitinfo if highleveljitinfo.sys_executable: debug_print('SYS_EXECUTABLE', highleveljitinfo.sys_executable) else: debug_print('SYS_EXECUTABLE', '??') # HEX = '0123456789ABCDEF' dump = [] src = rffi.cast(rffi.CCHARP, addr) for p in range(self.get_relative_pos()): o = ord(src[p]) dump.append(HEX[o >> 4]) dump.append(HEX[o & 15]) debug_print('CODE_DUMP', '@%x' % addr, '+0 ', # backwards compatibility ''.join(dump)) # debug_stop(logname)
def send_bridge_to_backend(jitdriver_sd, metainterp_sd, faildescr, inputargs, operations, original_loop_token): if not we_are_translated(): show_procedures(metainterp_sd) seen = dict.fromkeys(inputargs) TreeLoop.check_consistency_of_branch(operations, seen) if metainterp_sd.warmrunnerdesc is not None: hooks = metainterp_sd.warmrunnerdesc.hooks debug_info = JitDebugInfo( jitdriver_sd, metainterp_sd.logger_ops, original_loop_token, operations, "bridge", fail_descr=faildescr ) hooks.before_compile_bridge(debug_info) else: hooks = None debug_info = None operations = get_deep_immutable_oplist(operations) metainterp_sd.profiler.start_backend() debug_start("jit-backend") try: asminfo = do_compile_bridge(metainterp_sd, faildescr, inputargs, operations, original_loop_token) finally: debug_stop("jit-backend") metainterp_sd.profiler.end_backend() if hooks is not None: debug_info.asminfo = asminfo hooks.after_compile_bridge(debug_info) if not we_are_translated(): metainterp_sd.stats.compiled() metainterp_sd.log("compiled new bridge") # if asminfo is not None: ops_offset = asminfo.ops_offset else: ops_offset = None metainterp_sd.logger_ops.log_bridge(inputargs, operations, None, faildescr, ops_offset)
def _instantiate_linklet(file_name_for_log, linkl): debug_start("instantiating-linklet") debug_print("Instantiating : %s" % file_name_for_log) instantiate_linklet = get_primitive("instantiate-linklet") linkl_instance = instantiate_linklet.call_interpret([linkl, w_null, w_false, w_false]) debug_print("DONE Instantiating %s ...." % file_name_for_log) debug_stop("instantiating-linklet") return linkl_instance
def _check_rawsize_alloced(self, size_estimate): self.large_objects_collect_trigger -= size_estimate if self.large_objects_collect_trigger < 0: debug_start("gc-rawsize-collect") debug_print("allocated", (self._initial_trigger - self.large_objects_collect_trigger), "bytes, triggering full collection") self.semispace_collect() debug_stop("gc-rawsize-collect")
def log_abort_loop(self, trace, memo=None): debug_start("jit-abort-log") if not have_debug_prints(): return inputargs, operations = self._unpack_trace(trace) logops = self._log_operations(inputargs, operations, ops_offset=None, memo=memo) debug_stop("jit-abort-log") return logops
def _invalid_read(self, message, offset, length, descr): debug_start('jit-log-rawbuffer') debug_print('Invalid read: %s' % message) debug_print(" offset: %d" % offset) debug_print(" length: %d" % length) debug_print(" descr: %s" % self._repr_of_descr(descr)) self._dump_to_log() debug_stop('jit-log-rawbuffer') raise InvalidRawRead
def debug_print(self, logops): if 0: debug_start('jit-short-boxes') for box, op in self.short_boxes.items(): if op: debug_print(logops.repr_of_arg(box) + ': ' + logops.repr_of_resop(op)) else: debug_print(logops.repr_of_arg(box) + ': None') debug_stop('jit-short-boxes')
def debug_print(self, logops): if 0: debug_start("jit-short-boxes") for box, op in self.short_boxes.items(): if op: debug_print(logops.repr_of_arg(box) + ": " + logops.repr_of_resop(op)) else: debug_print(logops.repr_of_arg(box) + ": None") debug_stop("jit-short-boxes")
def entry_point(argv): debug_start("foo") debug_print("test line") childpid = os.fork() debug_print("childpid =", childpid) if childpid == 0: childpid2 = os.fork() # double-fork debug_print("childpid2 =", childpid2) debug_stop("foo") return 0
def _invalid_write(self, message, offset, length, descr, value): debug_start('jit-log-rawbuffer') debug_print('Invalid write: %s' % message) debug_print(" offset: %d" % offset) debug_print(" length: %d" % length) debug_print(" descr: %s" % self._repr_of_descr(descr)) debug_print(" value: %s" % self._repr_of_value(value)) self._dump_to_log() debug_stop('jit-log-rawbuffer') raise InvalidRawWrite
def optimize_vector(trace, metainterp_sd, jitdriver_sd, warmstate, loop_info, loop_ops, jitcell_token=None): """ Enter the world of SIMD. Bails if it cannot transform the trace. """ user_code = not jitdriver_sd.vec and warmstate.vec_all e = len(loop_ops)-1 assert e > 0 assert rop.is_final(loop_ops[e].getopnum()) loop = VectorLoop(loop_info.label_op, loop_ops[:e], loop_ops[-1]) if user_code and user_loop_bail_fast_path(loop, warmstate): return loop_info, loop_ops # the original loop (output of optimize_unroll) info = LoopVersionInfo(loop_info) version = info.snapshot(loop) loop.setup_vectorization() try: debug_start("vec-opt-loop") metainterp_sd.logger_noopt.log_loop([], loop.finaloplist(label=True), -2, None, None, "pre vectorize") metainterp_sd.profiler.count(Counters.OPT_VECTORIZE_TRY) # start = time.clock() opt = VectorizingOptimizer(metainterp_sd, jitdriver_sd, warmstate.vec_cost) index_vars = opt.run_optimization(info, loop) gso = GuardStrengthenOpt(index_vars) gso.propagate_all_forward(info, loop, user_code) end = time.clock() # metainterp_sd.profiler.count(Counters.OPT_VECTORIZED) metainterp_sd.logger_noopt.log_loop([], loop.finaloplist(label=True), -2, None, None, "post vectorize") nano = int((end-start)*10.0**9) debug_print("# vecopt factor: %d opcount: (%d -> %d) took %dns" % \ (opt.unroll_count+1, len(version.loop.operations), len(loop.operations), nano)) debug_stop("vec-opt-loop") # info.label_op = loop.label return info, loop.finaloplist(jitcell_token=jitcell_token, reset_label_token=False) except NotAVectorizeableLoop: debug_stop("vec-opt-loop") # vectorization is not possible return loop_info, version.loop.finaloplist() except NotAProfitableLoop: debug_stop("vec-opt-loop") # cost model says to skip this loop return loop_info, version.loop.finaloplist() except Exception as e: debug_stop("vec-opt-loop") debug_print("failed to vectorize loop. THIS IS A FATAL ERROR!") if we_are_translated(): from rpython.rtyper.lltypesystem import lltype from rpython.rtyper.lltypesystem.lloperation import llop llop.debug_print_traceback(lltype.Void) else: raise finally: loop.teardown_vectorization() return loop_info, loop_ops
def get_total_memory_darwin(result): debug_start("gc-hardware") if result <= 0: debug_print("get_total_memory() failed") result = addressable_size else: debug_print("memtotal = ", result) if result > addressable_size: result = addressable_size debug_stop("gc-hardware") return result
def __init__(self, cpu, number): cpu.tracker.total_compiled_loops += 1 self.cpu = cpu self.number = number self.bridges_count = 0 self.invalidate_positions = [] # a list of weakrefs to looptokens that has been redirected to # this one self.looptokens_redirected_to = [] debug_start("jit-mem-looptoken-alloc") debug_print("allocating Loop #", self.number) debug_stop("jit-mem-looptoken-alloc")
def finish_once(self): if self._debug: debug_start('jit-backend-counts') for i in range(len(self.loop_run_counters)): struct = self.loop_run_counters[i] if struct.type == 'l': prefix = 'TargetToken(%d)' % struct.number elif struct.type == 'b': prefix = 'bridge ' + str(struct.number) else: prefix = 'entry ' + str(struct.number) debug_print(prefix + ':' + str(struct.i)) debug_stop('jit-backend-counts')
def done(self): from rpython.rlib.debug import debug_start, debug_stop, debug_print self._bigints_dict = {} self._refs_dict = llhelper.new_ref_dict_3() debug_start("jit-trace-done") debug_print("trace length: " + str(self._pos)) debug_print(" total snapshots: " + str(self._total_snapshots)) debug_print(" bigint consts: " + str(self._consts_bigint) + " " + str(len(self._bigints))) debug_print(" float consts: " + str(self._consts_float) + " " + str(len(self._floats))) debug_print(" ref consts: " + str(self._consts_ptr) + " " + str(len(self._refs))) debug_print(" descrs: " + str(len(self._descrs))) debug_stop("jit-trace-done") return 0 # completely different than TraceIter.done, but we have to
def log_bridge(self, inputargs, operations, extra=None, descr=None, ops_offset=None, memo=None): if extra == "noopt": debug_start("jit-log-noopt-bridge") debug_print("# bridge out of Guard", "0x%x" % compute_unique_id(descr), "with", len(operations), "ops") logops = self._log_operations(inputargs, operations, ops_offset, memo) debug_stop("jit-log-noopt-bridge") elif extra == "rewritten": debug_start("jit-log-rewritten-bridge") debug_print("# bridge out of Guard", "0x%x" % compute_unique_id(descr), "with", len(operations), "ops") logops = self._log_operations(inputargs, operations, ops_offset, memo) debug_stop("jit-log-rewritten-bridge") elif extra == "compiling": debug_start("jit-log-compiling-bridge") logops = self._log_operations(inputargs, operations, ops_offset, memo) debug_stop("jit-log-compiling-bridge") else: debug_start("jit-log-opt-bridge") debug_print("# bridge out of Guard", "0x%x" % r_uint(compute_unique_id(descr)), "with", len(operations), "ops") logops = self._log_operations(inputargs, operations, ops_offset, memo) debug_stop("jit-log-opt-bridge") return logops
def log_loop(self, inputargs, operations, number=0, type=None, ops_offset=None, name='', memo=None): if type is None: debug_start("jit-log-noopt-loop") debug_print("# Loop", number, '(%s)' % name, ":", "noopt", "with", len(operations), "ops") logops = self._log_operations(inputargs, operations, ops_offset, memo) debug_stop("jit-log-noopt-loop") elif type == "rewritten": debug_start("jit-log-rewritten-loop") debug_print("# Loop", number, '(%s)' % name, ":", type, "with", len(operations), "ops") logops = self._log_operations(inputargs, operations, ops_offset, memo) debug_stop("jit-log-rewritten-loop") elif number == -2: debug_start("jit-log-compiling-loop") logops = self._log_operations(inputargs, operations, ops_offset, memo) debug_stop("jit-log-compiling-loop") else: debug_start("jit-log-opt-loop") debug_print("# Loop", number, '(%s)' % name, ":", type, "with", len(operations), "ops") logops = self._log_operations(inputargs, operations, ops_offset, memo) debug_stop("jit-log-opt-loop") return logops
def finish_once(self): if self._debug: debug_start('jit-backend-counts') for i in range(len(self.loop_run_counters)): struct = self.loop_run_counters[i] if struct.type == 'l': prefix = 'TargetToken(%d)' % struct.number else: num = struct.number if num == -1: num = '-1' else: num = str(r_uint(num)) if struct.type == 'b': prefix = 'bridge %s' % num else: prefix = 'entry %s' % num debug_print(prefix + ':' + str(struct.i)) debug_stop('jit-backend-counts')
def test_debuglog_summary(debuglog): debug_start('foo') debug_start('bar') # this is nested, so not counted in the summary by default debug_stop('bar') debug_stop('foo') debug_start('foo') debug_stop('foo') debug_start('bar') debug_stop('bar') # assert debuglog.summary() == {'foo': 2, 'bar': 1} assert debuglog.summary(flatten=True) == {'foo': 2, 'bar': 2}
def _dump(self, addr, logname, backend=None): debug_start(logname) if have_debug_prints(): # if backend is not None: debug_print('BACKEND', backend) # from rpython.jit.backend.hlinfo import highleveljitinfo if highleveljitinfo.sys_executable: debug_print('SYS_EXECUTABLE', highleveljitinfo.sys_executable) else: debug_print('SYS_EXECUTABLE', '??') # dump = self.copy_core_dump(addr) debug_print('CODE_DUMP', '@%x' % addr, '+0 ', # backwards compatibility dump) # debug_stop(logname)
def settle(basename): """ Settle Tags, shapes and transformations to a '.docked' file punching and marshalling all current Tags. """ debug_start("theseus-settle") path = basename + '.docked' buf = [] marshaller(buf, punch_tags(W_Tag.tags)) try: f = open_file_as_stream(path, mode="w", buffering=0) except OSError as e: os.write(2, "Error(settle)%s -- %s\n" % (os.strerror(e.errno), path)) return try: f.write(''.join(buf)) finally: f.close() debug_stop("theseus-settle")
def optimize_trace(self, metainterp_sd, jitdriver_sd, memo): """Optimize loop.operations to remove internal overheadish operations. """ from rpython.jit.metainterp.optimizeopt import build_opt_chain # mark that a new trace has been started log = metainterp_sd.jitlog.log_trace(jl.MARK_TRACE, metainterp_sd, None) log.write_trace(self.trace) if self.log_noopt: metainterp_sd.logger_noopt.log_loop_from_trace(self.trace, memo=memo) self.box_names_memo = memo optimizations = build_opt_chain(self.enable_opts) debug_start("jit-optimize") try: return self.optimize(metainterp_sd, jitdriver_sd, optimizations) finally: self.forget_optimization_info() debug_stop("jit-optimize")
def optimize_trace(metainterp_sd, jitdriver_sd, compile_data, memo=None): """Optimize loop.operations to remove internal overheadish operations. """ debug_start("jit-optimize") try: # mark that a new trace has been started log = metainterp_sd.jitlog.log_trace(jl.MARK_TRACE, metainterp_sd, None) log.write_trace(compile_data.trace) if compile_data.log_noopt: metainterp_sd.logger_noopt.log_loop_from_trace(compile_data.trace, memo=memo) if memo is None: memo = {} compile_data.box_names_memo = memo optimizations, unroll = build_opt_chain(metainterp_sd, compile_data.enable_opts) return compile_data.optimize(metainterp_sd, jitdriver_sd, optimizations, unroll) finally: compile_data.forget_optimization_info() debug_stop("jit-optimize")
def load_inst_linklet_json(json_file_name, debug=False, set_version=False, generate_zo=False, expose_vars=False): from pycket.env import w_version debug_start("loading-linklet") debug_print("loading and instantiating : %s" % json_file_name) console_log("Loading linklet from %s" % json_file_name) linkl, sys_config = W_Linklet.load_linklet(json_file_name, set_version, generate_zo=generate_zo) debug_print("DONE with loading : %s" % json_file_name) console_log("Instantiating %s ...." % json_file_name) debug_print("Instantiating %s ...." % json_file_name) instantiate_linklet = get_primitive("instantiate-linklet") linkl_instance = instantiate_linklet.call_interpret([linkl, w_null, w_false, w_false]) debug_print("DONE Instantiating %s ...." % json_file_name) debug_stop("loading-linklet") if expose_vars: console_log("Exporting vars of %s" % json_file_name) linkl_instance.expose_vars_to_prim_env() console_log("DONE with the %s." % json_file_name) return linkl_instance, sys_config
def get_L2cache_darwin(): """Try to estimate the best nursery size at run-time, depending on the machine we are running on. """ debug_start("gc-hardware") L2cache = get_darwin_sysctl_signed("hw.l2cachesize") L3cache = get_darwin_sysctl_signed("hw.l3cachesize") debug_print("L2cache =", L2cache) debug_print("L3cache =", L3cache) debug_stop("gc-hardware") mangled = L2cache + L3cache if mangled > 0: return mangled else: # Print a top-level warning even in non-debug builds llop.debug_print( lltype.Void, "Warning: cannot find your CPU L2 cache size with sysctl()") return -1
def send_bridge_to_backend(jitdriver_sd, metainterp_sd, faildescr, inputargs, operations, original_loop_token): if not we_are_translated(): show_procedures(metainterp_sd) seen = dict.fromkeys(inputargs) TreeLoop.check_consistency_of_branch(operations, seen) if metainterp_sd.warmrunnerdesc is not None: hooks = metainterp_sd.warmrunnerdesc.hooks debug_info = JitDebugInfo(jitdriver_sd, metainterp_sd.logger_ops, original_loop_token, operations, 'bridge', fail_descr=faildescr) hooks.before_compile_bridge(debug_info) else: hooks = None debug_info = None operations = get_deep_immutable_oplist(operations) metainterp_sd.profiler.start_backend() debug_start("jit-backend") try: asminfo = do_compile_bridge(metainterp_sd, faildescr, inputargs, operations, original_loop_token) finally: debug_stop("jit-backend") metainterp_sd.profiler.end_backend() if hooks is not None: debug_info.asminfo = asminfo hooks.after_compile_bridge(debug_info) if not we_are_translated(): metainterp_sd.stats.compiled() metainterp_sd.log("compiled new bridge") # if asminfo is not None: ops_offset = asminfo.ops_offset else: ops_offset = None metainterp_sd.logger_ops.log_bridge(inputargs, operations, None, faildescr, ops_offset)
def optimize_trace(metainterp_sd, jitdriver_sd, loop, enable_opts, inline_short_preamble=True, start_state=None, export_state=True): """Optimize loop.operations to remove internal overheadish operations. """ debug_start("jit-optimize") try: loop.logops = metainterp_sd.logger_noopt.log_loop(loop.inputargs, loop.operations) optimizations, unroll = build_opt_chain(metainterp_sd, enable_opts) if unroll: return optimize_unroll(metainterp_sd, jitdriver_sd, loop, optimizations, inline_short_preamble, start_state, export_state) else: optimizer = Optimizer(metainterp_sd, jitdriver_sd, loop, optimizations) optimizer.propagate_all_forward() finally: debug_stop("jit-optimize")
def collect_nursery(self): if self.nursery_size > self.top_of_space - self.free: # the semispace is running out, do a full collect self.obtain_free_space(self.nursery_size) ll_assert(self.nursery_size <= self.top_of_space - self.free, "obtain_free_space failed to do its job") if self.nursery: debug_start("gc-minor") debug_print("--- minor collect ---") debug_print("nursery:", self.nursery, "to", self.nursery_top) # a nursery-only collection scan = beginning = self.free self.collect_oldrefs_to_nursery() self.collect_roots_in_nursery() self.collect_young_objects_with_finalizers() scan = self.scan_objects_just_copied_out_of_nursery(scan) # at this point, all static and old objects have got their # GCFLAG_NO_YOUNG_PTRS set again by trace_and_drag_out_of_nursery if self.young_objects_with_weakrefs.non_empty(): self.invalidate_young_weakrefs() if self.young_objects_with_id.length() > 0: self.update_young_objects_with_id() # mark the nursery as free and fill it with zeroes again llarena.arena_reset(self.nursery, self.nursery_size, 2) debug_print("survived (fraction of the size):", float(scan - beginning) / self.nursery_size) debug_stop("gc-minor") #self.debug_check_consistency() # -- quite expensive else: # no nursery - this occurs after a full collect, triggered either # just above or by some previous non-nursery-based allocation. # Grab a piece of the current space for the nursery. self.nursery = self.free self.nursery_top = self.nursery + self.nursery_size self.free = self.nursery_top self.nursery_free = self.nursery # at this point we know that the nursery is empty self.change_nursery_hash_base() return self.nursery_free
def finish_once(self): if self._debug: # TODO remove the old logging system when jitlog is complete debug_start('jit-backend-counts') length = len(self.loop_run_counters) for i in range(length): struct = self.loop_run_counters[i] if struct.type == 'l': prefix = 'TargetToken(%d)' % struct.number else: num = struct.number if num == -1: num = '-1' else: num = str(r_uint(num)) if struct.type == 'b': prefix = 'bridge %s' % num else: prefix = 'entry %s' % num debug_print(prefix + ':' + str(struct.i)) debug_stop('jit-backend-counts') self.flush_trace_counters()
def _kill_old_loops_now(self): debug_start("jit-mem-collect") oldtotal = len(self.alive_loops) #print self.alive_loops.keys() debug_print("Current generation:", self.current_generation) debug_print("Loop tokens before:", oldtotal) max_generation = self.current_generation - (self.max_age - 1) for looptoken in self.alive_loops.keys(): if (0 <= looptoken.generation < max_generation or looptoken.invalidated): del self.alive_loops[looptoken] newtotal = len(self.alive_loops) debug_print("Loop tokens freed: ", oldtotal - newtotal) debug_print("Loop tokens left: ", newtotal) #print self.alive_loops.keys() if not we_are_translated() and oldtotal != newtotal: looptoken = None from rpython.rlib import rgc # a single one is not enough for all tests :-( rgc.collect() rgc.collect() rgc.collect() debug_stop("jit-mem-collect")
def log_bridge(self, inputargs, operations, extra=None, descr=None, ops_offset=None, memo=None): if extra == "noopt": # XXX this case no longer used debug_start("jit-log-noopt-bridge") debug_print("# bridge out of Guard", "0x%x" % compute_unique_id(descr), "with", len(operations), "ops") logops = self._log_operations(inputargs, operations, ops_offset, memo) debug_stop("jit-log-noopt-bridge") elif extra == "rewritten": debug_start("jit-log-rewritten-bridge") debug_print("# bridge out of Guard", "0x%x" % compute_unique_id(descr), "with", len(operations), "ops") logops = self._log_operations(inputargs, operations, ops_offset, memo) debug_stop("jit-log-rewritten-bridge") elif extra == "compiling": debug_start("jit-log-compiling-bridge") logops = self._log_operations(inputargs, operations, ops_offset, memo) debug_stop("jit-log-compiling-bridge") else: debug_start("jit-log-opt-bridge") debug_print("# bridge out of Guard", "0x%x" % r_uint(compute_unique_id(descr)), "with", len(operations), "ops") logops = self._log_operations(inputargs, operations, ops_offset, memo) debug_stop("jit-log-opt-bridge") return logops
def log_loop(self, inputargs, operations, number=0, type=None, ops_offset=None, name='', memo=None): if type is None: # XXX this case not normally used any more, I think debug_start("jit-log-noopt-loop") debug_print("# Loop", number, '(%s)' % name, ":", "noopt", "with", len(operations), "ops") logops = self._log_operations(inputargs, operations, ops_offset, memo) debug_stop("jit-log-noopt-loop") elif type == "rewritten": debug_start("jit-log-rewritten-loop") debug_print("# Loop", number, '(%s)' % name, ":", type, "with", len(operations), "ops") logops = self._log_operations(inputargs, operations, ops_offset, memo) debug_stop("jit-log-rewritten-loop") elif number == -2: debug_start("jit-log-compiling-loop") logops = self._log_operations(inputargs, operations, ops_offset, memo) debug_stop("jit-log-compiling-loop") else: debug_start("jit-log-opt-loop") debug_print("# Loop", number, '(%s)' % name, ":", type, "with", len(operations), "ops") logops = self._log_operations(inputargs, operations, ops_offset, memo) debug_stop("jit-log-opt-loop") return logops
def send_loop_to_backend(greenkey, jitdriver_sd, metainterp_sd, loop, type, orig_inpargs, memo): forget_optimization_info(loop.operations) forget_optimization_info(loop.inputargs) vinfo = jitdriver_sd.virtualizable_info if vinfo is not None: vable = orig_inpargs[jitdriver_sd.index_of_virtualizable].getref_base() patch_new_loop_to_load_virtualizable_fields(loop, jitdriver_sd, vable) original_jitcell_token = loop.original_jitcell_token original_jitcell_token.number = n = metainterp_sd.jitlog.trace_id if not we_are_translated(): show_procedures(metainterp_sd, loop) loop.check_consistency() debug_info = None hooks = None if metainterp_sd.warmrunnerdesc is not None: hooks = metainterp_sd.warmrunnerdesc.hooks if hooks.are_hooks_enabled(): debug_info = JitDebugInfo(jitdriver_sd, metainterp_sd.logger_ops, original_jitcell_token, loop.operations, type, greenkey) hooks.before_compile(debug_info) else: hooks = None operations = get_deep_immutable_oplist(loop.operations) metainterp_sd.profiler.start_backend() debug_start("jit-backend") log = have_debug_prints() or jl.jitlog_enabled() try: loopname = jitdriver_sd.warmstate.get_location_str(greenkey) unique_id = jitdriver_sd.warmstate.get_unique_id(greenkey) asminfo = do_compile_loop(jitdriver_sd.index, unique_id, metainterp_sd, loop.inputargs, operations, original_jitcell_token, name=loopname, log=log, memo=memo) finally: debug_stop("jit-backend") metainterp_sd.profiler.end_backend() if hooks is not None: debug_info.asminfo = asminfo hooks.after_compile(debug_info) metainterp_sd.stats.add_new_loop(loop) if not we_are_translated(): metainterp_sd.stats.compiled() metainterp_sd.log("compiled new " + type) # if asminfo is not None: ops_offset = asminfo.ops_offset else: ops_offset = None metainterp_sd.logger_ops.log_loop(loop.inputargs, loop.operations, n, type, ops_offset, name=loopname) # if metainterp_sd.warmrunnerdesc is not None: # for tests metainterp_sd.warmrunnerdesc.memory_manager.keep_loop_alive( original_jitcell_token)
def jump_to_already_compiled_trace(self, jumpop, patchguardop): jumpop = jumpop.clone() assert jumpop.getopnum() == rop.JUMP cell_token = jumpop.getdescr() assert isinstance(cell_token, JitCellToken) if not cell_token.target_tokens: return False if not self.inline_short_preamble: assert cell_token.target_tokens[0].virtual_state is None jumpop.setdescr(cell_token.target_tokens[0]) self.optimizer.send_extra_operation(jumpop) return True args = jumpop.getarglist() virtual_state = self.get_virtual_state(args) values = [self.getvalue(arg) for arg in jumpop.getarglist()] debug_start('jit-log-virtualstate') virtual_state.debug_print("Looking for ", metainterp_sd=self.optimizer.metainterp_sd) for target in cell_token.target_tokens: if not target.virtual_state: continue extra_guards = [] try: cpu = self.optimizer.cpu state = target.virtual_state.generate_guards(virtual_state, values, cpu) extra_guards = state.extra_guards if extra_guards: debugmsg = 'Guarded to match ' else: debugmsg = 'Matched ' except VirtualStatesCantMatch, e: debugmsg = 'Did not match:\n%s\n' % (e.msg, ) target.virtual_state.debug_print(debugmsg, e.state.bad, metainterp_sd=self.optimizer.metainterp_sd) continue assert patchguardop is not None or (extra_guards == [] and len(target.short_preamble) == 1) target.virtual_state.debug_print(debugmsg, {}) debug_stop('jit-log-virtualstate') args = target.virtual_state.make_inputargs(values, self.optimizer, keyboxes=True) short_inputargs = target.short_preamble[0].getarglist() inliner = Inliner(short_inputargs, args) for guard in extra_guards: if guard.is_guard(): assert isinstance(patchguardop, GuardResOp) assert isinstance(guard, GuardResOp) guard.rd_snapshot = patchguardop.rd_snapshot guard.rd_frame_info_list = patchguardop.rd_frame_info_list guard.setdescr(compile.ResumeAtPositionDescr()) self.optimizer.send_extra_operation(guard) try: # NB: the short_preamble ends with a jump self._inline_short_preamble(target.short_preamble, inliner, patchguardop, target.assumed_classes) except InvalidLoop: #debug_print("Inlining failed unexpectedly", # "jumping to preamble instead") assert cell_token.target_tokens[0].virtual_state is None jumpop.setdescr(cell_token.target_tokens[0]) self.optimizer.send_extra_operation(jumpop) return True
def op_debug_start(category): debug.debug_start(_normalize(category))
def entry_point(argv): debug_start(argv[1]) debug_print(argv[2]) debug_stop(argv[1]) return 0
def get_L2cache_linux2_ia64(): debug_start("gc-hardware") cpu = 0 L2cache = sys.maxint L3cache = sys.maxint while True: cpudir = '/sys/devices/system/cpu/cpu' + assert_str0(str(cpu)) index = 0 while True: cachedir = cpudir + '/cache/index' + assert_str0(str(index)) try: fd = os.open(cachedir + '/level', os.O_RDONLY, 0644) try: level = int(os.read(fd, 4096)[:-1]) finally: os.close(fd) except OSError: break if level not in (2, 3): index += 1 continue try: fd = os.open(cachedir + '/size', os.O_RDONLY, 0644) try: data = os.read(fd, 4096) finally: os.close(fd) except OSError: break end = 0 while '0' <= data[end] <= '9': end += 1 if end == 0: index += 1 continue if data[end] not in ('K', 'k'): # assume kilobytes for now index += 1 continue number = int(data[:end]) number *= 1024 if level == 2: if number < L2cache: L2cache = number if level == 3: if number < L3cache: L3cache = number index += 1 if index == 0: break cpu += 1 mangled = L2cache + L3cache debug_print("L2cache =", mangled) debug_stop("gc-hardware") if mangled > 0: return mangled else: # Print a top-level warning even in non-debug builds llop.debug_print( lltype.Void, "Warning: cannot find your CPU L2 & L3 cache size in " "/sys/devices/system/cpu/cpuX/cache") return -1
def log_short_preamble(self, inputargs, operations, memo=None): debug_start("jit-log-short-preamble") logops = self._log_operations(inputargs, operations, ops_offset=None, memo=memo) debug_stop("jit-log-short-preamble") return logops
def release_all_loops(self): debug_start("jit-mem-releaseall") debug_print("Loop tokens cleared:", len(self.alive_loops)) self.alive_loops.clear() debug_stop("jit-mem-releaseall")
def entry_point(argv): debug_start("mycat") os._exit(0) return 0
def __init__(self, msg='?'): debug_start("jit-abort") debug_print(msg) debug_stop("jit-abort") self.msg = msg
def load_linklet_from_json(file_name, set_version=False): debug_start("loading-linklet") debug_print("Loading linklet from json -- %s" % file_name) linkl, sys_config = W_Linklet.load_linklet(file_name, set_version) debug_stop("loading-linklet") return linkl
def optimize_vector(trace, metainterp_sd, jitdriver_sd, warmstate, loop_info, loop_ops, jitcell_token=None): """ Enter the world of SIMD. Bails if it cannot transform the trace. """ user_code = not jitdriver_sd.vec and warmstate.vec_all e = len(loop_ops) - 1 assert e > 0 assert rop.is_final(loop_ops[e].getopnum()) loop = VectorLoop(loop_info.label_op, loop_ops[:e], loop_ops[-1]) if user_code and user_loop_bail_fast_path(loop, warmstate): return loop_info, loop_ops # the original loop (output of optimize_unroll) info = LoopVersionInfo(loop_info) version = info.snapshot(loop) loop.setup_vectorization() try: debug_start("vec-opt-loop") metainterp_sd.logger_noopt.log_loop([], loop.finaloplist(label=True), -2, None, None, "pre vectorize") metainterp_sd.profiler.count(Counters.OPT_VECTORIZE_TRY) # start = time.clock() opt = VectorizingOptimizer(metainterp_sd, jitdriver_sd, warmstate.vec_cost) oplist = opt.run_optimization(metainterp_sd, info, loop, jitcell_token, user_code) end = time.clock() # metainterp_sd.profiler.count(Counters.OPT_VECTORIZED) metainterp_sd.logger_noopt.log_loop([], loop.finaloplist(label=True), -2, None, None, "post vectorize") nano = int((end - start) * 10.0**9) debug_print("# vecopt factor: %d opcount: (%d -> %d) took %dns" % \ (opt.unroll_count+1, len(version.loop.operations), len(loop.operations), nano)) debug_stop("vec-opt-loop") # info.label_op = loop.label return info, oplist except NotAVectorizeableLoop: debug_stop("vec-opt-loop") # vectorization is not possible return loop_info, version.loop.finaloplist() except NotAProfitableLoop: debug_stop("vec-opt-loop") debug_print( "failed to vectorize loop, cost model indicated it is not profitable" ) # cost model says to skip this loop return loop_info, version.loop.finaloplist() except Exception as e: debug_stop("vec-opt-loop") debug_print("failed to vectorize loop. THIS IS A FATAL ERROR!") if we_are_translated(): llop.debug_print_traceback(lltype.Void) else: raise finally: loop.teardown_vectorization() return loop_info, loop_ops
def semispace_collect(self, size_changing=False): debug_start("gc-collect") debug_print() debug_print(".----------- Full collection ------------------") start_usage = self.free - self.tospace debug_print("| used before collection: ", start_usage, "bytes") #start_time = time.time() #llop.debug_print(lltype.Void, 'semispace_collect', int(size_changing)) # Switch the spaces. We copy everything over to the empty space # (self.fromspace at the beginning of the collection), and clear the old # one (self.tospace at the beginning). Their purposes will be reversed # for the next collection. tospace = self.fromspace fromspace = self.tospace self.fromspace = fromspace self.tospace = tospace self.top_of_space = tospace + self.space_size scan = self.free = tospace self.starting_full_collect() self.collect_roots() self.copy_pending_finalizers(self.copy) scan = self.scan_copied(scan) if self.objects_with_light_finalizers.non_empty(): self.deal_with_objects_with_light_finalizers() if self.objects_with_finalizers.non_empty(): scan = self.deal_with_objects_with_finalizers(scan) if self.objects_with_weakrefs.non_empty(): self.invalidate_weakrefs() self.update_objects_with_id() self.finished_full_collect() self.debug_check_consistency() if not size_changing: llarena.arena_reset(fromspace, self.space_size, True) self.record_red_zone() self.execute_finalizers() #llop.debug_print(lltype.Void, 'collected', self.space_size, size_changing, self.top_of_space - self.free) if have_debug_prints(): #end_time = time.time() #elapsed_time = end_time - start_time #self.total_collection_time += elapsed_time self.total_collection_count += 1 #total_program_time = end_time - self.program_start_time end_usage = self.free - self.tospace debug_print("| used after collection: ", end_usage, "bytes") debug_print("| freed: ", start_usage - end_usage, "bytes") debug_print("| size of each semispace: ", self.space_size, "bytes") debug_print("| fraction of semispace now used: ", end_usage * 100.0 / self.space_size, "%") #ct = self.total_collection_time cc = self.total_collection_count debug_print("| number of semispace_collects: ", cc) #debug_print("| i.e.: ", # cc / total_program_time, "per second") #debug_print("| total time in semispace_collect: ", # ct, "seconds") #debug_print("| i.e.: ", # ct * 100.0 / total_program_time, "%") debug_print("`----------------------------------------------") debug_stop("gc-collect")
def print_stats(self): debug_start("jit-summary") if have_debug_prints(): self._print_stats() debug_stop("jit-summary")
def op_debug_start(category, timestamp): return debug.debug_start(_normalize(category), timestamp)
def debug_bridge(descr_number, rawstart, codeendpos): debug_start("jit-backend-addr") debug_print("bridge out of Guard 0x%x has address 0x%x to 0x%x" % (r_uint(descr_number), r_uint(rawstart), r_uint(rawstart + codeendpos))) debug_stop("jit-backend-addr")