def _call_assembler_patch_je(self, result_loc, jmp_location):
pos = self.mc.currpos()
self.mc.BRK()
#
pmc = OverwritingBuilder(self.mc, jmp_location, WORD)
pmc.B_ofs_cond(self.mc.currpos() - jmp_location, c.EQ)
return pos
def _emit_op_cond_call(self, op, arglocs, fcond):
if len(arglocs) == 2:
res_loc = arglocs[1] # cond_call_value
else:
res_loc = None # cond_call
# see x86.regalloc for why we skip res_loc in the gcmap
if arglocs[0] is not None: # otherwise result already in CC
self.mc.CMP_ri(arglocs[0].value, 0)
gcmap = self._regalloc.get_gcmap([res_loc])
jmp_adr = self.mc.currpos()
self.mc.BRK() # patched later: the conditional jump
#
self.push_gcmap(self.mc, gcmap)
self.mc.gen_load_int(r.ip1.value,
rffi.cast(lltype.Signed,
op.getarg(1).getint()))
#
callee_only = False
floats = False
if self._regalloc is not None:
for reg in self._regalloc.rm.reg_bindings.values():
if reg not in self._regalloc.rm.save_around_call_regs:
break
else:
callee_only = True
if self._regalloc.vfprm.reg_bindings:
floats = True
cond_call_adr = self.cond_call_slowpath[floats * 2 + callee_only]
assert cond_call_adr
self.mc.BL(cond_call_adr)
# if this is a COND_CALL_VALUE, we need to move the result in place
# from its current location
if res_loc is not None:
self.mc.MOV_rr(res_loc.value, r.ip1.value)
#
self.pop_gcmap(self.mc)
pmc = OverwritingBuilder(self.mc, jmp_adr, WORD)
pmc.B_ofs_cond(self.mc.currpos() - jmp_adr, fcond)
# might be overridden again to skip over the following
# guard_no_exception too
self.previous_cond_call_jcond = jmp_adr, fcond
def move_real_result_and_call_reacqgil_addr(self, fastgil):
# try to reacquire the lock. The following two values are saved
# across the call and are still alive now:
RTHREADID = r.x19 # our thread ident
RSHADOWOLD = r.x20 # old value of the shadowstack pointer
RPYFASTGIL = r.ip2 # &rpy_fastgil, loaded now:
self.mc.gen_load_int(RPYFASTGIL.value, fastgil)
# this comes from gcc compiling this code:
# __sync_bool_compare_and_swap(&rpy_fastgil, old=0, new=RTHREADID);
self.mc.LDXR(r.x1.value, RPYFASTGIL.value)
self.mc.CBNZ(r.x1.value, +12)
self.mc.STLXR(r.x3.value, RTHREADID.value, RPYFASTGIL.value)
self.mc.CBNZ_w(r.x3.value, -12)
self.mc.DMB_ISH()
# now x1 is the old value of the lock, and if x1 == 0 then the lock
# now contains RTHREADID
b1_location = self.mc.currpos()
self.mc.BRK() # boehm: patched with a CBZ (jump if x1 == 0)
# shadowstack: patched with CBNZ instead
gcrootmap = self.asm.cpu.gc_ll_descr.gcrootmap
if gcrootmap:
# When doing a call_release_gil with shadowstack, there
# is the risk that the 'rpy_fastgil' was free but the
# current shadowstack can be the one of a different
# thread. So here we check if the shadowstack pointer
# is still the same as before we released the GIL (saved
# in 'x20'), and if not, we fall back to 'reacqgil_addr'.
rst = gcrootmap.get_root_stack_top_addr()
self.mc.gen_load_int(r.ip1.value, rst)
self.mc.LDR_ri(r.ip0.value, r.ip1.value, 0) # new shadowstack
self.mc.CMP_rr(r.ip0.value, RSHADOWOLD.value)
b3_location = self.mc.currpos()
self.mc.BRK() # B.eq forward
# revert the rpy_fastgil acquired above, so that the
# general 'reacqgil_addr' below can acquire it again...
self.mc.STR_ri(r.xzr.value, RPYFASTGIL.value, 0)
# patch the b1_location above, with "CBNZ here"
pmc = OverwritingBuilder(self.mc, b1_location, WORD)
pmc.CBNZ(r.x1.value, self.mc.currpos() - b1_location)
open_location = b3_location
else:
open_location = b1_location
# Yes, we need to call the reacqgil() function.
# save the result we just got
RSAVEDRES = RTHREADID # can reuse this reg here to save things
reg = self.resloc
if reg is not None:
if reg.is_core_reg():
self.mc.MOV_rr(RSAVEDRES.value, reg.value)
elif reg.is_vfp_reg():
self.mc.SUB_ri(r.sp.value, r.sp.value, 2 * WORD)
self.mc.STR_di(reg.value, r.sp.value, 0)
# call the function
self.mc.BL(self.asm.reacqgil_addr)
# restore the saved register
if reg is not None:
if reg.is_core_reg():
self.mc.MOV_rr(reg.value, RSAVEDRES.value)
elif reg.is_vfp_reg():
self.mc.LDR_di(reg.value, r.sp.value, 0)
self.mc.ADD_ri(r.sp.value, r.sp.value, 2 * WORD)
# now patch the still-open jump above:
# boehm: patch b1_location with a CBZ(x1)
# shadowstack: patch b3_location with BEQ
pmc = OverwritingBuilder(self.mc, open_location, WORD)
offset = self.mc.currpos() - open_location
if gcrootmap:
pmc.B_ofs_cond(offset, c.EQ)
else:
pmc.CBZ(r.x1.value, offset)
if not we_are_translated(): # for testing: now we can accesss
self.mc.SUB_ri(r.fp.value, r.fp.value, 1) # fp again
def _write_barrier_fastpath(self, mc, descr, arglocs, array=False, is_frame=False):
# Write code equivalent to write_barrier() in the GC: it checks
# a flag in the object at arglocs[0], and if set, it calls a
# helper piece of assembler. The latter saves registers as needed
# and call the function remember_young_pointer() from the GC.
if we_are_translated():
cls = self.cpu.gc_ll_descr.has_write_barrier_class()
assert cls is not None and isinstance(descr, cls)
#
card_marking = False
mask = descr.jit_wb_if_flag_singlebyte
if array and descr.jit_wb_cards_set != 0:
# assumptions the rest of the function depends on:
assert (descr.jit_wb_cards_set_byteofs ==
descr.jit_wb_if_flag_byteofs)
assert descr.jit_wb_cards_set_singlebyte == -0x80
card_marking = True
mask = descr.jit_wb_if_flag_singlebyte | -0x80
#
loc_base = arglocs[0]
if is_frame:
assert loc_base is r.fp
mc.LDRB_ri(r.ip0.value, loc_base.value, descr.jit_wb_if_flag_byteofs)
mask &= 0xFF
mc.MOVZ_r_u16(r.ip1.value, mask, 0)
mc.TST_rr_shift(r.ip0.value, r.ip1.value, 0)
jz_location = mc.currpos()
mc.BRK()
# for cond_call_gc_wb_array, also add another fast path:
# if GCFLAG_CARDS_SET, then we can just set one bit and be done
if card_marking:
mc.MOVZ_r_u16(r.ip1.value, 0x80, 0)
# GCFLAG_CARDS_SET is in this byte at 0x80
mc.TST_rr_shift(r.ip0.value, r.ip1.value, 0)
js_location = mc.currpos()
mc.BRK()
else:
js_location = 0
# Write only a CALL to the helper prepared in advance, passing it as
# argument the address of the structure we are writing into
# (the first argument to COND_CALL_GC_WB).
helper_num = card_marking
if is_frame:
helper_num = 4
elif self._regalloc is not None and self._regalloc.vfprm.reg_bindings:
helper_num += 2
if self.wb_slowpath[helper_num] == 0: # tests only
assert not we_are_translated()
self.cpu.gc_ll_descr.write_barrier_descr = descr
self._build_wb_slowpath(card_marking,
bool(self._regalloc.vfprm.reg_bindings))
assert self.wb_slowpath[helper_num] != 0
#
if loc_base is not r.x0:
# push two registers to keep stack aligned
mc.SUB_ri(r.sp.value, r.sp.value, 2 * WORD)
mc.STR_ri(r.x0.value, r.sp.value, WORD)
mc.STR_ri(loc_base.value, r.sp.value, 0)
mc.MOV_rr(r.x0.value, loc_base.value)
if is_frame:
assert loc_base is r.fp
mc.BL(self.wb_slowpath[helper_num])
if loc_base is not r.x0:
mc.LDR_ri(r.x0.value, r.sp.value, WORD)
mc.LDR_ri(loc_base.value, r.sp.value, 0)
mc.ADD_ri(r.sp.value, r.sp.value, 2 * WORD)
if card_marking:
# The helper ends again with a check of the flag in the object. So
# here, we can simply write again a conditional jump, which will be
# taken if GCFLAG_CARDS_SET is still not set.
jns_location = mc.currpos()
mc.BRK()
#
# patch the JS above
offset = mc.currpos() - js_location
pmc = OverwritingBuilder(mc, js_location, WORD)
pmc.B_ofs_cond(offset, c.NE) # We want to jump if the z flag isn't set
#
# case GCFLAG_CARDS_SET: emit a few instructions to do
# directly the card flag setting
loc_index = arglocs[1]
assert loc_index.is_core_reg()
tmp1 = r.ip1
tmp2 = arglocs[-1] # the last item is a preallocated tmp
# lr = byteofs
s = 3 + descr.jit_wb_card_page_shift
mc.MVN_rr_shifted(r.lr.value, loc_index.value, s, shifttype=shift.LSR)
# tmp1 = byte_index
mc.MOVZ_r_u16(r.ip0.value, 7, 0)
mc.AND_rr_shift(tmp1.value, r.ip0.value, loc_index.value,
descr.jit_wb_card_page_shift, shifttype=shift.LSR)
# set the bit
mc.MOVZ_r_u16(tmp2.value, 1, 0)
mc.LDRB_rr(r.ip0.value, loc_base.value, r.lr.value)
mc.LSL_rr(tmp2.value, tmp2.value, tmp1.value)
mc.ORR_rr(r.ip0.value, r.ip0.value, tmp2.value)
mc.STR_size_rr(0, r.ip0.value, loc_base.value, r.lr.value)
# done
#
# patch the JNS above
offset = mc.currpos() - jns_location
pmc = OverwritingBuilder(mc, jns_location, WORD)
pmc.B_ofs_cond(offset, c.EQ) # We want to jump if the z flag is set
offset = mc.currpos() - jz_location
pmc = OverwritingBuilder(mc, jz_location, WORD)
pmc.B_ofs_cond(offset, c.EQ)
def _call_assembler_patch_jmp(self, jmp_location):
# merge point
currpos = self.mc.currpos()
pmc = OverwritingBuilder(self.mc, jmp_location, WORD)
pmc.B_ofs(currpos - jmp_location)