def reallocd(self, event, begin, begin_new, end_new): overlap = self._addr_ivals[begin] end = overlap.end if overlap.state is not None else min( overlap.end, begin + 4) ival_freed = AddrIval(begin, end, FREED) if overlap.state in (None, FREED): inew_alloc = AddrIval(begin_new, end_new, ALLOCD) self.allocd(event, inew_alloc.begin, inew_alloc.end) logger.info( '%d\tI: replaced unmatched realloc(%x, %d) with alloc(%d)', run.timestamp_ns, begin, inew_alloc.size, inew_alloc.size) return if overlap.state is ALLOCD and ival_freed.begin != overlap.begin: inew_alloc = AddrIval(begin_new, end_new, ALLOCD) inew_allocd = self._shrink_allocd(event, overlap, ival_freed.begin) self.allocd(event, inew_alloc.begin, inew_alloc.end) logger.info( '%d\tI: replaced mis-realloc(%x, %d) with alloc(%d)' ' and free+alloc %s', run.timestamp_ns, begin, inew_alloc.size, inew_alloc.size, inew_allocd) return self._addr_ivals.add(ival_freed) self._allocd(event, begin_new, end_new, caller='realloc') trace.reallocd(None, event, begin, begin_new, end_new)
def reused(self, alloc_state, event, begin, end): # Query for colour intervals in the range. # Trim result so that no colour interval expands beyond the range being reused. # Calculate the range's new colour. overlaps_c = self._addr_ivals_c[begin:end] if overlaps_c[0].begin < begin: overlaps_c[0] = AddrIval(begin, overlaps_c[0].end, overlaps_c[0].value) if overlaps_c[-1].end > end: overlaps_c[-1] = AddrIval(overlaps_c[-1].begin, end, overlaps_c[-1].value) ival_colour = (max(ic.value for ic in overlaps_c) + 1) % self._ccount # Fall back to the sweeping revoker if any colour wrapped around due to # being maximal. # Reset colours after the sweep. if ival_colour == 0: if args.exit_on_colour_reuse: ivals_max_colour = [i for i in overlaps_c if i.value == self._ccount-1] logger.critical("%d\tCrit: New allocation %s re-uses %s more than %d times (colours), " "exiting as instructed by --exit-on-colour-reuse", run.timestamp, AddrIval(begin, end, AddrIvalState.ALLOCD), ivals_max_colour, self._ccount) sys.exit(1) ivals_revoked = self._sweeping_revoker.reused(alloc_state, event, begin, end) self._reset_addr_ivals_colours(ivals_revoked) return ivals_revoked # No colour wrapped around, commit the interval's new colour. self._addr_ivals_c.add(AddrIval(begin, end, ival_colour)) return []
def __init__(self, tslam=None, cliargs=[], **kwargs): super().__init__() self._ts = tslam self._aa = IntervalTree() self._aa.add(AddrIval(0, 2**64, AState.REVOKED)) self._am = IntervalTree() self._am.add(AddrIval(0, 2**64, AState.UNMAPD)) self._ls = {} # Argument parsing ---------------------------------------------------- {{{ argp = argparse.ArgumentParser() argp.add_argument('--fix', action='store_const', const=True, default=False, help="Automatically insert fixups for reports") argp.add_argument('--skip-map', action='store_const', const=True, default=False, help="Ignore map/unmap constraints") argp.add_argument('--drop-safe', action='store_const', const=True, default=False, help="Suppress warnings for safely dropped events") self._arg = argp.parse_args(cliargs)
def reallocd(self, event, begin_old, begin_new, end_new): interval_old = self.addr_ival(begin_old) if not interval_old: logger.warning('%d\tW: No existing allocation to realloc at %x, doing just alloc', run.timestamp, begin_old) self.allocd(event, begin_new, end_new) return if interval_old.state is not AddrIvalState.ALLOCD: logger.warning('%d\tW: Realloc of non-allocated interval %s, assuming it is allocated', run.timestamp, interval_old) # If the realloc is in place, just remove the old allocation. It is not # right to mark it as freed, because if the freed part is then reused, # an invalid revocation that is not aligned with the start of the # reallocated part is generated. This event is reported as an error # instead (see allocd()), since it is allocator behaviour that cannot be # made safe. interval_new = AddrIval(begin_new, end_new, AddrIvalState.ALLOCD) if interval_new.begin == interval_old.begin: super()._update(AddrIval(interval_old.begin, interval_old.end, None)) self.__addr_ivals.remove(interval_old) if interval_new.size < interval_old.size: ival_old_stub = AddrIval(interval_new.end, interval_old.end, AddrIvalState.FREED) self._realloc_stubs.add(ival_old_stub) else: # XXX use _freed and eliminate spurious W/E reporting self.freed(event, begin_old) self.allocd(event, begin_new, end_new)
def revoked(self, stk, tid, *bes): if not isinstance(bes[0], tuple): bes = [(bes[0], bes[1])] query_and_overlaps = [((b, e), self.addr_ivals_sorted(b, e)) for b, e in bes] overlaps_allocd = [ i for i in itertools.chain(*(overlaps for _, overlaps in query_and_overlaps)) if i.state is AddrIvalState.ALLOCD ] if overlaps_allocd: raise ValueError( 'Bug: revoking address intervals that are still allocated', run.timestamp, overlaps_allocd) misrevokes = [[ i for i in overlaps if i.state is AddrIvalState.FREED and not (b <= i.begin and i.end <= e) ] for (b, e), overlaps in query_and_overlaps] if any(misrevokes): query_and_misrevokes = [ (AddrIval(b, e, AddrIvalState.REVOKED), mis) for ((b, e), _), mis in zip(query_and_overlaps, misrevokes) if mis ] raise ValueError( 'Bug (incomplete revocation): revoking intervals that do not fully capture ' 'the underlying freed allocations', run.timestamp, query_and_misrevokes) for begin, end in bes: ival = AddrIval(begin, end, AddrIvalState.REVOKED) super()._update(ival) self.__addr_ivals.add(ival) self._realloc_stubs.remove(ival)
def _shrink_allocd(self, event, ival_allocd, end_new): inew_freed = AddrIval(ival_allocd.begin, ival_allocd.end, FREED) inew_allocd = AddrIval(ival_allocd.begin, end_new, ALLOCD) self._addr_ivals.add(inew_freed) self._addr_ivals.add(inew_allocd) trace.freed(None, event, inew_freed.begin) trace.allocd(None, event, inew_allocd.begin, inew_allocd.end) return inew_allocd
def freed(self, event, begin): interval = self.addr_ival(begin) if interval: if begin != interval.begin or interval.state is not AddrIvalState.ALLOCD: logger.warning('%d\tW: Freed(%x) misfrees %s', run.timestamp, begin, interval) interval = AddrIval(interval.begin, interval.end, AddrIvalState.FREED) else: logger.warning('%d\tW: No existing allocation to free at %x, defaulting to one of size 1', run.timestamp, begin) interval = AddrIval(begin, begin + 1, AddrIvalState.FREED) super()._update(interval) self.__addr_ivals.add(interval)
def revoked(self, event, *bes): if not isinstance(bes[0], tuple): bes = [(bes[0], bes[1])] query_and_overlaps = [((b, e), self.addr_ivals_sorted(b, e)) for b, e in bes] overlaps_allocd = [i for i in itertools.chain(*(overlaps for _, overlaps in query_and_overlaps)) if i.state is AddrIvalState.ALLOCD] if overlaps_allocd: raise ValueError('Bug: revoking address intervals that are still allocated', run.timestamp, overlaps_allocd) spans_to_mark_revoked = list(bes) misrevoked = [[i for i in overlaps if i.state is AddrIvalState.FREED and not (b <= i.begin and i.end <= e)] for (b, e), overlaps in query_and_overlaps] if any(misrevoked): # When the exit_on_reuse mechanism is being used, # allow misrevokes and rather disallow later reuse of any part of # incompletely revoked FREED regions via the exit_on_reuse mechanism. # Shrink the spans to avoid overlapping any part of the misrevoked # interval. if args.exit_on_reuse or args.exit_on_colour_reuse: _adjusted = [] for i, mis in ((i, m) for i, m in enumerate(misrevoked) if m): b, e = b_new, e_new = bes[i] # [ FREED ] # [ REVOKE --> if mis[0].begin < b: b_new = mis[0].end # [ FREED ] # <-- REVOKE ] if e < mis[-1].end: e_new = mis[-1].begin if b_new < e_new: spans_to_mark_revoked[i] = b_new, e_new ival_new = AddrIval(b_new, e_new, AddrIvalState.REVOKED) # [ FREED ] # [ REVOKE ] X # [ REVOKE ] X else: spans_to_mark_revoked[i] = None ival_new = None _adjusted.append(((AddrIval(b, e, AddrIvalState.REVOKED), ival_new), mis)) logger.warning('%d\tW: Adjusted intervals to revoke %s', run.timestamp, _adjusted) else: query_and_misrevokes = [(AddrIval(b, e, AddrIvalState.REVOKED), mis) for ((b, e), _), mis in zip(query_and_overlaps, misrevoked) if mis] raise ValueError('Bug (incomplete revocation): revoking intervals that do not fully capture ' 'the underlying freed allocations', run.timestamp, query_and_misrevokes) for begin, end in (s for s in spans_to_mark_revoked if s is not None): ival = AddrIval(begin, end, AddrIvalState.REVOKED) super()._update(ival) self.__addr_ivals.add(ival) self._realloc_stubs.remove(ival)
def _allocd(self, event, begin, end, *, caller='alloc'): ival = AddrIval(begin, end, ALLOCD) overlaps = self._addr_ivals[begin:end] overlaps_allocd = [o for o in overlaps if o.state is ALLOCD] if overlaps_allocd: for o in overlaps_allocd: inew = AddrIval(o.begin, o.end, FREED) self._addr_ivals.add(inew) trace.freed(None, event, inew.begin) logger.info('%d\tI: inserted %d frees before overlapping %s %s', run.timestamp_ns, len(overlaps_allocd), caller, ival) self._addr_ivals.add(ival)
def allocd(self, stk, tid, begin, end): interval = AddrIval(begin, end, AddrIvalState.ALLOCD) overlaps = self.addr_ivals_sorted(begin, end) overlaps_allocd = [ o for o in overlaps if o.state is AddrIvalState.ALLOCD ] overlaps_freed = [ o for o in overlaps if o.state is AddrIvalState.FREED ] overlaps_stubs = [ o for o in self._realloc_stubs[begin:end] if o.state is AddrIvalState.FREED ] if overlaps_allocd: oa = overlaps_allocd logger.warning( '%d\tW: New allocation %s overlaps existing allocations %s, chopping them out', run.timestamp, interval, overlaps_allocd) for o in oa: self.__addr_ivals.remove(o) if oa[0].contains_point(interval.begin): left_stub = AddrIval(oa[0].begin, interval.begin, AddrIvalState.ALLOCD) self.__addr_ivals.add(left_stub) if oa[-1].contains_point(interval.end): right_stub = AddrIval(interval.end, oa[-1].end, AddrIvalState.ALLOCD) self.__addr_ivals.add(right_stub) if overlaps_stubs: err_fmt = '%d\t%s: New allocation %s reuses old allocation stub from realloc '\ '(invalid revocation ahead)' if args.exit_on_unsafe: err_fmt += ', exiting as instructed by --exit-on-unsafe' logger.critical(err_fmt, run.timestamp, 'Crit', interval) sys.exit(1) else: logger.error(err_fmt, run.timestamp, 'E', interval) if overlaps_freed: if args.exit_on_reuse: logger.critical( "%d\tCrit: New allocation %s re-uses %s, exiting as instructed " "by --exit-on-reuse", run.timestamp, interval, overlaps_freed) sys.exit(1) self._publish('reused', stk, tid, begin, end) super()._update(interval) self.__addr_ivals.add(interval)
def _allocd(self, begin, end): overlaps_a = self._aa[begin:end] overlaps_m = self._am[begin:end] if not self._arg.skip_map: overlaps_unmapped = [ o for o in overlaps_m if o.state == AState.UNMAPD ] if overlaps_unmapped: logging.warning("Allocation ts=%d b=%x e=%x overlaps unmap=%r", self._ts(), begin, end, overlaps_unmapped) # XXX fix by mapping pages overlaps_allocated = [ o for o in overlaps_a if o.state == AState.ALLOCD ] if overlaps_allocated: logging.error("Allocation ts=%d b=%x e=%x overlaps alloc=%r", self._ts(), begin, end, overlaps_allocated) if self._arg.fix: for oa in overlaps_allocated: self._publish('free', '', oa.begin) self._aa.chop(begin, end) self._aa.add(AddrIval(begin, end, AState.ALLOCD))
def _freed(self, addr): doalloc = False end = addr + 1 # Will be fixed up later overlaps_a = self._aa[addr:end] overlaps_m = self._am[addr:end] if not self._arg.skip_map: overlaps_unmapped = [ o for o in overlaps_m if o.state == AState.UNMAPD ] if overlaps_unmapped: logging.error("Free ts=%d a=%x overlaps unmap=%r", self._ts(), addr, overlaps_unmapped) allocations = [o for o in overlaps_a if o.state == AState.ALLOCD] overlaps_free = [o for o in overlaps_a if o.state == AState.FREED] if overlaps_free != []: logging.warning("Free ts=%d a=%x overlaps free=%r", self._ts(), addr, overlaps_free) if allocations == [] and len( overlaps_free) == 1 and self._arg.drop_safe: return False else: for of in overlaps_free: if of.begin <= addr: end = max(end, of.end) if self._arg.fix: doalloc = True if len(allocations) > 1 or (allocations != [] and overlaps_free != []): logging.error("Free ts=%d a=%x multiply-attested alloc=%r free=%r", self._ts(), addr, allocations, overlaps_free) elif allocations == [] and overlaps_free == []: logging.warning("Free ts=%d a=%x no corresponding alloc", self._ts(), addr) if self._arg.fix and not self._arg.drop_safe: doalloc = True else: assert doalloc == False return False else: for a in allocations: if a.begin != addr: # Likely to leave cruft behind, indicative of serious errors logging.error("Free ts=%d a=%x within alloc=%r", self._ts(), addr, a) else: end = max(end, a.end) self._aa.chop(addr, end) self._aa.add(AddrIval(addr, end, AState.FREED)) if doalloc: self._publish('allocd', '', addr, end) return True
def _get_memory_pools(): ivals = [i for i in alloc_state.addr_ivals_coalesced_sorted() if i.state is not AddrIvalState.UNMAPD] ivals.reverse() pools = [] while ivals: pool = [] pool.append(ivals.pop()) while ivals and (ivals[-1].begin - pool[-1].end < AllocationMapOutput.POOL_MAX_ARTIFICIAL_GROWTH): assert pool[-1].end <= ivals[-1].begin, 'Bug: overlapping intervals {0} {1}'\ .format(pool[-1], ivals[-1]) pool.append(ivals.pop()) pools.append(AddrIval(pool[0].begin, pool[-1].end, None)) return pools
def freed(self, event, begin): overlap = self._addr_ivals[begin] if overlap.state in (None, FREED): logger.info('%d\tI: dropped unmatched free(%x)', run.timestamp_ns, begin) return ival = AddrIval(begin, overlap.end, FREED) if overlap.state is ALLOCD and ival.begin != overlap.begin: inew_allocd = self._shrink_allocd(event, overlap, ival.begin) logger.info('%d\tI: replaced mis-free(%x) with free+alloc %s', run.timestamp_ns, begin, inew_allocd) return self._addr_ivals.add(ival) trace.freed(None, event, ival.begin)
def revoked(self, event, *begs_ends): if not isinstance(begs_ends[0], tuple): begs_ends = [(begs_ends[0], begs_ends[1])] query_and_overlaps = [((b, e), self._addr_ivals_c[b:e]) for b, e in begs_ends] # Note: this code is similar to AllocatedAddrSpaceModel.revoked() spans_to_mark_revoked = list(begs_ends) misrevoked = [[i for i in overlaps if i.value and not (b <= i.begin and i.end <= e)] for (b, e), overlaps in query_and_overlaps] if any(misrevoked): # ColouringRevoker should not clear the colour of misrevokes, # to correctly count later reuse of incompletely revoked FREED # intervals via the exit_on_colour_reuse mechanism. for i, mis in ((i, m) for i, m in enumerate(misrevoked) if m): b, e = b_new, e_new = begs_ends[i] if mis[0].begin < b: b_new = mis[0].end if e < mis[-1].end: e_new = mis[-1].begin if b_new < e_new: spans_to_mark_revoked[i] = b_new, e_new else: spans_to_mark_revoked[i] = None # Filter out any None span introduced i = 0; li = len(spans_to_mark_revoked) - 1 while i < li: if spans_to_mark_revoked[i] is None: spans_to_mark_revoked[i] = spans_to_mark_revoked.pop() li -= 1 else: i += 1 if spans_to_mark_revoked[-1] is None: spans_to_mark_revoked.pop() self._sweeping_revoker.revoked(event, *spans_to_mark_revoked) self._reset_addr_ivals_colours(AddrIval(b, e, AddrIvalState.REVOKED) for b, e in spans_to_mark_revoked)
def __init__(self, *, calc_total_for_state): super().__init__() self.__addr_ivals = IntervalMap.from_valued_interval_domain(AddrIval(0, 2**64, None)) self._total = 0 self._calc_total_for_state = calc_total_for_state
def reused(self, alloc_state, event, begin, end): self._addr_ivals_reused.add(AddrIval(begin, end, None))
def mapd(self, _, __, begin, end, ___): self._update(AddrIval(begin, end, AddrIvalState.MAPD))
def __init__(self, colour_count, sweeping_revoker): self._sweeping_revoker = sweeping_revoker self._ccount = colour_count self._addr_ivals_c = IntervalMap.from_valued_interval_domain(AddrIval(0, 2**64, 0))
def revoked(self, event, *bes): self._sweep(addr_space.sweep_size, [AddrIval(b, e, AddrIvalState.FREED) for b, e in bes])
def mapd(self, event, begin, end, prot): if prot == 0b11 and AMAS.call_is_from_allocator(event['callstack']): self._update(AddrIval(begin, end, AddrIvalState.MAPD))
def unmapd(self, event, begin, end): self._update(AddrIval(begin, end, AddrIvalState.UNMAPD))
def mapd(self, event, begin, end, _): self._update(AddrIval(begin, end, AddrIvalState.MAPD))
def mapd(self, callstack, tid, begin, end, prot): if prot == 0b11 and AMAS.call_is_from_allocator(callstack): self._update(AddrIval(begin, end, AddrIvalState.MAPD))
def revoked(self, stk, tid, *begs_ends): if not isinstance(begs_ends[0], tuple): begs_ends = [(begs_ends[0], begs_ends[1])] self._reset_addr_ivals_colours( AddrIval(b, e, AddrIvalState.REVOKED) for b, e in begs_ends) self._sweeping_revoker.revoked(stk, tid, *begs_ends)
def __init__(self): bkg_ival = AddrIval(0, 2**64, None) self._addr_ivals = \ IntervalMap.from_valued_interval_domain(bkg_ival, coalescing=False)
def __init__(self): super().__init__(calc_total_for_state=AddrIvalState.ALLOCD) bkg_ival = AddrIval(0, 2**64, None) self.__addr_ivals = IntervalMap.from_valued_interval_domain(bkg_ival, coalescing=False) self._realloc_stubs = IntervalMap.from_valued_interval_domain(bkg_ival)