def myrepr(self, referrer, refersTo):
pre = ''
if isinstance(referrer, dict):
for k, v in referrer.items():
if v is refersTo:
pre = self.truncateAtNewLine(fastRepr(k)) + ']-> '
break
elif isinstance(referrer, (list, tuple)):
for x, ref in enumerate(referrer):
if ref is refersTo:
pre = '%s]-> ' % (x)
break
if isinstance(refersTo, dict):
post = 'dict['
elif isinstance(refersTo, list):
post = 'list['
elif isinstance(refersTo, tuple):
post = 'tuple['
elif isinstance(refersTo, set):
post = 'set->'
else:
post = self.truncateAtNewLine(fastRepr(refersTo)) + "-> "
return '%s%s' % (pre, post)
def finished(self):
print('RefPath(%s): Finished ReferrerSearch for %s' %
(self._id, fastRepr(self.obj)))
self.obj = None
safeReprNotify = _getSafeReprNotify()
safeReprNotify.setInfo(self.info)
def _varDump__print(exc):
global sReentry
global notify
if sReentry > 0:
return
sReentry += 1
if not exc._savedExcString:
s = ''
foundRun = False
for frame in reversed(exc._savedStackFrames):
filename = frame.f_code.co_filename
codename = frame.f_code.co_name
if not foundRun and codename != 'run':
# don't print stack frames before run(),
# they contain builtins and are huge
continue
foundRun = True
s += '\nlocals for %s:%s\n' % (filename, codename)
locals = frame.f_locals
for var in locals:
obj = locals[var]
rep = fastRepr(obj)
s += '::%s = %s\n' % (var, rep)
exc._savedExcString = s
exc._savedStackFrames = None
notify.info(exc._savedExcString)
sReentry -= 1
def _varDump__print(exc):
global sReentry
global notify
if sReentry > 0:
return
sReentry += 1
if not exc._savedExcString:
s = ''
foundRun = False
for frame in reversed(exc._savedStackFrames):
filename = frame.f_code.co_filename
codename = frame.f_code.co_name
if not foundRun and codename != 'run':
# don't print stack frames before run(),
# they contain builtins and are huge
continue
foundRun = True
s += '\nlocals for %s:%s\n' % (filename, codename)
locals = frame.f_locals
for var in locals:
obj = locals[var]
rep = fastRepr(obj)
s += '::%s = %s\n' % (var, rep)
exc._savedExcString = s
exc._savedStackFrames = None
notify.info(exc._savedExcString)
sReentry -= 1
def isManyRef(self, at, path, referrers):
if (len(referrers) > self.maxRefs and \
at is not self.obj):
if not isinstance(at, (list, tuple, dict, set)):
sys.stdout.write("RefPath(%s): ManyRefs(%s)[%s]-> " %
(self._id, len(referrers), fastRepr(at)))
path = list(reversed(path))
path.insert(0, 0)
for x in range(len(path) - 1):
sys.stdout.write(self.myrepr(path[x], path[x + 1]))
print("")
return True
else:
sys.stdout.write("RefPath(%s): ManyRefsAllowed(%s)[%s]-> " %
(self._id, len(referrers),
fastRepr(at, maxLen=1, strFactor=30)))
print("")
return False
def run(self):
safeReprNotify = _getSafeReprNotify()
self.info = safeReprNotify.getInfo()
safeReprNotify.setInfo(0)
print('RefPath(%s): Beginning ReferrerSearch for %s' %
(self._id, fastRepr(self.obj)))
self.visited = set()
for x in self.stepGenerator(0, [self.obj]):
yield None
yield Job.Done
def _excepthookDumpVars(eType, eValue, tb):
origTb = tb
excStrs = traceback.format_exception(eType, eValue, origTb)
s = 'printing traceback in case variable repr crashes the process...\n'
for excStr in excStrs:
s += excStr
notify.info(s)
s = 'DUMPING STACK FRAME VARIABLES'
#import pdb;pdb.set_trace()
#foundRun = False
foundRun = True
while tb is not None:
frame = tb.tb_frame
code = frame.f_code
# this is a list of every string identifier used in this stack frame's code
codeNames = set(code.co_names)
# skip everything before the 'run' method, those frames have lots of
# not-useful information
if not foundRun:
if code.co_name == 'run':
foundRun = True
else:
tb = tb.tb_next
continue
s += '\n File "%s", line %s, in %s' % (
code.co_filename, frame.f_lineno, code.co_name)
stateStack = Stack()
# prime the stack with the variables we should visit from the frame's data structures
# grab all of the local, builtin and global variables that appear in the code's name list
name2obj = {}
for name, obj in frame.f_builtins.items():
if name in codeNames:
name2obj[name] = obj
for name, obj in frame.f_globals.items():
if name in codeNames:
name2obj[name] = obj
for name, obj in frame.f_locals.items():
if name in codeNames:
name2obj[name] = obj
# show them in alphabetical order
names = name2obj.keys()
names.sort()
# push them in reverse order so they'll be popped in the correct order
names.reverse()
traversedIds = set()
for name in names:
stateStack.push([name, name2obj[name], traversedIds])
while len(stateStack) > 0:
name, obj, traversedIds = stateStack.pop()
#notify.info('%s, %s, %s' % (name, fastRepr(obj), traversedIds))
r = fastRepr(obj, maxLen=10)
if type(r) is types.StringType:
r = r.replace('\n', '\\n')
s += '\n %s = %s' % (name, r)
# if we've already traversed through this object, don't traverse through it again
if id(obj) not in traversedIds:
attrName2obj = {}
for attrName in codeNames:
attr = getattr(obj, attrName, _AttrNotFound)
if (attr is not _AttrNotFound):
# prevent infinite recursion on method wrappers (__init__.__init__.__init__...)
try:
className = attr.__class__.__name__
except:
pass
else:
if className == 'method-wrapper':
continue
attrName2obj[attrName] = attr
if len(attrName2obj):
# show them in alphabetical order
attrNames = attrName2obj.keys()
attrNames.sort()
# push them in reverse order so they'll be popped in the correct order
attrNames.reverse()
ids = set(traversedIds)
ids.add(id(obj))
for attrName in attrNames:
obj = attrName2obj[attrName]
stateStack.push(['%s.%s' % (name, attrName), obj, ids])
tb = tb.tb_next
if foundRun:
s += '\n'
if wantStackDumpLog:
notify.info(s)
if wantStackDumpUpload:
excStrs = traceback.format_exception(eType, eValue, origTb)
for excStr in excStrs:
s += excStr
timeMgr = None
try:
timeMgr = base.cr.timeManager
except:
try:
timeMgr = simbase.air.timeManager
except:
pass
if timeMgr:
timeMgr.setStackDump(s)
oldExcepthook(eType, eValue, origTb)
def run(self):
# do the garbage collection
oldFlags = gc.get_debug()
if self._args.delOnly:
# do a collect without SAVEALL, to identify the instances that are involved in
# cycles with instances that define __del__
# cycles that do not involve any instances that define __del__ are cleaned up
# automatically by Python, but they also appear in gc.garbage when SAVEALL is set
gc.set_debug(0)
if self._args.collect:
gc.collect()
garbageInstances = gc.garbage[:]
del gc.garbage[:]
# only yield if there's more time-consuming work to do,
# if there's no garbage, give instant feedback
if len(garbageInstances) > 0:
yield None
# don't repr the garbage list if we don't have to
if self.notify.getDebug():
self.notify.debug('garbageInstances == %s' %
fastRepr(garbageInstances))
self.numGarbageInstances = len(garbageInstances)
# grab the ids of the garbage instances (objects with __del__)
self.garbageInstanceIds = set()
for i in xrange(len(garbageInstances)):
self.garbageInstanceIds.add(id(garbageInstances[i]))
if not (i % 20):
yield None
# then release the list of instances so that it doesn't interfere with the gc.collect() below
del garbageInstances
else:
self.garbageInstanceIds = set()
# do a SAVEALL pass so that we have all of the objects involved in legitimate garbage cycles
# without SAVEALL, gc.garbage only contains objects with __del__ methods
gc.set_debug(gc.DEBUG_SAVEALL)
if self._args.collect:
gc.collect()
self.garbage = gc.garbage[:]
del gc.garbage[:]
# only yield if there's more time-consuming work to do,
# if there's no garbage, give instant feedback
if len(self.garbage) > 0:
yield None
# don't repr the garbage list if we don't have to
if self.notify.getDebug():
self.notify.debug('self.garbage == %s' % fastRepr(self.garbage))
gc.set_debug(oldFlags)
self.numGarbage = len(self.garbage)
# only yield if there's more time-consuming work to do,
# if there's no garbage, give instant feedback
if self.numGarbage > 0:
yield None
if self._args.verbose:
self.notify.info('found %s garbage items' % self.numGarbage)
""" spammy
# print the types of the garbage first, in case the repr of an object
# causes a crash
if self.numGarbage > 0:
self.notify.info('TYPES ONLY (this is only needed if a crash occurs before GarbageReport finishes):')
for result in printNumberedTypesGen(self.garbage):
yield None
"""
# Py obj id -> garbage list index
self._id2index = {}
self.referrersByReference = {}
self.referrersByNumber = {}
self.referentsByReference = {}
self.referentsByNumber = {}
self._id2garbageInfo = {}
self.cycles = []
self.cyclesBySyntax = []
self.uniqueCycleSets = set()
self.cycleIds = set()
# make the id->index table to speed up the next steps
for i in xrange(self.numGarbage):
self._id2index[id(self.garbage[i])] = i
if not (i % 20):
yield None
# grab the referrers (pointing to garbage)
if self._args.fullReport and (self.numGarbage != 0):
if self._args.verbose:
self.notify.info('getting referrers...')
for i in xrange(self.numGarbage):
yield None
for result in self._getReferrers(self.garbage[i]):
yield None
byNum, byRef = result
self.referrersByNumber[i] = byNum
self.referrersByReference[i] = byRef
# grab the referents (pointed to by garbage)
if self.numGarbage > 0:
if self._args.verbose:
self.notify.info('getting referents...')
for i in xrange(self.numGarbage):
yield None
for result in self._getReferents(self.garbage[i]):
yield None
byNum, byRef = result
self.referentsByNumber[i] = byNum
self.referentsByReference[i] = byRef
for i in xrange(self.numGarbage):
if hasattr(self.garbage[i], '_garbageInfo') and callable(
self.garbage[i]._garbageInfo):
try:
info = self.garbage[i]._garbageInfo()
except Exception, e:
info = str(e)
self._id2garbageInfo[id(self.garbage[i])] = info
yield None
else:
if not (i % 20):
yield None
class GarbageReport(Job):
"""Detects leaked Python objects (via gc.collect()) and reports on garbage
items, garbage-to-garbage references, and garbage cycles.
If you just want to dump the report to the log, use GarbageLogger."""
notify = directNotify.newCategory("GarbageReport")
def __init__(self,
name,
log=True,
verbose=False,
fullReport=False,
findCycles=True,
threaded=False,
doneCallback=None,
autoDestroy=False,
priority=None,
safeMode=False,
delOnly=False,
collect=True):
# if autoDestroy is True, GarbageReport will self-destroy after logging
# if false, caller is responsible for calling destroy()
# if threaded is True, processing will be performed over multiple frames
# if collect is False, we assume that the caller just did a collect and the results
# are still in gc.garbage
Job.__init__(self, name)
# stick the arguments onto a ScratchPad so we can delete them all at once
self._args = ScratchPad(name=name,
log=log,
verbose=verbose,
fullReport=fullReport,
findCycles=findCycles,
doneCallback=doneCallback,
autoDestroy=autoDestroy,
safeMode=safeMode,
delOnly=delOnly,
collect=collect)
if priority is not None:
self.setPriority(priority)
jobMgr.add(self)
if not threaded:
jobMgr.finish(self)
def run(self):
# do the garbage collection
oldFlags = gc.get_debug()
if self._args.delOnly:
# do a collect without SAVEALL, to identify the instances that are involved in
# cycles with instances that define __del__
# cycles that do not involve any instances that define __del__ are cleaned up
# automatically by Python, but they also appear in gc.garbage when SAVEALL is set
gc.set_debug(0)
if self._args.collect:
gc.collect()
garbageInstances = gc.garbage[:]
del gc.garbage[:]
# only yield if there's more time-consuming work to do,
# if there's no garbage, give instant feedback
if len(garbageInstances) > 0:
yield None
# don't repr the garbage list if we don't have to
if self.notify.getDebug():
self.notify.debug('garbageInstances == %s' %
fastRepr(garbageInstances))
self.numGarbageInstances = len(garbageInstances)
# grab the ids of the garbage instances (objects with __del__)
self.garbageInstanceIds = set()
for i in xrange(len(garbageInstances)):
self.garbageInstanceIds.add(id(garbageInstances[i]))
if not (i % 20):
yield None
# then release the list of instances so that it doesn't interfere with the gc.collect() below
del garbageInstances
else:
self.garbageInstanceIds = set()
# do a SAVEALL pass so that we have all of the objects involved in legitimate garbage cycles
# without SAVEALL, gc.garbage only contains objects with __del__ methods
gc.set_debug(gc.DEBUG_SAVEALL)
if self._args.collect:
gc.collect()
self.garbage = gc.garbage[:]
del gc.garbage[:]
# only yield if there's more time-consuming work to do,
# if there's no garbage, give instant feedback
if len(self.garbage) > 0:
yield None
# don't repr the garbage list if we don't have to
if self.notify.getDebug():
self.notify.debug('self.garbage == %s' % fastRepr(self.garbage))
gc.set_debug(oldFlags)
self.numGarbage = len(self.garbage)
# only yield if there's more time-consuming work to do,
# if there's no garbage, give instant feedback
if self.numGarbage > 0:
yield None
if self._args.verbose:
self.notify.info('found %s garbage items' % self.numGarbage)
""" spammy
# print the types of the garbage first, in case the repr of an object
# causes a crash
if self.numGarbage > 0:
self.notify.info('TYPES ONLY (this is only needed if a crash occurs before GarbageReport finishes):')
for result in printNumberedTypesGen(self.garbage):
yield None
"""
# Py obj id -> garbage list index
self._id2index = {}
self.referrersByReference = {}
self.referrersByNumber = {}
self.referentsByReference = {}
self.referentsByNumber = {}
self._id2garbageInfo = {}
self.cycles = []
self.cyclesBySyntax = []
self.uniqueCycleSets = set()
self.cycleIds = set()
# make the id->index table to speed up the next steps
for i in xrange(self.numGarbage):
self._id2index[id(self.garbage[i])] = i
if not (i % 20):
yield None
# grab the referrers (pointing to garbage)
if self._args.fullReport and (self.numGarbage != 0):
if self._args.verbose:
self.notify.info('getting referrers...')
for i in xrange(self.numGarbage):
yield None
for result in self._getReferrers(self.garbage[i]):
yield None
byNum, byRef = result
self.referrersByNumber[i] = byNum
self.referrersByReference[i] = byRef
# grab the referents (pointed to by garbage)
if self.numGarbage > 0:
if self._args.verbose:
self.notify.info('getting referents...')
for i in xrange(self.numGarbage):
yield None
for result in self._getReferents(self.garbage[i]):
yield None
byNum, byRef = result
self.referentsByNumber[i] = byNum
self.referentsByReference[i] = byRef
for i in xrange(self.numGarbage):
if hasattr(self.garbage[i], '_garbageInfo') and callable(
self.garbage[i]._garbageInfo):
try:
info = self.garbage[i]._garbageInfo()
except Exception, e:
info = str(e)
self._id2garbageInfo[id(self.garbage[i])] = info
yield None
else:
if not (i % 20):
yield None
# find the cycles
if self._args.findCycles and self.numGarbage > 0:
if self._args.verbose:
self.notify.info('calculating cycles...')
for i in xrange(self.numGarbage):
yield None
for newCycles in self._getCycles(i, self.uniqueCycleSets):
yield None
self.cycles.extend(newCycles)
# create a representation of the cycle in human-readable form
newCyclesBySyntax = []
for cycle in newCycles:
cycleBySyntax = ''
objs = []
# leave off the last index, it's a repeat of the first index
for index in cycle[:-1]:
objs.append(self.garbage[index])
yield None
# make the list repeat so we can safely iterate off the end
numObjs = len(objs) - 1
objs.extend(objs)
# state variables for our loop below
numToSkip = 0
objAlreadyRepresented = False
# if cycle starts off with an instance dict, start with the instance instead
startIndex = 0
# + 1 to include a reference back to the first object
endIndex = numObjs + 1
if type(objs[-1]) is types.InstanceType and type(
objs[0]) is types.DictType:
startIndex -= 1
endIndex -= 1
for index in xrange(startIndex, endIndex):
if numToSkip:
numToSkip -= 1
continue
obj = objs[index]
if type(obj) is types.InstanceType:
if not objAlreadyRepresented:
cycleBySyntax += '%s' % obj.__class__.__name__
cycleBySyntax += '.'
# skip past the instance dict and get the member obj
numToSkip += 1
member = objs[index + 2]
for key, value in obj.__dict__.iteritems():
if value is member:
break
yield None
else:
key = '<unknown member name>'
cycleBySyntax += '%s' % key
objAlreadyRepresented = True
elif type(obj) is types.DictType:
cycleBySyntax += '{'
# get object referred to by dict
val = objs[index + 1]
for key, value in obj.iteritems():
if value is val:
break
yield None
else:
key = '<unknown key>'
cycleBySyntax += '%s}' % fastRepr(key)
objAlreadyRepresented = True
elif type(obj) in (types.TupleType, types.ListType):
brackets = {
types.TupleType: '()',
types.ListType: '[]',
}[type(obj)]
# get object being referenced by container
nextObj = objs[index + 1]
cycleBySyntax += brackets[0]
for index in xrange(len(obj)):
if obj[index] is nextObj:
index = str(index)
break
yield None
else:
index = '<unknown index>'
cycleBySyntax += '%s%s' % (index, brackets[1])
objAlreadyRepresented = True
else:
cycleBySyntax += '%s --> ' % itype(obj)
objAlreadyRepresented = False
newCyclesBySyntax.append(cycleBySyntax)
yield None
self.cyclesBySyntax.extend(newCyclesBySyntax)
# if we're not doing a full report, add this cycle's IDs to the master set
if not self._args.fullReport:
for cycle in newCycles:
yield None
self.cycleIds.update(set(cycle))
self.numCycles = len(self.cycles)
if self._args.findCycles:
s = [
'===== GarbageReport: \'%s\' (%s %s) =====' %
(self._args.name, self.numCycles,
('cycle' if self.numCycles == 1 else 'cycles'))
]
else:
s = ['===== GarbageReport: \'%s\' =====' % (self._args.name)]
if self.numGarbage > 0:
# make a list of the ids we will actually be printing
if self._args.fullReport:
garbageIndices = range(self.numGarbage)
else:
garbageIndices = list(self.cycleIds)
garbageIndices.sort()
numGarbage = len(garbageIndices)
# log each individual item with a number in front of it
if not self._args.fullReport:
abbrev = '(abbreviated) '
else:
abbrev = ''
s.append('===== Garbage Items %s=====' % abbrev)
digits = 0
n = numGarbage
while n > 0:
yield None
digits += 1
n /= 10
digits = digits
format = '%0' + '%s' % digits + 'i:%s \t%s'
for i in xrange(numGarbage):
yield None
idx = garbageIndices[i]
if self._args.safeMode:
# in safe mode, don't try to repr any of the objects
objStr = repr(itype(self.garbage[idx]))
else:
objStr = fastRepr(self.garbage[idx])
maxLen = 5000
if len(objStr) > maxLen:
snip = '<SNIP>'
objStr = '%s%s' % (objStr[:(maxLen - len(snip))], snip)
s.append(format % (idx, itype(self.garbage[idx]), objStr))
# also log the types of the objects
s.append('===== Garbage Item Types %s=====' % abbrev)
for i in xrange(numGarbage):
yield None
idx = garbageIndices[i]
objStr = str(deeptype(self.garbage[idx]))
maxLen = 5000
if len(objStr) > maxLen:
snip = '<SNIP>'
objStr = '%s%s' % (objStr[:(maxLen - len(snip))], snip)
s.append(format % (idx, itype(self.garbage[idx]), objStr))
if self._args.findCycles:
s.append('===== Garbage Cycles (Garbage Item Numbers) =====')
ac = AlphabetCounter()
for i in xrange(self.numCycles):
yield None
s.append('%s:%s' % (ac.next(), self.cycles[i]))
if self._args.findCycles:
s.append('===== Garbage Cycles (Python Syntax) =====')
ac = AlphabetCounter()
for i in xrange(len(self.cyclesBySyntax)):
yield None
s.append('%s:%s' % (ac.next(), self.cyclesBySyntax[i]))
if len(self._id2garbageInfo):
s.append('===== Garbage Custom Info =====')
ac = AlphabetCounter()
for i in xrange(len(self.cyclesBySyntax)):
yield None
counter = ac.next()
_id = id(self.garbage[i])
if _id in self._id2garbageInfo:
s.append('%s:%s' %
(counter, self._id2garbageInfo[_id]))
if self._args.fullReport:
format = '%0' + '%s' % digits + 'i:%s'
s.append(
'===== Referrers By Number (what is referring to garbage item?) ====='
)
for i in xrange(numGarbage):
yield None
s.append(format % (i, self.referrersByNumber[i]))
s.append(
'===== Referents By Number (what is garbage item referring to?) ====='
)
for i in xrange(numGarbage):
yield None
s.append(format % (i, self.referentsByNumber[i]))
s.append(
'===== Referrers (what is referring to garbage item?) ====='
)
for i in xrange(numGarbage):
yield None
s.append(format % (i, self.referrersByReference[i]))
s.append(
'===== Referents (what is garbage item referring to?) ====='
)
for i in xrange(numGarbage):
yield None
s.append(format % (i, self.referentsByReference[i]))
self._report = s
if self._args.log:
self.printingBegin()
for i in xrange(len(self._report)):
if self.numGarbage > 0:
yield None
self.notify.info(self._report[i])
self.notify.info('===== Garbage Report Done =====')
self.printingEnd()
yield Job.Done
def run(self):
# do the garbage collection
oldFlags = gc.get_debug()
if self._args.delOnly:
# do a collect without SAVEALL, to identify the instances that are involved in
# cycles with instances that define __del__
# cycles that do not involve any instances that define __del__ are cleaned up
# automatically by Python, but they also appear in gc.garbage when SAVEALL is set
gc.set_debug(0)
if self._args.collect:
gc.collect()
garbageInstances = gc.garbage[:]
del gc.garbage[:]
# only yield if there's more time-consuming work to do,
# if there's no garbage, give instant feedback
if len(garbageInstances) > 0:
yield None
# don't repr the garbage list if we don't have to
if self.notify.getDebug():
self.notify.debug('garbageInstances == %s' % fastRepr(garbageInstances))
self.numGarbageInstances = len(garbageInstances)
# grab the ids of the garbage instances (objects with __del__)
self.garbageInstanceIds = set()
for i in xrange(len(garbageInstances)):
self.garbageInstanceIds.add(id(garbageInstances[i]))
if not (i % 20):
yield None
# then release the list of instances so that it doesn't interfere with the gc.collect() below
del garbageInstances
else:
self.garbageInstanceIds = set()
# do a SAVEALL pass so that we have all of the objects involved in legitimate garbage cycles
# without SAVEALL, gc.garbage only contains objects with __del__ methods
gc.set_debug(gc.DEBUG_SAVEALL)
if self._args.collect:
gc.collect()
self.garbage = gc.garbage[:]
del gc.garbage[:]
# only yield if there's more time-consuming work to do,
# if there's no garbage, give instant feedback
if len(self.garbage) > 0:
yield None
# don't repr the garbage list if we don't have to
if self.notify.getDebug():
self.notify.debug('self.garbage == %s' % fastRepr(self.garbage))
gc.set_debug(oldFlags)
self.numGarbage = len(self.garbage)
# only yield if there's more time-consuming work to do,
# if there's no garbage, give instant feedback
if self.numGarbage > 0:
yield None
if self._args.verbose:
self.notify.info('found %s garbage items' % self.numGarbage)
""" spammy
# print the types of the garbage first, in case the repr of an object
# causes a crash
if self.numGarbage > 0:
self.notify.info('TYPES ONLY (this is only needed if a crash occurs before GarbageReport finishes):')
for result in printNumberedTypesGen(self.garbage):
yield None
"""
# Py obj id -> garbage list index
self._id2index = {}
self.referrersByReference = {}
self.referrersByNumber = {}
self.referentsByReference = {}
self.referentsByNumber = {}
self._id2garbageInfo = {}
self.cycles = []
self.cyclesBySyntax = []
self.uniqueCycleSets = set()
self.cycleIds = set()
# make the id->index table to speed up the next steps
for i in xrange(self.numGarbage):
self._id2index[id(self.garbage[i])] = i
if not (i % 20):
yield None
# grab the referrers (pointing to garbage)
if self._args.fullReport and (self.numGarbage != 0):
if self._args.verbose:
self.notify.info('getting referrers...')
for i in xrange(self.numGarbage):
yield None
for result in self._getReferrers(self.garbage[i]):
yield None
byNum, byRef = result
self.referrersByNumber[i] = byNum
self.referrersByReference[i] = byRef
# grab the referents (pointed to by garbage)
if self.numGarbage > 0:
if self._args.verbose:
self.notify.info('getting referents...')
for i in xrange(self.numGarbage):
yield None
for result in self._getReferents(self.garbage[i]):
yield None
byNum, byRef = result
self.referentsByNumber[i] = byNum
self.referentsByReference[i] = byRef
for i in xrange(self.numGarbage):
if hasattr(self.garbage[i], '_garbageInfo') and callable(self.garbage[i]._garbageInfo):
try:
info = self.garbage[i]._garbageInfo()
except Exception, e:
info = str(e)
self._id2garbageInfo[id(self.garbage[i])] = info
yield None
else:
if not (i % 20):
yield None
def _excepthookDumpVars(eType, eValue, tb):
origTb = tb
excStrs = traceback.format_exception(eType, eValue, origTb)
s = 'printing traceback in case variable repr crashes the process...\n'
for excStr in excStrs:
s += excStr
notify.info(s)
s = 'DUMPING STACK FRAME VARIABLES'
foundRun = True
while tb is not None:
frame = tb.tb_frame
code = frame.f_code
codeNames = set(code.co_names)
if not foundRun:
if code.co_name == 'run':
foundRun = True
else:
tb = tb.tb_next
s += '\n File "%s", line %s, in %s' % (code.co_filename, frame.f_lineno, code.co_name)
stateStack = Stack()
name2obj = { }
for (name, obj) in frame.f_builtins.items():
if name in codeNames:
name2obj[name] = obj
continue
for (name, obj) in frame.f_globals.items():
if name in codeNames:
name2obj[name] = obj
continue
for (name, obj) in frame.f_locals.items():
if name in codeNames:
name2obj[name] = obj
continue
names = name2obj.keys()
names.sort()
names.reverse()
traversedIds = set()
for name in names:
stateStack.push([
name,
name2obj[name],
traversedIds])
while len(stateStack) > 0:
(name, obj, traversedIds) = stateStack.pop()
r = fastRepr(obj, maxLen = 10)
if type(r) is types.StringType:
r = r.replace('\n', '\\n')
s += '\n %s = %s' % (name, r)
if id(obj) not in traversedIds:
attrName2obj = { }
for attrName in codeNames:
attr = getattr(obj, attrName, _AttrNotFound)
if attr is not _AttrNotFound:
try:
className = attr.__class__.__name__
except:
pass
if className == 'method-wrapper':
continue
attrName2obj[attrName] = attr
continue
if len(attrName2obj):
attrNames = attrName2obj.keys()
attrNames.sort()
attrNames.reverse()
ids = set(traversedIds)
ids.add(id(obj))
for attrName in attrNames:
obj = attrName2obj[attrName]
stateStack.push([
'%s.%s' % (name, attrName),
obj,
ids])
len(attrName2obj)
tb = tb.tb_next
if foundRun:
s += '\n'
if wantStackDumpLog:
notify.info(s)
if wantStackDumpUpload:
excStrs = traceback.format_exception(eType, eValue, origTb)
for excStr in excStrs:
s += excStr
timeMgr = None
try:
timeMgr = base.cr.timeManager
except:
try:
timeMgr = simbase.air.timeManager
if timeMgr:
timeMgr.setStackDump(s)
oldExcepthook(eType, eValue, origTb)
def run(self):
oldFlags = gc.get_debug()
if self._args.delOnly:
gc.set_debug(0)
if self._args.collect:
gc.collect()
garbageInstances = gc.garbage[:]
del gc.garbage[:]
if len(garbageInstances) > 0:
yield None
if self.notify.getDebug():
self.notify.debug('garbageInstances == %s' %
fastRepr(garbageInstances))
self.numGarbageInstances = len(garbageInstances)
self.garbageInstanceIds = set()
for i in xrange(len(garbageInstances)):
self.garbageInstanceIds.add(id(garbageInstances[i]))
if not i % 20:
yield None
continue
del garbageInstances
else:
self.garbageInstanceIds = set()
gc.set_debug(gc.DEBUG_SAVEALL)
if self._args.collect:
gc.collect()
self.garbage = gc.garbage[:]
del gc.garbage[:]
if len(self.garbage) > 0:
yield None
if self.notify.getDebug():
self.notify.debug('self.garbage == %s' % fastRepr(self.garbage))
gc.set_debug(oldFlags)
self.numGarbage = len(self.garbage)
if self.numGarbage > 0:
yield None
self.notify.info('found %s items in gc.garbage' % self.numGarbage)
self._id2index = {}
self.referrersByReference = {}
self.referrersByNumber = {}
self.referentsByReference = {}
self.referentsByNumber = {}
self._id2garbageInfo = {}
self.cycles = []
self.cyclesBySyntax = []
self.uniqueCycleSets = set()
self.cycleIds = set()
for i in xrange(self.numGarbage):
self._id2index[id(self.garbage[i])] = i
if not i % 20:
yield None
continue
if self._args.fullReport and self.numGarbage != 0:
if self._args.verbose:
self.notify.info('getting referrers...')
for i in xrange(self.numGarbage):
yield None
for result in self._getReferrers(self.garbage[i]):
yield None
(byNum, byRef) = result
self.referrersByNumber[i] = byNum
self.referrersByReference[i] = byRef
if self.numGarbage > 0:
if self._args.verbose:
self.notify.info('getting referents...')
for i in xrange(self.numGarbage):
yield None
for result in self._getReferents(self.garbage[i]):
yield None
(byNum, byRef) = result
self.referentsByNumber[i] = byNum
self.referentsByReference[i] = byRef
for i in xrange(self.numGarbage):
if hasattr(self.garbage[i], '_garbageInfo') and callable(
self.garbage[i]._garbageInfo):
try:
info = self.garbage[i]._garbageInfo()
except Exception:
e = None
info = str(e)
self._id2garbageInfo[id(self.garbage[i])] = info
yield None
continue
if not i % 20:
yield None
continue
if self._args.findCycles and self.numGarbage > 0:
if self._args.verbose:
self.notify.info('calculating cycles...')
for i in xrange(self.numGarbage):
yield None
for newCycles in self._getCycles(i, self.uniqueCycleSets):
yield None
self.cycles.extend(newCycles)
newCyclesBySyntax = []
for cycle in newCycles:
cycleBySyntax = ''
objs = []
for index in cycle[:-1]:
objs.append(self.garbage[index])
yield None
numObjs = len(objs) - 1
objs.extend(objs)
numToSkip = 0
objAlreadyRepresented = False
startIndex = 0
endIndex = numObjs + 1
if type(objs[-1]) is types.InstanceType and type(
objs[0]) is types.DictType:
startIndex -= 1
endIndex -= 1
for index in xrange(startIndex, endIndex):
if numToSkip:
numToSkip -= 1
continue
obj = objs[index]
if type(obj) is types.InstanceType:
if not objAlreadyRepresented:
cycleBySyntax += '%s' % obj.__class__.__name__
cycleBySyntax += '.'
numToSkip += 1
member = objs[index + 2]
for (key, value) in obj.__dict__.iteritems():
if value is member:
break
yield None
else:
key = '<unknown member name>'
cycleBySyntax += '%s' % key
objAlreadyRepresented = True
continue
if type(obj) is types.DictType:
cycleBySyntax += '{'
val = objs[index + 1]
for (key, value) in obj.iteritems():
if value is val:
break
yield None
else:
key = '<unknown key>'
cycleBySyntax += '%s}' % fastRepr(key)
objAlreadyRepresented = True
continue
if type(obj) in (types.TupleType, types.ListType):
brackets = {
types.TupleType: '()',
types.ListType: '[]'
}[type(obj)]
nextObj = objs[index + 1]
cycleBySyntax += brackets[0]
for index in xrange(len(obj)):
if obj[index] is nextObj:
index = str(index)
break
yield None
else:
index = '<unknown index>'
cycleBySyntax += '%s%s' % (index, brackets[1])
objAlreadyRepresented = True
continue
cycleBySyntax += '%s --> ' % itype(obj)
objAlreadyRepresented = False
newCyclesBySyntax.append(cycleBySyntax)
yield None
self.cyclesBySyntax.extend(newCyclesBySyntax)
if not self._args.fullReport:
for cycle in newCycles:
yield None
self.cycleIds.update(set(cycle))
self.numCycles = len(self.cycles)
if self._args.findCycles:
s = [
"===== GarbageReport: '%s' (%s %s) =====" %
(self._args.name, self.numCycles,
choice(self.numCycles == 1, 'cycle', 'cycles'))
]
else:
s = ["===== GarbageReport: '%s' =====" % self._args.name]
if self.numGarbage > 0:
if self._args.fullReport:
garbageIndices = range(self.numGarbage)
else:
garbageIndices = list(self.cycleIds)
garbageIndices.sort()
numGarbage = len(garbageIndices)
if not self._args.fullReport:
abbrev = '(abbreviated) '
else:
abbrev = ''
s.append('===== Garbage Items %s=====' % abbrev)
digits = 0
n = numGarbage
while n > 0:
yield None
digits += 1
n /= 10
digits = digits
format = '%0' + '%s' % digits + 'i:%s \t%s'
for i in xrange(numGarbage):
yield None
idx = garbageIndices[i]
if self._args.safeMode:
objStr = repr(itype(self.garbage[idx]))
else:
objStr = fastRepr(self.garbage[idx])
maxLen = 5000
if len(objStr) > maxLen:
snip = '<SNIP>'
objStr = '%s%s' % (objStr[:maxLen - len(snip)], snip)
s.append(format % (idx, itype(self.garbage[idx]), objStr))
s.append('===== Garbage Item Types %s=====' % abbrev)
for i in xrange(numGarbage):
yield None
idx = garbageIndices[i]
objStr = str(deeptype(self.garbage[idx]))
maxLen = 5000
if len(objStr) > maxLen:
snip = '<SNIP>'
objStr = '%s%s' % (objStr[:maxLen - len(snip)], snip)
s.append(format % (idx, itype(self.garbage[idx]), objStr))
if self._args.findCycles:
s.append('===== Garbage Cycles (Garbage Item Numbers) =====')
ac = AlphabetCounter()
for i in xrange(self.numCycles):
yield None
s.append('%s:%s' % (ac.next(), self.cycles[i]))
if self._args.findCycles:
s.append('===== Garbage Cycles (Python Syntax) =====')
ac = AlphabetCounter()
for i in xrange(len(self.cyclesBySyntax)):
yield None
s.append('%s:%s' % (ac.next(), self.cyclesBySyntax[i]))
if len(self._id2garbageInfo):
format = '%0' + '%s' % digits + 'i:%s'
s.append('===== Garbage Custom Info =====')
ids = self._id2garbageInfo.keys()
yield None
indices = []
for _id in ids:
indices.append(self._id2index[_id])
yield None
indices.sort()
yield None
for i in indices:
_id = id(self.garbage[i])
s.append(format % (i, self._id2garbageInfo[_id]))
yield None
if self._args.fullReport:
format = '%0' + '%s' % digits + 'i:%s'
s.append(
'===== Referrers By Number (what is referring to garbage item?) ====='
)
for i in xrange(numGarbage):
yield None
s.append(format % (i, self.referrersByNumber[i]))
s.append(
'===== Referents By Number (what is garbage item referring to?) ====='
)
for i in xrange(numGarbage):
yield None
s.append(format % (i, self.referentsByNumber[i]))
s.append(
'===== Referrers (what is referring to garbage item?) ====='
)
for i in xrange(numGarbage):
yield None
s.append(format % (i, self.referrersByReference[i]))
s.append(
'===== Referents (what is garbage item referring to?) ====='
)
for i in xrange(numGarbage):
yield None
s.append(format % (i, self.referentsByReference[i]))
self._report = s
if self._args.log:
self.printingBegin()
for i in xrange(len(self._report)):
if self.numGarbage > 0:
yield None
self.notify.info(self._report[i])
self.notify.info('===== Garbage Report Done =====')
self.printingEnd()
yield Job.Done
def run(self):
# do the garbage collection
wasOn = gcDebugOn()
oldFlags = gc.get_debug()
if not wasOn:
gc.set_debug(gc.DEBUG_SAVEALL)
gc.collect()
yield None
# don't repr the garbage list if we don't have to
if self.notify.getDebug():
self.notify.debug('gc.garbage == %s' % fastRepr(gc.garbage))
yield None
self.garbage = list(gc.garbage)
# don't repr the garbage list if we don't have to
if self.notify.getDebug():
self.notify.debug('self.garbage == %s' % self.garbage)
del gc.garbage[:]
if not wasOn:
gc.set_debug(oldFlags)
self.numGarbage = len(self.garbage)
yield None
if self._args.verbose:
self.notify.info('found %s garbage items' % self.numGarbage)
self.referrersByReference = {}
self.referrersByNumber = {}
self.referentsByReference = {}
self.referentsByNumber = {}
self.cycles = []
self.cycleSets = []
self.cycleIds = set()
# grab the referrers (pointing to garbage)
if self._args.fullReport and (self.numGarbage != 0):
if self._args.verbose:
self.notify.info('getting referrers...')
for i in xrange(self.numGarbage):
yield None
for result in self._getReferrers(self.garbage[i]):
yield None
byNum, byRef = result
self.referrersByNumber[i] = byNum
self.referrersByReference[i] = byRef
# grab the referents (pointed to by garbage)
if self.numGarbage > 0:
if self._args.verbose:
self.notify.info('getting referents...')
for i in xrange(self.numGarbage):
yield None
for result in self._getReferents(self.garbage[i]):
yield None
byNum, byRef = result
self.referentsByNumber[i] = byNum
self.referentsByReference[i] = byRef
# find the cycles
if self._args.findCycles and self.numGarbage > 0:
if self._args.verbose:
self.notify.info('detecting cycles...')
for i in xrange(self.numGarbage):
yield None
for newCycles in self._getCycles(i, self.cycleSets):
yield None
self.cycles.extend(newCycles)
# if we're not doing a full report, add this cycle's IDs to the master set
if not self._args.fullReport:
for cycle in newCycles:
yield None
self.cycleIds.update(set(cycle))
if self._args.findCycles:
s = ['===== GarbageReport: \'%s\' (%s items, %s cycles) =====' % (
self._args.name, self.numGarbage, len(self.cycles))]
else:
s = ['===== GarbageReport: \'%s\' (%s items) =====' % (
self._args.name, self.numGarbage)]
if self.numGarbage > 0:
# make a list of the ids we will actually be printing
if self._args.fullReport:
garbageIds = range(self.numGarbage)
else:
garbageIds = list(self.cycleIds)
garbageIds.sort()
numGarbage = len(garbageIds)
# log each individual item with a number in front of it
if not self._args.fullReport:
abbrev = '(abbreviated) '
else:
abbrev = ''
s.append('===== Garbage Items %s=====' % abbrev)
digits = 0
n = numGarbage
while n > 0:
yield None
digits += 1
n /= 10
digits = digits
format = '%0' + '%s' % digits + 'i:%s \t%s'
for i in xrange(numGarbage):
yield None
id = garbageIds[i]
objStr = safeRepr(self.garbage[id])
maxLen = 5000
if len(objStr) > maxLen:
snip = '<SNIP>'
objStr = '%s%s' % (objStr[:(maxLen-len(snip))], snip)
s.append(format % (id, itype(self.garbage[id]), objStr))
if self._args.findCycles:
s.append('===== Garbage Cycles =====')
for i in xrange(len(self.cycles)):
yield None
s.append('%s' % self.cycles[i])
if self._args.fullReport:
format = '%0' + '%s' % digits + 'i:%s'
s.append('===== Referrers By Number (what is referring to garbage item?) =====')
for i in xrange(numGarbage):
yield None
s.append(format % (i, self.referrersByNumber[i]))
s.append('===== Referents By Number (what is garbage item referring to?) =====')
for i in xrange(numGarbage):
yield None
s.append(format % (i, self.referentsByNumber[i]))
s.append('===== Referrers (what is referring to garbage item?) =====')
for i in xrange(numGarbage):
yield None
s.append(format % (i, self.referrersByReference[i]))
s.append('===== Referents (what is garbage item referring to?) =====')
for i in xrange(numGarbage):
yield None
s.append(format % (i, self.referentsByReference[i]))
self._report = s
if self._args.log:
self.printingBegin()
for i in xrange(len(self._report)):
yield None
self.notify.info(self._report[i])
self.printingEnd()
yield Job.Done
def _excepthookDumpVars(eType, eValue, tb):
global oldExcepthook
global wantStackDumpLog
global wantStackDumpUpload
origTb = tb
excStrs = traceback.format_exception(eType, eValue, origTb)
s = 'printing traceback in case variable repr crashes the process...\n'
for excStr in excStrs:
s += excStr
notify.info(s)
s = 'DUMPING STACK FRAME VARIABLES'
foundRun = True
while tb is not None:
frame = tb.tb_frame
code = frame.f_code
codeNames = set(code.co_names)
if not foundRun:
if code.co_name == 'run':
foundRun = True
else:
tb = tb.tb_next
continue
s += '\n File "%s", line %s, in %s' % (code.co_filename, frame.f_lineno, code.co_name)
stateStack = Stack()
name2obj = {}
for name, obj in frame.f_builtins.items():
if name in codeNames:
name2obj[name] = obj
for name, obj in frame.f_globals.items():
if name in codeNames:
name2obj[name] = obj
for name, obj in frame.f_locals.items():
if name in codeNames:
name2obj[name] = obj
names = name2obj.keys()
names.sort()
names.reverse()
traversedIds = set()
for name in names:
stateStack.push([name, name2obj[name], traversedIds])
while len(stateStack) > 0:
name, obj, traversedIds = stateStack.pop()
r = fastRepr(obj, maxLen=10)
if type(r) is types.StringType:
r = r.replace('\n', '\\n')
s += '\n %s = %s' % (name, r)
if id(obj) not in traversedIds:
attrName2obj = {}
for attrName in codeNames:
attr = getattr(obj, attrName, _AttrNotFound)
if attr is not _AttrNotFound:
try:
className = attr.__class__.__name__
except:
pass
else:
if className == 'method-wrapper':
continue
attrName2obj[attrName] = attr
if len(attrName2obj):
attrNames = attrName2obj.keys()
attrNames.sort()
attrNames.reverse()
ids = set(traversedIds)
ids.add(id(obj))
for attrName in attrNames:
obj = attrName2obj[attrName]
stateStack.push(['%s.%s' % (name, attrName), obj, ids])
tb = tb.tb_next
if foundRun:
s += '\n'
if wantStackDumpLog:
notify.info(s)
if wantStackDumpUpload:
excStrs = traceback.format_exception(eType, eValue, origTb)
for excStr in excStrs:
s += excStr
timeMgr = None
try:
timeMgr = base.cr.timeManager
except:
try:
timeMgr = simbase.air.timeManager
except:
pass
if timeMgr:
timeMgr.setStackDump(s)
oldExcepthook(eType, eValue, origTb)
return
def run(self):
try:
self._leakDetector._index2containerId2len[self._index] = { }
ids = self._leakDetector.getContainerIds()
for objId in ids:
yield None
try:
for result in self._leakDetector.getContainerByIdGen(objId):
yield None
container = result
except Exception:
e = None
if self.notify.getDebug():
for contName in self._leakDetector.getContainerNameByIdGen(objId):
yield None
self.notify.debug('%s no longer exists; caught exception in getContainerById (%s)' % (contName, e))
self._leakDetector.removeContainerById(objId)
continue
if container is None:
if self.notify.getDebug():
for contName in self._leakDetector.getContainerNameByIdGen(objId):
yield None
self.notify.debug('%s no longer exists; getContainerById returned None' % contName)
self._leakDetector.removeContainerById(objId)
continue
try:
cLen = len(container)
except Exception:
e = None
if self.notify.getDebug():
for contName in self._leakDetector.getContainerNameByIdGen(objId):
yield None
self.notify.debug('%s is no longer a container, it is now %s (%s)' % (contName, safeRepr(container), e))
self._leakDetector.removeContainerById(objId)
continue
self._leakDetector._index2containerId2len[self._index][objId] = cLen
if self._index > 0:
idx2id2len = self._leakDetector._index2containerId2len
for objId in idx2id2len[self._index]:
yield None
if objId in idx2id2len[self._index - 1]:
diff = idx2id2len[self._index][objId] - idx2id2len[self._index - 1][objId]
if self._index > 2 and objId in idx2id2len[self._index - 2] and objId in idx2id2len[self._index - 3]:
diff2 = idx2id2len[self._index - 1][objId] - idx2id2len[self._index - 2][objId]
diff3 = idx2id2len[self._index - 2][objId] - idx2id2len[self._index - 3][objId]
if self._index <= 4:
if diff > 0 and diff2 > 0 and diff3 > 0:
name = self._leakDetector.getContainerNameById(objId)
try:
for container in self._leakDetector.getContainerByIdGen(objId):
yield None
except:
self.notify.debug('caught exception in getContainerByIdGen (2)')
msg = '%s (%s) consistently increased in size over the last 3 periods (%s items at last measurement, current contents: %s)' % (name, itype(container), idx2id2len[self._index][objId], fastRepr(container, maxLen = CheckContainers.ReprItems))
self.notify.warning(msg)
yield None
elif objId in idx2id2len[self._index - 4] and objId in idx2id2len[self._index - 5]:
diff4 = idx2id2len[self._index - 3][objId] - idx2id2len[self._index - 4][objId]
diff5 = idx2id2len[self._index - 4][objId] - idx2id2len[self._index - 5][objId]
if diff > 0 and diff2 > 0 and diff3 > 0 and diff4 > 0 and diff5 > 0:
name = self._leakDetector.getContainerNameById(objId)
try:
for container in self._leakDetector.getContainerByIdGen(objId):
yield None
except:
self.notify.debug('caught exception in getContainerByIdGen (3)')
msg = 'leak detected: %s (%s) consistently increased in size over the last 5 periods (%s items at last measurement, current contents: %s)' % (name, itype(container), idx2id2len[self._index][objId], fastRepr(container, maxLen = CheckContainers.ReprItems))
self.notify.warning(msg)
yield None
messenger.send(self._leakDetector.getLeakEvent(), [
container,
name])
if config.GetBool('pdb-on-leak-detect', 0):
import pdb as pdb
pdb.set_trace()
objId in idx2id2len[self._index - 3]
except Exception:
e = None
print 'CheckContainers job caught exception: %s' % e
if __dev__:
raise
yield Job.Done
def run(self):
try:
self._leakDetector._index2containerId2len[self._index] = {}
ids = self._leakDetector.getContainerIds()
# record the current len of each container
for objId in ids:
yield None
try:
for result in self._leakDetector.getContainerByIdGen(objId):
yield None
container = result
except Exception as e:
# this container no longer exists
if self.notify.getDebug():
for contName in self._leakDetector.getContainerNameByIdGen(objId):
yield None
self.notify.debug(
'%s no longer exists; caught exception in getContainerById (%s)' % (
contName, e))
self._leakDetector.removeContainerById(objId)
continue
if container is None:
# this container no longer exists
if self.notify.getDebug():
for contName in self._leakDetector.getContainerNameByIdGen(objId):
yield None
self.notify.debug('%s no longer exists; getContainerById returned None' %
contName)
self._leakDetector.removeContainerById(objId)
continue
try:
cLen = len(container)
except Exception as e:
# this container no longer exists
if self.notify.getDebug():
for contName in self._leakDetector.getContainerNameByIdGen(objId):
yield None
self.notify.debug(
'%s is no longer a container, it is now %s (%s)' %
(contName, safeRepr(container), e))
self._leakDetector.removeContainerById(objId)
continue
self._leakDetector._index2containerId2len[self._index][objId] = cLen
# compare the current len of each container to past lens
if self._index > 0:
idx2id2len = self._leakDetector._index2containerId2len
for objId in idx2id2len[self._index]:
yield None
if objId in idx2id2len[self._index-1]:
diff = idx2id2len[self._index][objId] - idx2id2len[self._index-1][objId]
"""
# this check is too spammy
if diff > 20:
if diff > idx2id2len[self._index-1][objId]:
minutes = (self._leakDetector._index2delay[self._index] -
self._leakDetector._index2delay[self._index-1]) / 60.
name = self._leakDetector.getContainerNameById(objId)
if idx2id2len[self._index-1][objId] != 0:
percent = 100. * (float(diff) / float(idx2id2len[self._index-1][objId]))
try:
for container in self._leakDetector.getContainerByIdGen(objId):
yield None
except:
# TODO
self.notify.debug('caught exception in getContainerByIdGen (1)')
else:
self.notify.warning(
'%s (%s) grew %.2f%% in %.2f minutes (%s items at last measurement, current contents: %s)' % (
name, itype(container), percent, minutes, idx2id2len[self._index][objId],
fastRepr(container, maxLen=CheckContainers.ReprItems)))
yield None
"""
if (self._index > 2 and
objId in idx2id2len[self._index-2] and
objId in idx2id2len[self._index-3]):
diff2 = idx2id2len[self._index-1][objId] - idx2id2len[self._index-2][objId]
diff3 = idx2id2len[self._index-2][objId] - idx2id2len[self._index-3][objId]
if self._index <= 4:
if diff > 0 and diff2 > 0 and diff3 > 0:
name = self._leakDetector.getContainerNameById(objId)
try:
for container in self._leakDetector.getContainerByIdGen(objId):
yield None
except:
# TODO
self.notify.debug('caught exception in getContainerByIdGen (2)')
else:
msg = ('%s (%s) consistently increased in size over the last '
'3 periods (%s items at last measurement, current contents: %s)' %
(name, itype(container), idx2id2len[self._index][objId],
fastRepr(container, maxLen=CheckContainers.ReprItems)))
self.notify.warning(msg)
yield None
elif (objId in idx2id2len[self._index-4] and
objId in idx2id2len[self._index-5]):
# if size has consistently increased over the last 5 checks,
# send out a warning
diff4 = idx2id2len[self._index-3][objId] - idx2id2len[self._index-4][objId]
diff5 = idx2id2len[self._index-4][objId] - idx2id2len[self._index-5][objId]
if diff > 0 and diff2 > 0 and diff3 > 0 and diff4 > 0 and diff5 > 0:
name = self._leakDetector.getContainerNameById(objId)
try:
for container in self._leakDetector.getContainerByIdGen(objId):
yield None
except:
# TODO
self.notify.debug('caught exception in getContainerByIdGen (3)')
else:
msg = ('leak detected: %s (%s) consistently increased in size over the last '
'5 periods (%s items at last measurement, current contents: %s)' %
(name, itype(container), idx2id2len[self._index][objId],
fastRepr(container, maxLen=CheckContainers.ReprItems)))
self.notify.warning(msg)
yield None
messenger.send(self._leakDetector.getLeakEvent(), [container, name])
if config.GetBool('pdb-on-leak-detect', 0):
import pdb;pdb.set_trace()
pass
except Exception as e:
print('CheckContainers job caught exception: %s' % e)
if __dev__:
raise
yield Job.Done
def run(self):
# set of ids of objects that we know are always attached to builtin;
# if an object is attached to one of these, it's attached to builtin
# this cuts down on the amount of searching that needs to be done
builtinIds = set()
builtinIds.add(id(builtins.__dict__))
try:
builtinIds.add(id(base))
builtinIds.add(id(base.cr))
builtinIds.add(id(base.cr.doId2do))
except:
pass
try:
builtinIds.add(id(simbase))
builtinIds.add(id(simbase.air))
builtinIds.add(id(simbase.air.doId2do))
except:
pass
try:
builtinIds.add(id(uber))
builtinIds.add(id(uber.air))
builtinIds.add(id(uber.air.doId2do))
except:
pass
while True:
yield None
objects = list(messenger._Messenger__objectEvents.keys())
assert self.notify.debug('%s objects in the messenger' %
len(objects))
for object in objects:
yield None
assert self.notify.debug('---> new object: %s' % itype(object))
# try to find a path to builtin that doesn't involve the messenger
# lists of objects for breadth-first search
# iterate through one list while populating other list
objList1 = []
objList2 = []
curObjList = objList1
nextObjList = objList2
visitedObjIds = set()
# add the id of the object, and the messenger containers so that
# the search for builtin will stop at the messenger; we're looking
# for any path to builtin that don't involve the messenger
visitedObjIds.add(id(object))
visitedObjIds.add(id(messenger._Messenger__objectEvents))
visitedObjIds.add(id(messenger._Messenger__callbacks))
nextObjList.append(object)
foundBuiltin = False
# breadth-first search, go until you run out of new objects or you find __builtin__
while len(nextObjList) > 0:
if foundBuiltin:
break
# swap the lists, prepare for the next pass
curObjList = nextObjList
nextObjList = []
assert self.notify.debug(
'next search iteration, num objects: %s' %
len(curObjList))
for curObj in curObjList:
if foundBuiltin:
break
yield None
referrers = gc.get_referrers(curObj)
assert self.notify.debug(
'curObj: %s @ %s, %s referrers, repr=%s' %
(itype(curObj), hex(id(curObj)), len(referrers),
fastRepr(curObj, maxLen=2)))
for referrer in referrers:
#assert self.notify.debug('referrer: %s' % itype(curObj))
yield None
refId = id(referrer)
# don't go in a loop
if refId in visitedObjIds:
#assert self.notify.debug('already visited')
continue
# don't self-reference
if referrer is curObjList or referrer is nextObjList:
continue
if refId in builtinIds:
# not a leak, there is a path to builtin that does not involve the messenger
#assert self.notify.debug('object has another path to __builtin__, it\'s not a messenger leak')
foundBuiltin = True
break
else:
visitedObjIds.add(refId)
nextObjList.append(referrer)
if not foundBuiltin:
self.notify.warning(
'%s is referenced only by the messenger' %
(itype(object)))
def run(self):
# do the garbage collection
oldFlags = gc.get_debug()
if self._args.delOnly:
# do a collect without SAVEALL, to identify the instances that are involved in
# cycles with instances that define __del__
# cycles that do not involve any instances that define __del__ are cleaned up
# automatically by Python, but they also appear in gc.garbage when SAVEALL is set
gc.set_debug(0)
if self._args.collect:
gc.collect()
garbageInstances = gc.garbage[:]
del gc.garbage[:]
# only yield if there's more time-consuming work to do,
# if there's no garbage, give instant feedback
if len(garbageInstances) > 0:
yield None
# don't repr the garbage list if we don't have to
if self.notify.getDebug():
self.notify.debug('garbageInstances == %s' % fastRepr(garbageInstances))
self.numGarbageInstances = len(garbageInstances)
# grab the ids of the garbage instances (objects with __del__)
self.garbageInstanceIds = set()
for i in xrange(len(garbageInstances)):
self.garbageInstanceIds.add(id(garbageInstances[i]))
if not (i % 20):
yield None
# then release the list of instances so that it doesn't interfere with the gc.collect() below
del garbageInstances
else:
self.garbageInstanceIds = set()
# do a SAVEALL pass so that we have all of the objects involved in legitimate garbage cycles
# without SAVEALL, gc.garbage only contains objects with __del__ methods
gc.set_debug(gc.DEBUG_SAVEALL)
if self._args.collect:
gc.collect()
self.garbage = gc.garbage[:]
del gc.garbage[:]
# only yield if there's more time-consuming work to do,
# if there's no garbage, give instant feedback
if len(self.garbage) > 0:
yield None
# don't repr the garbage list if we don't have to
if self.notify.getDebug():
self.notify.debug('self.garbage == %s' % fastRepr(self.garbage))
gc.set_debug(oldFlags)
self.numGarbage = len(self.garbage)
# only yield if there's more time-consuming work to do,
# if there's no garbage, give instant feedback
if self.numGarbage > 0:
yield None
if self._args.verbose:
self.notify.info('found %s garbage items' % self.numGarbage)
""" spammy
# print the types of the garbage first, in case the repr of an object
# causes a crash
if self.numGarbage > 0:
self.notify.info('TYPES ONLY (this is only needed if a crash occurs before GarbageReport finishes):')
for result in printNumberedTypesGen(self.garbage):
yield None
"""
# Py obj id -> garbage list index
self._id2index = {}
self.referrersByReference = {}
self.referrersByNumber = {}
self.referentsByReference = {}
self.referentsByNumber = {}
self._id2garbageInfo = {}
self.cycles = []
self.cyclesBySyntax = []
self.uniqueCycleSets = set()
self.cycleIds = set()
# make the id->index table to speed up the next steps
for i in xrange(self.numGarbage):
self._id2index[id(self.garbage[i])] = i
if not (i % 20):
yield None
# grab the referrers (pointing to garbage)
if self._args.fullReport and (self.numGarbage != 0):
if self._args.verbose:
self.notify.info('getting referrers...')
for i in xrange(self.numGarbage):
yield None
for result in self._getReferrers(self.garbage[i]):
yield None
byNum, byRef = result
self.referrersByNumber[i] = byNum
self.referrersByReference[i] = byRef
# grab the referents (pointed to by garbage)
if self.numGarbage > 0:
if self._args.verbose:
self.notify.info('getting referents...')
for i in xrange(self.numGarbage):
yield None
for result in self._getReferents(self.garbage[i]):
yield None
byNum, byRef = result
self.referentsByNumber[i] = byNum
self.referentsByReference[i] = byRef
for i in xrange(self.numGarbage):
if hasattr(self.garbage[i], '_garbageInfo') and callable(self.garbage[i]._garbageInfo):
try:
info = self.garbage[i]._garbageInfo()
except Exception as e:
info = str(e)
self._id2garbageInfo[id(self.garbage[i])] = info
yield None
else:
if not (i % 20):
yield None
# find the cycles
if self._args.findCycles and self.numGarbage > 0:
if self._args.verbose:
self.notify.info('calculating cycles...')
for i in xrange(self.numGarbage):
yield None
for newCycles in self._getCycles(i, self.uniqueCycleSets):
yield None
self.cycles.extend(newCycles)
# create a representation of the cycle in human-readable form
newCyclesBySyntax = []
for cycle in newCycles:
cycleBySyntax = ''
objs = []
# leave off the last index, it's a repeat of the first index
for index in cycle[:-1]:
objs.append(self.garbage[index])
yield None
# make the list repeat so we can safely iterate off the end
numObjs = len(objs) - 1
objs.extend(objs)
# state variables for our loop below
numToSkip = 0
objAlreadyRepresented = False
# if cycle starts off with an instance dict, start with the instance instead
startIndex = 0
# + 1 to include a reference back to the first object
endIndex = numObjs + 1
if type(objs[-1]) is types.InstanceType and type(objs[0]) is dict:
startIndex -= 1
endIndex -= 1
for index in xrange(startIndex, endIndex):
if numToSkip:
numToSkip -= 1
continue
obj = objs[index]
if type(obj) is types.InstanceType:
if not objAlreadyRepresented:
cycleBySyntax += '%s' % obj.__class__.__name__
cycleBySyntax += '.'
# skip past the instance dict and get the member obj
numToSkip += 1
member = objs[index+2]
for key, value in obj.__dict__.items():
if value is member:
break
yield None
else:
key = '<unknown member name>'
cycleBySyntax += '%s' % key
objAlreadyRepresented = True
elif type(obj) is dict:
cycleBySyntax += '{'
# get object referred to by dict
val = objs[index+1]
for key, value in obj.items():
if value is val:
break
yield None
else:
key = '<unknown key>'
cycleBySyntax += '%s}' % fastRepr(key)
objAlreadyRepresented = True
elif type(obj) in (tuple, list):
brackets = {
tuple: '()',
list: '[]',
}[type(obj)]
# get object being referenced by container
nextObj = objs[index+1]
cycleBySyntax += brackets[0]
for index in xrange(len(obj)):
if obj[index] is nextObj:
index = str(index)
break
yield None
else:
index = '<unknown index>'
cycleBySyntax += '%s%s' % (index, brackets[1])
objAlreadyRepresented = True
else:
cycleBySyntax += '%s --> ' % itype(obj)
objAlreadyRepresented = False
newCyclesBySyntax.append(cycleBySyntax)
yield None
self.cyclesBySyntax.extend(newCyclesBySyntax)
# if we're not doing a full report, add this cycle's IDs to the master set
if not self._args.fullReport:
for cycle in newCycles:
yield None
self.cycleIds.update(set(cycle))
self.numCycles = len(self.cycles)
if self._args.findCycles:
s = ['===== GarbageReport: \'%s\' (%s %s) =====' % (
self._args.name, self.numCycles,
('cycle' if self.numCycles == 1 else 'cycles'))]
else:
s = ['===== GarbageReport: \'%s\' =====' % (
self._args.name)]
if self.numGarbage > 0:
# make a list of the ids we will actually be printing
if self._args.fullReport:
garbageIndices = range(self.numGarbage)
else:
garbageIndices = list(self.cycleIds)
garbageIndices.sort()
numGarbage = len(garbageIndices)
# log each individual item with a number in front of it
if not self._args.fullReport:
abbrev = '(abbreviated) '
else:
abbrev = ''
s.append('===== Garbage Items %s=====' % abbrev)
digits = 0
n = numGarbage
while n > 0:
yield None
digits += 1
n /= 10
digits = digits
format = '%0' + '%s' % digits + 'i:%s \t%s'
for i in xrange(numGarbage):
yield None
idx = garbageIndices[i]
if self._args.safeMode:
# in safe mode, don't try to repr any of the objects
objStr = repr(itype(self.garbage[idx]))
else:
objStr = fastRepr(self.garbage[idx])
maxLen = 5000
if len(objStr) > maxLen:
snip = '<SNIP>'
objStr = '%s%s' % (objStr[:(maxLen-len(snip))], snip)
s.append(format % (idx, itype(self.garbage[idx]), objStr))
# also log the types of the objects
s.append('===== Garbage Item Types %s=====' % abbrev)
for i in xrange(numGarbage):
yield None
idx = garbageIndices[i]
objStr = str(deeptype(self.garbage[idx]))
maxLen = 5000
if len(objStr) > maxLen:
snip = '<SNIP>'
objStr = '%s%s' % (objStr[:(maxLen-len(snip))], snip)
s.append(format % (idx, itype(self.garbage[idx]), objStr))
if self._args.findCycles:
s.append('===== Garbage Cycles (Garbage Item Numbers) =====')
ac = AlphabetCounter()
for i in xrange(self.numCycles):
yield None
s.append('%s:%s' % (ac.next(), self.cycles[i]))
if self._args.findCycles:
s.append('===== Garbage Cycles (Python Syntax) =====')
ac = AlphabetCounter()
for i in xrange(len(self.cyclesBySyntax)):
yield None
s.append('%s:%s' % (ac.next(), self.cyclesBySyntax[i]))
if len(self._id2garbageInfo):
s.append('===== Garbage Custom Info =====')
ac = AlphabetCounter()
for i in xrange(len(self.cyclesBySyntax)):
yield None
counter = ac.next()
_id = id(self.garbage[i])
if _id in self._id2garbageInfo:
s.append('%s:%s' % (counter, self._id2garbageInfo[_id]))
if self._args.fullReport:
format = '%0' + '%s' % digits + 'i:%s'
s.append('===== Referrers By Number (what is referring to garbage item?) =====')
for i in xrange(numGarbage):
yield None
s.append(format % (i, self.referrersByNumber[i]))
s.append('===== Referents By Number (what is garbage item referring to?) =====')
for i in xrange(numGarbage):
yield None
s.append(format % (i, self.referentsByNumber[i]))
s.append('===== Referrers (what is referring to garbage item?) =====')
for i in xrange(numGarbage):
yield None
s.append(format % (i, self.referrersByReference[i]))
s.append('===== Referents (what is garbage item referring to?) =====')
for i in xrange(numGarbage):
yield None
s.append(format % (i, self.referentsByReference[i]))
self._report = s
if self._args.log:
self.printingBegin()
for i in xrange(len(self._report)):
if self.numGarbage > 0:
yield None
self.notify.info(self._report[i])
self.notify.info('===== Garbage Report Done =====')
self.printingEnd()
yield Job.Done
def _excepthookDumpVars(eType, eValue, tb):
excStrs = traceback.format_exception(eType, eValue, tb)
s = 'printing traceback in case variable repr crashes the process...\n'
for excStr in excStrs:
s += excStr
notify.info(s)
s = 'DUMPING STACK FRAME VARIABLES'
origTb = tb
#import pdb;pdb.set_trace()
#foundRun = False
foundRun = True
while tb is not None:
frame = tb.tb_frame
code = frame.f_code
# this is a list of every string identifier used in this stack frame's code
codeNames = set(code.co_names)
# skip everything before the 'run' method, those frames have lots of
# not-useful information
if not foundRun:
if code.co_name == 'run':
foundRun = True
else:
tb = tb.tb_next
continue
s += '\n File "%s", line %s, in %s' % (
code.co_filename, frame.f_lineno, code.co_name)
stateStack = Stack()
# prime the stack with the variables we should visit from the frame's data structures
# grab all of the local, builtin and global variables that appear in the code's name list
name2obj = {}
for name, obj in frame.f_builtins.items():
if name in codeNames:
name2obj[name] = obj
for name, obj in frame.f_globals.items():
if name in codeNames:
name2obj[name] = obj
for name, obj in frame.f_locals.items():
if name in codeNames:
name2obj[name] = obj
# show them in alphabetical order
names = name2obj.keys()
names.sort()
# push them in reverse order so they'll be popped in the correct order
names.reverse()
traversedIds = set()
for name in names:
stateStack.push([name, name2obj[name], traversedIds])
while len(stateStack) > 0:
name, obj, traversedIds = stateStack.pop()
#notify.info('%s, %s, %s' % (name, fastRepr(obj), traversedIds))
r = fastRepr(obj, maxLen=10)
if type(r) is types.StringType:
r = r.replace('\n', '\\n')
s += '\n %s = %s' % (name, r)
# if we've already traversed through this object, don't traverse through it again
if id(obj) not in traversedIds:
attrName2obj = {}
for attrName in codeNames:
attr = getattr(obj, attrName, _AttrNotFound)
if (attr is not _AttrNotFound):
# prevent infinite recursion on method wrappers (__init__.__init__.__init__...)
try:
className = attr.__class__.__name__
except:
pass
else:
if className == 'method-wrapper':
continue
attrName2obj[attrName] = attr
if len(attrName2obj):
# show them in alphabetical order
attrNames = attrName2obj.keys()
attrNames.sort()
# push them in reverse order so they'll be popped in the correct order
attrNames.reverse()
ids = set(traversedIds)
ids.add(id(obj))
for attrName in attrNames:
obj = attrName2obj[attrName]
stateStack.push(['%s.%s' % (name, attrName), obj, ids])
tb = tb.tb_next
if foundRun:
s += '\n'
notify.info(s)
oldExcepthook(eType, eValue, origTb)
def run(self):
try:
self._leakDetector._index2containerId2len[self._index] = {}
ids = self._leakDetector.getContainerIds()
# record the current len of each container
for objId in ids:
yield None
try:
for result in self._leakDetector.getContainerByIdGen(objId):
yield None
container = result
except Exception as e:
# this container no longer exists
if self.notify.getDebug():
for contName in self._leakDetector.getContainerNameByIdGen(objId):
yield None
self.notify.debug(
'%s no longer exists; caught exception in getContainerById (%s)' % (
contName, e))
self._leakDetector.removeContainerById(objId)
continue
if container is None:
# this container no longer exists
if self.notify.getDebug():
for contName in self._leakDetector.getContainerNameByIdGen(objId):
yield None
self.notify.debug('%s no longer exists; getContainerById returned None' %
contName)
self._leakDetector.removeContainerById(objId)
continue
try:
cLen = len(container)
except Exception as e:
# this container no longer exists
if self.notify.getDebug():
for contName in self._leakDetector.getContainerNameByIdGen(objId):
yield None
self.notify.debug(
'%s is no longer a container, it is now %s (%s)' %
(contName, safeRepr(container), e))
self._leakDetector.removeContainerById(objId)
continue
self._leakDetector._index2containerId2len[self._index][objId] = cLen
# compare the current len of each container to past lens
if self._index > 0:
idx2id2len = self._leakDetector._index2containerId2len
for objId in idx2id2len[self._index]:
yield None
if objId in idx2id2len[self._index-1]:
diff = idx2id2len[self._index][objId] - idx2id2len[self._index-1][objId]
"""
# this check is too spammy
if diff > 20:
if diff > idx2id2len[self._index-1][objId]:
minutes = (self._leakDetector._index2delay[self._index] -
self._leakDetector._index2delay[self._index-1]) / 60.
name = self._leakDetector.getContainerNameById(objId)
if idx2id2len[self._index-1][objId] != 0:
percent = 100. * (float(diff) / float(idx2id2len[self._index-1][objId]))
try:
for container in self._leakDetector.getContainerByIdGen(objId):
yield None
except:
# TODO
self.notify.debug('caught exception in getContainerByIdGen (1)')
else:
self.notify.warning(
'%s (%s) grew %.2f%% in %.2f minutes (%s items at last measurement, current contents: %s)' % (
name, itype(container), percent, minutes, idx2id2len[self._index][objId],
fastRepr(container, maxLen=CheckContainers.ReprItems)))
yield None
"""
if (self._index > 2 and
objId in idx2id2len[self._index-2] and
objId in idx2id2len[self._index-3]):
diff2 = idx2id2len[self._index-1][objId] - idx2id2len[self._index-2][objId]
diff3 = idx2id2len[self._index-2][objId] - idx2id2len[self._index-3][objId]
if self._index <= 4:
if diff > 0 and diff2 > 0 and diff3 > 0:
name = self._leakDetector.getContainerNameById(objId)
try:
for container in self._leakDetector.getContainerByIdGen(objId):
yield None
except:
# TODO
self.notify.debug('caught exception in getContainerByIdGen (2)')
else:
msg = ('%s (%s) consistently increased in size over the last '
'3 periods (%s items at last measurement, current contents: %s)' %
(name, itype(container), idx2id2len[self._index][objId],
fastRepr(container, maxLen=CheckContainers.ReprItems)))
self.notify.warning(msg)
yield None
elif (objId in idx2id2len[self._index-4] and
objId in idx2id2len[self._index-5]):
# if size has consistently increased over the last 5 checks,
# send out a warning
diff4 = idx2id2len[self._index-3][objId] - idx2id2len[self._index-4][objId]
diff5 = idx2id2len[self._index-4][objId] - idx2id2len[self._index-5][objId]
if diff > 0 and diff2 > 0 and diff3 > 0 and diff4 > 0 and diff5 > 0:
name = self._leakDetector.getContainerNameById(objId)
try:
for container in self._leakDetector.getContainerByIdGen(objId):
yield None
except:
# TODO
self.notify.debug('caught exception in getContainerByIdGen (3)')
else:
msg = ('leak detected: %s (%s) consistently increased in size over the last '
'5 periods (%s items at last measurement, current contents: %s)' %
(name, itype(container), idx2id2len[self._index][objId],
fastRepr(container, maxLen=CheckContainers.ReprItems)))
self.notify.warning(msg)
yield None
messenger.send(self._leakDetector.getLeakEvent(), [container, name])
if config.GetBool('pdb-on-leak-detect', 0):
import pdb;pdb.set_trace()
pass
except Exception as e:
print('CheckContainers job caught exception: %s' % e)
if __dev__:
raise
yield Job.Done
