def __init__(self, evalStr=None, dictKey=NoDictKey):
# if this is a dictionary lookup, pass dictKey instead of evalStr
self.evalStr = evalStr
self.dictKey = NoDictKey
# is the dictKey a weak reference?
self._isWeakRef = False
self._refCount = 0
if dictKey is not NoDictKey:
# if we can repr/eval the key, store it as an evalStr
keyRepr = safeRepr(dictKey)
useEval = False
try:
keyEval = eval(keyRepr)
useEval = True
except:
pass
if useEval:
# check to make sure the eval succeeded
if hash(keyEval) != hash(dictKey):
useEval = False
if useEval:
# eval/repr succeeded, store as an evalStr
self.evalStr = '[%s]' % keyRepr
else:
try:
# store a weakref to the key
self.dictKey = weakref.ref(dictKey)
self._isWeakRef = True
except TypeError, e:
ContainerLeakDetector.notify.debug('could not weakref dict key %s' % keyRepr)
self.dictKey = dictKey
self._isWeakRef = False
def __init__(self, evalStr = None, dictKey = NoDictKey):
self.evalStr = evalStr
self.dictKey = NoDictKey
self._isWeakRef = False
self._refCount = 0
if dictKey is not NoDictKey:
keyRepr = safeRepr(dictKey)
useEval = False
try:
keyEval = eval(keyRepr)
useEval = True
except:
pass
if useEval:
if hash(keyEval) != hash(dictKey):
useEval = False
if useEval:
self.evalStr = '[%s]' % keyRepr
else:
try:
self.dictKey = weakref.ref(dictKey)
self._isWeakRef = True
except TypeError:
e = None
ContainerLeakDetector.notify.debug('could not weakref dict key %s' % keyRepr)
self.dictKey = dictKey
self._isWeakRef = False
def __init__(self, evalStr=None, dictKey=NoDictKey):
# if this is a dictionary lookup, pass dictKey instead of evalStr
self.evalStr = evalStr
self.dictKey = NoDictKey
# is the dictKey a weak reference?
self._isWeakRef = False
self._refCount = 0
if dictKey is not NoDictKey:
# if we can repr/eval the key, store it as an evalStr
keyRepr = safeRepr(dictKey)
useEval = False
try:
keyEval = eval(keyRepr)
useEval = True
except:
pass
if useEval:
# check to make sure the eval succeeded
if hash(keyEval) != hash(dictKey):
useEval = False
if useEval:
# eval/repr succeeded, store as an evalStr
self.evalStr = '[%s]' % keyRepr
else:
try:
# store a weakref to the key
self.dictKey = weakref.ref(dictKey)
self._isWeakRef = True
except TypeError as e:
ContainerLeakDetector.notify.debug('could not weakref dict key %s' % keyRepr)
self.dictKey = dictKey
self._isWeakRef = False
def flush(self):
"""
Delete each item in the cache then clear all references to them
"""
assert self.checkCache()
CRCache.notify.debug("Flushing the cache")
# NOTE: delayDeleted objects should no longer get into the cache in the first place
# give objects a chance to clean themselves up before checking for DelayDelete leaks
messenger.send('clientCleanup')
# some of these objects might be holding delayDeletes on others
# track each object that is delayDeleted after it gets its chance to delete,
# and check them after all objects have had a chance to delete
delayDeleted = []
for distObj in self.dict.values():
distObj.deleteOrDelay()
if distObj.getDelayDeleteCount() != 0:
delayDeleted.append(distObj)
if distObj.getDelayDeleteCount() <= 0:
# make sure we're not leaking
distObj.detectLeaks()
# now that all objects have had a chance to delete, are there any objects left
# that are still delayDeleted?
delayDeleteLeaks = []
for distObj in delayDeleted:
if distObj.getDelayDeleteCount() != 0:
delayDeleteLeaks.append(distObj)
if len(delayDeleteLeaks) > 0:
s = 'CRCache.flush:'
for obj in delayDeleteLeaks:
s += ('\n could not delete %s (%s), delayDeletes=%s' %
(safeRepr(obj), itype(obj), obj.getDelayDeleteNames()))
self.notify.error(s)
# Null out all references to the objects so they will get gcd
self.dict = {}
self.fifo = []
def printDelayDeletes(self):
print('DelayDeletes:')
print('=============')
for obj in self._delayDeletedDOs.values():
print('%s\t%s (%s)\tdelayDeletes=%s' %
(obj.doId, safeRepr(obj), itype(obj),
obj.getDelayDeleteNames()))
def printReferrers(self, numEach = 3):
counts = list(set(self._count2types.keys()))
counts.sort()
counts.reverse()
for count in counts:
types = makeList(self._count2types[count])
for typ in types:
print '\n\nTYPE: %s' % repr(typ)
for i in xrange(min(numEach, len(self._type2objs[typ]))):
obj = self._type2objs[typ][i]
print '\nOBJ: %s\n' % safeRepr(obj)
referrers = gc.get_referrers(obj)
print '%s REFERRERS:\n' % len(referrers)
if len(referrers):
print getNumberedTypedString(referrers, maxLen = 80, numPrefix = 'REF')
continue
print '<No Referrers>'
def printReferrers(self, numEach=3):
"""referrers of the first few of each type of object"""
counts = list(set(self._count2types.keys()))
counts.sort()
counts.reverse()
for count in counts:
types = makeList(self._count2types[count])
for typ in types:
print "\n\nTYPE: %s" % repr(typ)
for i in xrange(min(numEach, len(self._type2objs[typ]))):
obj = self._type2objs[typ][i]
print "\nOBJ: %s\n" % safeRepr(obj)
referrers = gc.get_referrers(obj)
print "%s REFERRERS:\n" % len(referrers)
if len(referrers):
print getNumberedTypedString(referrers, maxLen=80, numPrefix="REF")
else:
print "<No Referrers>"
def run(self):
try:
self._leakDetector._index2containerId2len[self._index] = {}
ids = self._leakDetector.getContainerIds()
# record the current len of each container
for objId in ids:
yield None
try:
for result in self._leakDetector.getContainerByIdGen(objId):
yield None
container = result
except Exception, e:
# this container no longer exists
if self.notify.getDebug():
for contName in self._leakDetector.getContainerNameByIdGen(objId):
yield None
self.notify.debug(
'%s no longer exists; caught exception in getContainerById (%s)' % (
contName, e))
self._leakDetector.removeContainerById(objId)
continue
if container is None:
# this container no longer exists
if self.notify.getDebug():
for contName in self._leakDetector.getContainerNameByIdGen(objId):
yield None
self.notify.debug('%s no longer exists; getContainerById returned None' %
contName)
self._leakDetector.removeContainerById(objId)
continue
try:
cLen = len(container)
except Exception, e:
# this container no longer exists
if self.notify.getDebug():
for contName in self._leakDetector.getContainerNameByIdGen(objId):
yield None
self.notify.debug(
'%s is no longer a container, it is now %s (%s)' %
(contName, safeRepr(container), e))
self._leakDetector.removeContainerById(objId)
continue
self._leakDetector._index2containerId2len[self._index][objId] = cLen
def printReferrers(self, numEach=3):
"""referrers of the first few of each type of object"""
counts = list(set(self._count2types.keys()))
counts.sort()
counts.reverse()
for count in counts:
types = makeList(self._count2types[count])
for typ in types:
print('\n\nTYPE: %s' % repr(typ))
for i in range(min(numEach, len(self._type2objs[typ]))):
obj = self._type2objs[typ][i]
print('\nOBJ: %s\n' % safeRepr(obj))
referrers = gc.get_referrers(obj)
print('%s REFERRERS:\n' % len(referrers))
if len(referrers):
print(getNumberedTypedString(referrers, maxLen=80,
numPrefix='REF'))
else:
print('<No Referrers>')
def run(self):
try:
self._leakDetector._index2containerId2len[self._index] = {}
ids = self._leakDetector.getContainerIds()
# record the current len of each container
for objId in ids:
yield None
try:
for result in self._leakDetector.getContainerByIdGen(objId):
yield None
container = result
except Exception, e:
# this container no longer exists
if self.notify.getDebug():
for contName in self._leakDetector.getContainerNameByIdGen(objId):
yield None
self.notify.debug(
'%s no longer exists; caught exception in getContainerById (%s)' % (
contName, e))
self._leakDetector.removeContainerById(objId)
continue
if container is None:
# this container no longer exists
if self.notify.getDebug():
for contName in self._leakDetector.getContainerNameByIdGen(objId):
yield None
self.notify.debug('%s no longer exists; getContainerById returned None' %
contName)
self._leakDetector.removeContainerById(objId)
continue
try:
cLen = len(container)
except Exception, e:
# this container no longer exists
if self.notify.getDebug():
for contName in self._leakDetector.getContainerNameByIdGen(objId):
yield None
self.notify.debug(
'%s is no longer a container, it is now %s (%s)' %
(contName, safeRepr(container), e))
self._leakDetector.removeContainerById(objId)
continue
self._leakDetector._index2containerId2len[self._index][objId] = cLen
def getString(self, prevIndirection=None, nextIndirection=None):
# return our contribution to the full name of an object
instanceDictStr = '.__dict__'
if self.evalStr is not None:
# if we're an instance dict, skip over this one (obj.__dict__[keyName] == obj.keyName)
if nextIndirection is not None and self.evalStr[-len(instanceDictStr):] == instanceDictStr:
return self.evalStr[:-len(instanceDictStr)]
# if the previous indirection was an instance dict, change our syntax from ['key'] to .key
if prevIndirection is not None and prevIndirection.evalStr is not None:
if prevIndirection.evalStr[-len(instanceDictStr):] == instanceDictStr:
return '.%s' % self.evalStr[2:-2]
return self.evalStr
# we're stored as a dict key
keyRepr = safeRepr(self._getNonWeakDictKey())
# if the previous indirection was an instance dict, change our syntax from ['key'] to .key
if prevIndirection is not None and prevIndirection.evalStr is not None:
if prevIndirection.evalStr[-len(instanceDictStr):] == instanceDictStr:
return '.%s' % keyRepr
return '[%s]' % keyRepr
def getString(self, prevIndirection=None, nextIndirection=None):
# return our contribution to the full name of an object
instanceDictStr = '.__dict__'
if self.evalStr is not None:
# if we're an instance dict, skip over this one (obj.__dict__[keyName] == obj.keyName)
if nextIndirection is not None and self.evalStr[-len(instanceDictStr):] == instanceDictStr:
return self.evalStr[:-len(instanceDictStr)]
# if the previous indirection was an instance dict, change our syntax from ['key'] to .key
if prevIndirection is not None and prevIndirection.evalStr is not None:
if prevIndirection.evalStr[-len(instanceDictStr):] == instanceDictStr:
return '.%s' % self.evalStr[2:-2]
return self.evalStr
# we're stored as a dict key
keyRepr = safeRepr(self._getNonWeakDictKey())
# if the previous indirection was an instance dict, change our syntax from ['key'] to .key
if prevIndirection is not None and prevIndirection.evalStr is not None:
if prevIndirection.evalStr[-len(instanceDictStr):] == instanceDictStr:
return '.%s' % keyRepr
return '[%s]' % keyRepr
def getString(self, prevIndirection = None, nextIndirection = None):
instanceDictStr = '.__dict__'
if self.evalStr is not None:
if nextIndirection is not None and self.evalStr[-len(instanceDictStr):] == instanceDictStr:
return self.evalStr[:-len(instanceDictStr)]
if prevIndirection is not None and prevIndirection.evalStr is not None:
if prevIndirection.evalStr[-len(instanceDictStr):] == instanceDictStr:
return '.%s' % self.evalStr[2:-2]
return self.evalStr
keyRepr = safeRepr(self._getNonWeakDictKey())
if prevIndirection is not None and prevIndirection.evalStr is not None:
if prevIndirection.evalStr[-len(instanceDictStr):] == instanceDictStr:
return '.%s' % keyRepr
return '[%s]' % keyRepr
def leakContainer(task=None):
base = getBase()
if not hasattr(base, 'leakContainer'):
base.leakContainer = {}
# use tuples as keys since they can't be weakref'd, and use an instance
# since it can't be repr/eval'd
# that will force the leak detector to hold a normal 'non-weak' reference
class LeakKey:
pass
base.leakContainer[(LeakKey(),)] = {}
# test the non-weakref object reference handling
if random.random() < .01:
key = random.choice(list(base.leakContainer.keys()))
ContainerLeakDetector.notify.debug(
'removing reference to leakContainer key %s so it will be garbage-collected' % safeRepr(key))
del base.leakContainer[key]
taskMgr.doMethodLater(10, leakContainer, 'leakContainer-%s' % serialNum())
if task:
return task.done
def run(self):
ContainerReport.PrivateIds.update(set([
id(ContainerReport.PrivateIds),
id(self._visitedIds),
id(self._id2pathStr),
id(self._id2container),
id(self._type2id2len),
id(self._queue),
id(self._instanceDictIds),
]))
# push on a few things that we want to give priority
# for the sake of the variable-name printouts
try:
base
except:
pass
else:
self._enqueueContainer( base.__dict__,
'base')
try:
simbase
except:
pass
else:
self._enqueueContainer( simbase.__dict__,
'simbase')
self._queue.push(__builtins__)
self._id2pathStr[id(__builtins__)] = ''
while len(self._queue) > 0:
# yield up here instead of at the end, since we skip back to the
# top of the while loop from various points
yield None
parentObj = self._queue.pop()
#print '%s: %s, %s' % (id(parentObj), type(parentObj), self._id2pathStr[id(parentObj)])
isInstanceDict = False
if id(parentObj) in self._instanceDictIds:
isInstanceDict = True
try:
if parentObj.__class__.__name__ == 'method-wrapper':
continue
except:
pass
if type(parentObj) in (types.StringType, types.UnicodeType):
continue
if type(parentObj) in (types.ModuleType, types.InstanceType):
child = parentObj.__dict__
if self._examine(child):
assert (self._queue.back() is child)
self._instanceDictIds.add(id(child))
self._id2pathStr[id(child)] = str(self._id2pathStr[id(parentObj)])
continue
if type(parentObj) is dict:
key = None
attr = None
keys = list(parentObj.keys())
try:
keys.sort()
except TypeError as e:
self.notify.warning('non-sortable dict keys: %s: %s' % (self._id2pathStr[id(parentObj)], repr(e)))
for key in keys:
try:
attr = parentObj[key]
except KeyError as e:
self.notify.warning('could not index into %s with key %s' % (self._id2pathStr[id(parentObj)],
key))
if id(attr) not in self._visitedIds:
self._visitedIds.add(id(attr))
if self._examine(attr):
assert (self._queue.back() is attr)
if parentObj is __builtins__:
self._id2pathStr[id(attr)] = key
else:
if isInstanceDict:
self._id2pathStr[id(attr)] = self._id2pathStr[id(parentObj)] + '.%s' % key
else:
self._id2pathStr[id(attr)] = self._id2pathStr[id(parentObj)] + '[%s]' % safeRepr(key)
del key
del attr
continue
if type(parentObj) is not types.FileType:
try:
itr = iter(parentObj)
except:
pass
else:
try:
index = 0
while 1:
try:
attr = next(itr)
except:
# some custom classes don't do well when iterated
attr = None
break
if id(attr) not in self._visitedIds:
self._visitedIds.add(id(attr))
if self._examine(attr):
assert (self._queue.back() is attr)
self._id2pathStr[id(attr)] = self._id2pathStr[id(parentObj)] + '[%s]' % index
index += 1
del attr
except StopIteration as e:
pass
del itr
continue
try:
childNames = dir(parentObj)
except:
pass
else:
childName = None
child = None
for childName in childNames:
child = getattr(parentObj, childName)
if id(child) not in self._visitedIds:
self._visitedIds.add(id(child))
if self._examine(child):
assert (self._queue.back() is child)
self._id2pathStr[id(child)] = self._id2pathStr[id(parentObj)] + '.%s' % childName
del childName
del child
continue
if self._log:
self.printingBegin()
for i in self._output(limit=self._limit):
yield None
self.printingEnd()
yield Job.Done
def run(self):
try:
self._leakDetector._index2containerId2len[self._index] = { }
ids = self._leakDetector.getContainerIds()
for objId in ids:
yield None
try:
for result in self._leakDetector.getContainerByIdGen(objId):
yield None
container = result
except Exception:
e = None
if self.notify.getDebug():
for contName in self._leakDetector.getContainerNameByIdGen(objId):
yield None
self.notify.debug('%s no longer exists; caught exception in getContainerById (%s)' % (contName, e))
self._leakDetector.removeContainerById(objId)
continue
if container is None:
if self.notify.getDebug():
for contName in self._leakDetector.getContainerNameByIdGen(objId):
yield None
self.notify.debug('%s no longer exists; getContainerById returned None' % contName)
self._leakDetector.removeContainerById(objId)
continue
try:
cLen = len(container)
except Exception:
e = None
if self.notify.getDebug():
for contName in self._leakDetector.getContainerNameByIdGen(objId):
yield None
self.notify.debug('%s is no longer a container, it is now %s (%s)' % (contName, safeRepr(container), e))
self._leakDetector.removeContainerById(objId)
continue
self._leakDetector._index2containerId2len[self._index][objId] = cLen
if self._index > 0:
idx2id2len = self._leakDetector._index2containerId2len
for objId in idx2id2len[self._index]:
yield None
if objId in idx2id2len[self._index - 1]:
diff = idx2id2len[self._index][objId] - idx2id2len[self._index - 1][objId]
if self._index > 2 and objId in idx2id2len[self._index - 2] and objId in idx2id2len[self._index - 3]:
diff2 = idx2id2len[self._index - 1][objId] - idx2id2len[self._index - 2][objId]
diff3 = idx2id2len[self._index - 2][objId] - idx2id2len[self._index - 3][objId]
if self._index <= 4:
if diff > 0 and diff2 > 0 and diff3 > 0:
name = self._leakDetector.getContainerNameById(objId)
try:
for container in self._leakDetector.getContainerByIdGen(objId):
yield None
except:
self.notify.debug('caught exception in getContainerByIdGen (2)')
msg = '%s (%s) consistently increased in size over the last 3 periods (%s items at last measurement, current contents: %s)' % (name, itype(container), idx2id2len[self._index][objId], fastRepr(container, maxLen = CheckContainers.ReprItems))
self.notify.warning(msg)
yield None
elif objId in idx2id2len[self._index - 4] and objId in idx2id2len[self._index - 5]:
diff4 = idx2id2len[self._index - 3][objId] - idx2id2len[self._index - 4][objId]
diff5 = idx2id2len[self._index - 4][objId] - idx2id2len[self._index - 5][objId]
if diff > 0 and diff2 > 0 and diff3 > 0 and diff4 > 0 and diff5 > 0:
name = self._leakDetector.getContainerNameById(objId)
try:
for container in self._leakDetector.getContainerByIdGen(objId):
yield None
except:
self.notify.debug('caught exception in getContainerByIdGen (3)')
msg = 'leak detected: %s (%s) consistently increased in size over the last 5 periods (%s items at last measurement, current contents: %s)' % (name, itype(container), idx2id2len[self._index][objId], fastRepr(container, maxLen = CheckContainers.ReprItems))
self.notify.warning(msg)
yield None
messenger.send(self._leakDetector.getLeakEvent(), [
container,
name])
if config.GetBool('pdb-on-leak-detect', 0):
import pdb as pdb
pdb.set_trace()
objId in idx2id2len[self._index - 3]
except Exception:
e = None
print 'CheckContainers job caught exception: %s' % e
if __dev__:
raise
yield Job.Done
def run(self):
ContainerReport.PrivateIds.update(
set([
id(ContainerReport.PrivateIds),
id(self._visitedIds),
id(self._id2pathStr),
id(self._id2container),
id(self._type2id2len),
id(self._queue),
id(self._instanceDictIds)
]))
try:
pass
except:
pass
self._enqueueContainer(base.__dict__, 'base')
try:
pass
except:
pass
self._enqueueContainer(simbase.__dict__, 'simbase')
self._queue.push(__builtins__)
self._id2pathStr[id(__builtins__)] = ''
while len(self._queue) > 0:
yield None
parentObj = self._queue.pop()
isInstanceDict = False
if id(parentObj) in self._instanceDictIds:
isInstanceDict = True
try:
if parentObj.__class__.__name__ == 'method-wrapper':
continue
except:
pass
if type(parentObj) in (types.StringType, types.UnicodeType):
continue
if type(parentObj) in (types.ModuleType, types.InstanceType):
child = parentObj.__dict__
if self._examine(child):
self._instanceDictIds.add(id(child))
self._id2pathStr[id(child)] = str(
self._id2pathStr[id(parentObj)])
continue
continue
if type(parentObj) is types.DictType:
key = None
attr = None
keys = parentObj.keys()
try:
keys.sort()
except TypeError:
e = None
self.notify.warning(
'non-sortable dict keys: %s: %s' %
(self._id2pathStr[id(parentObj)], repr(e)))
for key in keys:
try:
attr = parentObj[key]
except KeyError:
e = None
self.notify.warning(
'could not index into %s with key %s' %
(self._id2pathStr[id(parentObj)], key))
if id(attr) not in self._visitedIds:
self._visitedIds.add(id(attr))
if self._examine(attr):
if parentObj is __builtins__:
self._id2pathStr[id(attr)] = key
elif isInstanceDict:
self._id2pathStr[id(attr)] = self._id2pathStr[
id(parentObj)] + '.%s' % key
else:
self._id2pathStr[id(attr)] = self._id2pathStr[
id(parentObj)] + '[%s]' % safeRepr(key)
self._examine(attr)
del key
del attr
continue
if type(parentObj) is not types.FileType:
try:
itr = iter(parentObj)
except:
pass
try:
index = 0
while None:
try:
attr = itr.next()
except:
attr = None
break
if id(attr) not in self._visitedIds:
self._visitedIds.add(id(attr))
if self._examine(attr):
self._id2pathStr[id(attr)] = self._id2pathStr[
id(parentObj)] + '[%s]' % index
index += 1
del attr
except StopIteration:
e = None
del itr
continue
try:
childNames = dir(parentObj)
except:
continue
childName = None
child = None
for childName in childNames:
child = getattr(parentObj, childName)
if id(child) not in self._visitedIds:
self._visitedIds.add(id(child))
if self._examine(child):
self._id2pathStr[id(child)] = self._id2pathStr[id(
parentObj)] + '.%s' % childName
self._examine(child)
del childName
del child
continue
if self._log:
self.printingBegin()
for i in self._output(limit=self._limit):
yield None
self.printingEnd()
yield Job.Done
class FindContainers(Job):
def __init__(self, name, leakDetector):
Job.__init__(self, name)
self._leakDetector = leakDetector
self._id2ref = self._leakDetector._id2ref
self._id2baseStartRef = { }
self._id2discoveredStartRef = { }
self._baseStartRefWorkingList = ScratchPad(refGen = nullGen(), source = self._id2baseStartRef)
self._discoveredStartRefWorkingList = ScratchPad(refGen = nullGen(), source = self._id2discoveredStartRef)
self.notify = self._leakDetector.notify
ContainerLeakDetector.addPrivateObj(self.__dict__)
ref = ObjectRef(Indirection(evalStr = '__builtin__.__dict__'), id(__builtin__.__dict__))
self._id2baseStartRef[id(__builtin__.__dict__)] = ref
if not hasattr(__builtin__, 'leakDetectors'):
__builtin__.leakDetectors = { }
ref = ObjectRef(Indirection(evalStr = 'leakDetectors'), id(leakDetectors))
self._id2baseStartRef[id(leakDetectors)] = ref
for i in self._addContainerGen(__builtin__.__dict__, ref):
pass
try:
pass
except:
pass
ref = ObjectRef(Indirection(evalStr = 'base.__dict__'), id(base.__dict__))
self._id2baseStartRef[id(base.__dict__)] = ref
for i in self._addContainerGen(base.__dict__, ref):
pass
try:
pass
except:
pass
ref = ObjectRef(Indirection(evalStr = 'simbase.__dict__'), id(simbase.__dict__))
self._id2baseStartRef[id(simbase.__dict__)] = ref
for i in self._addContainerGen(simbase.__dict__, ref):
pass
def destroy(self):
ContainerLeakDetector.removePrivateObj(self.__dict__)
Job.destroy(self)
def getPriority(self):
return Job.Priorities.Low
def getStartObjAffinity(startObj):
try:
return len(startObj)
except:
return 1
getStartObjAffinity = staticmethod(getStartObjAffinity)
def _isDeadEnd(self, obj, objName = None):
if type(obj) in (types.BooleanType, types.BuiltinFunctionType, types.BuiltinMethodType, types.ComplexType, types.FloatType, types.IntType, types.LongType, types.NoneType, types.NotImplementedType, types.TypeType, types.CodeType, types.FunctionType, types.StringType, types.UnicodeType, types.TupleType):
return True
if id(obj) in ContainerLeakDetector.PrivateIds:
return True
if type(objName) == types.StringType and objName in ('im_self', 'im_class'):
return True
try:
className = obj.__class__.__name__
except:
pass
if className == 'method-wrapper':
return True
return False
def _hasLength(self, obj):
return hasattr(obj, '__len__')
def _addContainerGen(self, cont, objRef):
contId = id(cont)
if contId in self._id2ref:
for existingRepr in self._id2ref[contId].getEvalStrGen():
yield None
for newRepr in objRef.getEvalStrGen():
yield None
if contId not in self._id2ref or len(newRepr) < len(existingRepr):
if contId in self._id2ref:
self._leakDetector.removeContainerById(contId)
self._id2ref[contId] = objRef
def _addDiscoveredStartRef(self, obj, ref):
objId = id(obj)
if objId in self._id2discoveredStartRef:
existingRef = self._id2discoveredStartRef[objId]
if type(existingRef) not in (types.IntType, types.LongType):
if existingRef.getNumIndirections() >= ref.getNumIndirections():
return None
if objId in self._id2ref:
if self._id2ref[objId].getNumIndirections() >= ref.getNumIndirections():
return None
storedItem = ref
if objId in self._id2ref:
storedItem = objId
self._id2discoveredStartRef[objId] = storedItem
def run(self):
try:
workingListSelector = nullGen()
curObjRef = None
while True:
yield None
if curObjRef is None:
try:
startRefWorkingList = workingListSelector.next()
except StopIteration:
baseLen = len(self._baseStartRefWorkingList.source)
discLen = len(self._discoveredStartRefWorkingList.source)
minLen = float(max(1, min(baseLen, discLen)))
minLen *= 3.0
workingListSelector = flywheel([
self._baseStartRefWorkingList,
self._discoveredStartRefWorkingList], [
baseLen / minLen,
discLen / minLen])
yield None
continue
while True:
yield None
try:
curObjRef = startRefWorkingList.refGen.next()
continue
except StopIteration:
if len(startRefWorkingList.source) == 0:
break
for fw in makeFlywheelGen(startRefWorkingList.source.values(), countFunc = lambda x: self.getStartObjAffinity(x), scale = 0.050000000000000003):
yield None
startRefWorkingList.refGen = fw
continue
if curObjRef is None:
continue
if type(curObjRef) in (types.IntType, types.LongType):
startId = curObjRef
curObjRef = None
try:
for containerRef in self._leakDetector.getContainerByIdGen(startId):
yield None
except:
self.notify.debug('invalid startRef, stored as id %s' % startId)
self._leakDetector.removeContainerById(startId)
continue
curObjRef = containerRef
try:
for curObj in curObjRef.getContainerGen():
yield None
except:
self.notify.debug('lost current container, ref.getContainerGen() failed')
curObjRef = None
continue
self.notify.debug('--> %s' % curObjRef)
parentObjRef = curObjRef
curObjRef = None
if hasattr(curObj, '__dict__'):
child = curObj.__dict__
hasLength = self._hasLength(child)
notDeadEnd = not self._isDeadEnd(child)
if hasLength or notDeadEnd:
for goesThrough in parentObjRef.goesThroughGen(child):
pass
if not goesThrough:
objRef = ObjectRef(Indirection(evalStr = '.__dict__'), id(child), parentObjRef)
yield None
if hasLength:
for i in self._addContainerGen(child, objRef):
yield None
if notDeadEnd:
self._addDiscoveredStartRef(child, objRef)
curObjRef = objRef
continue
if type(curObj) is types.DictType:
key = None
attr = None
keys = curObj.keys()
numKeysLeft = len(keys) + 1
for key in keys:
yield None
numKeysLeft -= 1
try:
attr = curObj[key]
except KeyError:
e = None
self.notify.debug('could not index into %s with key %s' % (parentObjRef, safeRepr(key)))
continue
hasLength = self._hasLength(attr)
notDeadEnd = False
if curObjRef is None:
notDeadEnd = not self._isDeadEnd(attr, key)
if hasLength or notDeadEnd:
for goesThrough in parentObjRef.goesThroughGen(curObj[key]):
pass
if not goesThrough:
if curObj is __builtin__.__dict__:
objRef = ObjectRef(Indirection(evalStr = '%s' % key), id(curObj[key]))
else:
objRef = ObjectRef(Indirection(dictKey = key), id(curObj[key]), parentObjRef)
yield None
if hasLength:
for i in self._addContainerGen(attr, objRef):
yield None
if notDeadEnd:
self._addDiscoveredStartRef(attr, objRef)
if curObjRef is None and random.randrange(numKeysLeft) == 0:
curObjRef = objRef
del key
del attr
continue
try:
childNames = dir(curObj)
except:
pass
try:
index = -1
attrs = []
while None:
yield None
try:
attr = itr.next()
except:
attr = None
break
numAttrsLeft = len(attrs) + 1
for attr in attrs:
yield None
index += 1
numAttrsLeft -= 1
hasLength = self._hasLength(attr)
notDeadEnd = False
if curObjRef is None:
notDeadEnd = not self._isDeadEnd(attr)
if hasLength or notDeadEnd:
for goesThrough in parentObjRef.goesThrough(curObj[index]):
pass
if not goesThrough:
objRef = ObjectRef(Indirection(evalStr = '[%s]' % index), id(curObj[index]), parentObjRef)
yield None
if hasLength:
for i in self._addContainerGen(attr, objRef):
yield None
if notDeadEnd:
self._addDiscoveredStartRef(attr, objRef)
if curObjRef is None and random.randrange(numAttrsLeft) == 0:
curObjRef = objRef
del attr
except StopIteration:
e = None
del itr
continue
continue
def leakContainer(task = None):
base = getBase()
if not hasattr(base, 'leakContainer'):
base.leakContainer = { }
class LeakKey:
pass
base.leakContainer[(LeakKey(),)] = { }
if random.random() < 0.01:
key = random.choice(base.leakContainer.keys())
ContainerLeakDetector.notify.debug('removing reference to leakContainer key %s so it will be garbage-collected' % safeRepr(key))
del base.leakContainer[key]
taskMgr.doMethodLater(10, leakContainer, 'leakContainer-%s' % serialNum())
if task:
return task.done
def run(self):
try:
# this yields a different set of start refs every time we start a new traversal
# force creation of a new workingListSelector inside the while loop right off the bat
workingListSelector = nullGen()
# this holds the current step of the current traversal
curObjRef = None
while True:
# yield up here instead of at the end, since we skip back to the
# top of the while loop from various points
yield None
#import pdb;pdb.set_trace()
if curObjRef is None:
# choose an object to start a traversal from
try:
startRefWorkingList = workingListSelector.next()
except StopIteration:
# do relative # of traversals on each set based on how many refs it contains
baseLen = len(self._baseStartRefWorkingList.source)
discLen = len(self._discoveredStartRefWorkingList.source)
minLen = float(max(1, min(baseLen, discLen)))
# this will cut down the traversals of the larger set by 2/3
minLen *= 3.
workingListSelector = flywheel([self._baseStartRefWorkingList, self._discoveredStartRefWorkingList],
[baseLen/minLen, discLen/minLen])
yield None
continue
# grab the next start ref from this sequence and see if it's still valid
while True:
yield None
try:
curObjRef = startRefWorkingList.refGen.next()
break
except StopIteration:
# we've run out of refs, grab a new set
if len(startRefWorkingList.source) == 0:
# ref set is empty, choose another
break
# make a generator that yields containers a # of times that is
# proportional to their length
for fw in makeFlywheelGen(
startRefWorkingList.source.values(),
countFunc=lambda x: self.getStartObjAffinity(x),
scale=.05):
yield None
startRefWorkingList.refGen = fw
if curObjRef is None:
# this ref set is empty, choose another
# the base set should never be empty (__builtin__ etc.)
continue
# do we need to go look up the object in _id2ref? sometimes we do that
# to avoid storing multiple redundant refs to a single item
if type(curObjRef) in (types.IntType, types.LongType):
startId = curObjRef
curObjRef = None
try:
for containerRef in self._leakDetector.getContainerByIdGen(startId):
yield None
except:
# ref is invalid
self.notify.debug('invalid startRef, stored as id %s' % startId)
self._leakDetector.removeContainerById(startId)
continue
curObjRef = containerRef
try:
for curObj in curObjRef.getContainerGen():
yield None
except:
self.notify.debug('lost current container, ref.getContainerGen() failed')
# that container is gone, try again
curObjRef = None
continue
self.notify.debug('--> %s' % curObjRef)
#import pdb;pdb.set_trace()
# store a copy of the current objRef
parentObjRef = curObjRef
# if we hit a dead end, start over from another container
curObjRef = None
if hasattr(curObj, '__dict__'):
child = curObj.__dict__
hasLength = self._hasLength(child)
notDeadEnd = not self._isDeadEnd(child)
if hasLength or notDeadEnd:
# prevent cycles in the references (i.e. base.loader.base)
for goesThrough in parentObjRef.goesThroughGen(child):
# don't yield, container might lose this element
pass
if not goesThrough:
objRef = ObjectRef(Indirection(evalStr='.__dict__'),
id(child), parentObjRef)
yield None
if hasLength:
for i in self._addContainerGen(child, objRef):
yield None
if notDeadEnd:
self._addDiscoveredStartRef(child, objRef)
curObjRef = objRef
continue
if type(curObj) is types.DictType:
key = None
attr = None
keys = curObj.keys()
# we will continue traversing the object graph via one key of the dict,
# choose it at random without taking a big chunk of CPU time
numKeysLeft = len(keys) + 1
for key in keys:
yield None
numKeysLeft -= 1
try:
attr = curObj[key]
except KeyError, e:
# this is OK because we are yielding during the iteration
self.notify.debug('could not index into %s with key %s' % (
parentObjRef, safeRepr(key)))
continue
hasLength = self._hasLength(attr)
notDeadEnd = False
# if we haven't picked the next ref, check if this one is a candidate
if curObjRef is None:
notDeadEnd = not self._isDeadEnd(attr, key)
if hasLength or notDeadEnd:
# prevent cycles in the references (i.e. base.loader.base)
for goesThrough in parentObjRef.goesThroughGen(curObj[key]):
# don't yield, container might lose this element
pass
if not goesThrough:
if curObj is __builtin__.__dict__:
objRef = ObjectRef(Indirection(evalStr='%s' % key),
id(curObj[key]))
else:
objRef = ObjectRef(Indirection(dictKey=key),
id(curObj[key]), parentObjRef)
yield None
if hasLength:
for i in self._addContainerGen(attr, objRef):
yield None
if notDeadEnd:
self._addDiscoveredStartRef(attr, objRef)
if curObjRef is None and random.randrange(numKeysLeft) == 0:
curObjRef = objRef
del key
del attr
continue
try:
childNames = dir(curObj)
except:
pass
else:
try:
index = -1
attrs = []
while 1:
yield None
try:
attr = itr.next()
except:
# some custom classes don't do well when iterated
attr = None
break
attrs.append(attr)
# we will continue traversing the object graph via one attr,
# choose it at random without taking a big chunk of CPU time
numAttrsLeft = len(attrs) + 1
for attr in attrs:
yield None
index += 1
numAttrsLeft -= 1
hasLength = self._hasLength(attr)
notDeadEnd = False
if curObjRef is None:
notDeadEnd = not self._isDeadEnd(attr)
if hasLength or notDeadEnd:
# prevent cycles in the references (i.e. base.loader.base)
for goesThrough in parentObjRef.goesThrough(curObj[index]):
# don't yield, container might lose this element
pass
if not goesThrough:
objRef = ObjectRef(Indirection(evalStr='[%s]' % index),
id(curObj[index]), parentObjRef)
yield None
if hasLength:
for i in self._addContainerGen(attr, objRef):
yield None
if notDeadEnd:
self._addDiscoveredStartRef(attr, objRef)
if curObjRef is None and random.randrange(numAttrsLeft) == 0:
curObjRef = objRef
del attr
except StopIteration, e:
pass
del itr
continue
def run(self):
try:
self._leakDetector._index2containerId2len[self._index] = {}
ids = self._leakDetector.getContainerIds()
# record the current len of each container
for objId in ids:
yield None
try:
for result in self._leakDetector.getContainerByIdGen(objId):
yield None
container = result
except Exception as e:
# this container no longer exists
if self.notify.getDebug():
for contName in self._leakDetector.getContainerNameByIdGen(objId):
yield None
self.notify.debug(
'%s no longer exists; caught exception in getContainerById (%s)' % (
contName, e))
self._leakDetector.removeContainerById(objId)
continue
if container is None:
# this container no longer exists
if self.notify.getDebug():
for contName in self._leakDetector.getContainerNameByIdGen(objId):
yield None
self.notify.debug('%s no longer exists; getContainerById returned None' %
contName)
self._leakDetector.removeContainerById(objId)
continue
try:
cLen = len(container)
except Exception as e:
# this container no longer exists
if self.notify.getDebug():
for contName in self._leakDetector.getContainerNameByIdGen(objId):
yield None
self.notify.debug(
'%s is no longer a container, it is now %s (%s)' %
(contName, safeRepr(container), e))
self._leakDetector.removeContainerById(objId)
continue
self._leakDetector._index2containerId2len[self._index][objId] = cLen
# compare the current len of each container to past lens
if self._index > 0:
idx2id2len = self._leakDetector._index2containerId2len
for objId in idx2id2len[self._index]:
yield None
if objId in idx2id2len[self._index-1]:
diff = idx2id2len[self._index][objId] - idx2id2len[self._index-1][objId]
"""
# this check is too spammy
if diff > 20:
if diff > idx2id2len[self._index-1][objId]:
minutes = (self._leakDetector._index2delay[self._index] -
self._leakDetector._index2delay[self._index-1]) / 60.
name = self._leakDetector.getContainerNameById(objId)
if idx2id2len[self._index-1][objId] != 0:
percent = 100. * (float(diff) / float(idx2id2len[self._index-1][objId]))
try:
for container in self._leakDetector.getContainerByIdGen(objId):
yield None
except:
# TODO
self.notify.debug('caught exception in getContainerByIdGen (1)')
else:
self.notify.warning(
'%s (%s) grew %.2f%% in %.2f minutes (%s items at last measurement, current contents: %s)' % (
name, itype(container), percent, minutes, idx2id2len[self._index][objId],
fastRepr(container, maxLen=CheckContainers.ReprItems)))
yield None
"""
if (self._index > 2 and
objId in idx2id2len[self._index-2] and
objId in idx2id2len[self._index-3]):
diff2 = idx2id2len[self._index-1][objId] - idx2id2len[self._index-2][objId]
diff3 = idx2id2len[self._index-2][objId] - idx2id2len[self._index-3][objId]
if self._index <= 4:
if diff > 0 and diff2 > 0 and diff3 > 0:
name = self._leakDetector.getContainerNameById(objId)
try:
for container in self._leakDetector.getContainerByIdGen(objId):
yield None
except:
# TODO
self.notify.debug('caught exception in getContainerByIdGen (2)')
else:
msg = ('%s (%s) consistently increased in size over the last '
'3 periods (%s items at last measurement, current contents: %s)' %
(name, itype(container), idx2id2len[self._index][objId],
fastRepr(container, maxLen=CheckContainers.ReprItems)))
self.notify.warning(msg)
yield None
elif (objId in idx2id2len[self._index-4] and
objId in idx2id2len[self._index-5]):
# if size has consistently increased over the last 5 checks,
# send out a warning
diff4 = idx2id2len[self._index-3][objId] - idx2id2len[self._index-4][objId]
diff5 = idx2id2len[self._index-4][objId] - idx2id2len[self._index-5][objId]
if diff > 0 and diff2 > 0 and diff3 > 0 and diff4 > 0 and diff5 > 0:
name = self._leakDetector.getContainerNameById(objId)
try:
for container in self._leakDetector.getContainerByIdGen(objId):
yield None
except:
# TODO
self.notify.debug('caught exception in getContainerByIdGen (3)')
else:
msg = ('leak detected: %s (%s) consistently increased in size over the last '
'5 periods (%s items at last measurement, current contents: %s)' %
(name, itype(container), idx2id2len[self._index][objId],
fastRepr(container, maxLen=CheckContainers.ReprItems)))
self.notify.warning(msg)
yield None
messenger.send(self._leakDetector.getLeakEvent(), [container, name])
if config.GetBool('pdb-on-leak-detect', 0):
import pdb;pdb.set_trace()
pass
except Exception as e:
print('CheckContainers job caught exception: %s' % e)
if __dev__:
raise
yield Job.Done
def run(self):
# do the garbage collection
wasOn = gcDebugOn()
oldFlags = gc.get_debug()
if not wasOn:
gc.set_debug(gc.DEBUG_SAVEALL)
gc.collect()
yield None
# don't repr the garbage list if we don't have to
if self.notify.getDebug():
self.notify.debug('gc.garbage == %s' % fastRepr(gc.garbage))
yield None
self.garbage = list(gc.garbage)
# don't repr the garbage list if we don't have to
if self.notify.getDebug():
self.notify.debug('self.garbage == %s' % self.garbage)
del gc.garbage[:]
if not wasOn:
gc.set_debug(oldFlags)
self.numGarbage = len(self.garbage)
yield None
if self._args.verbose:
self.notify.info('found %s garbage items' % self.numGarbage)
self.referrersByReference = {}
self.referrersByNumber = {}
self.referentsByReference = {}
self.referentsByNumber = {}
self.cycles = []
self.cycleSets = []
self.cycleIds = set()
# grab the referrers (pointing to garbage)
if self._args.fullReport and (self.numGarbage != 0):
if self._args.verbose:
self.notify.info('getting referrers...')
for i in xrange(self.numGarbage):
yield None
for result in self._getReferrers(self.garbage[i]):
yield None
byNum, byRef = result
self.referrersByNumber[i] = byNum
self.referrersByReference[i] = byRef
# grab the referents (pointed to by garbage)
if self.numGarbage > 0:
if self._args.verbose:
self.notify.info('getting referents...')
for i in xrange(self.numGarbage):
yield None
for result in self._getReferents(self.garbage[i]):
yield None
byNum, byRef = result
self.referentsByNumber[i] = byNum
self.referentsByReference[i] = byRef
# find the cycles
if self._args.findCycles and self.numGarbage > 0:
if self._args.verbose:
self.notify.info('detecting cycles...')
for i in xrange(self.numGarbage):
yield None
for newCycles in self._getCycles(i, self.cycleSets):
yield None
self.cycles.extend(newCycles)
# if we're not doing a full report, add this cycle's IDs to the master set
if not self._args.fullReport:
for cycle in newCycles:
yield None
self.cycleIds.update(set(cycle))
if self._args.findCycles:
s = ['===== GarbageReport: \'%s\' (%s items, %s cycles) =====' % (
self._args.name, self.numGarbage, len(self.cycles))]
else:
s = ['===== GarbageReport: \'%s\' (%s items) =====' % (
self._args.name, self.numGarbage)]
if self.numGarbage > 0:
# make a list of the ids we will actually be printing
if self._args.fullReport:
garbageIds = range(self.numGarbage)
else:
garbageIds = list(self.cycleIds)
garbageIds.sort()
numGarbage = len(garbageIds)
# log each individual item with a number in front of it
if not self._args.fullReport:
abbrev = '(abbreviated) '
else:
abbrev = ''
s.append('===== Garbage Items %s=====' % abbrev)
digits = 0
n = numGarbage
while n > 0:
yield None
digits += 1
n /= 10
digits = digits
format = '%0' + '%s' % digits + 'i:%s \t%s'
for i in xrange(numGarbage):
yield None
id = garbageIds[i]
objStr = safeRepr(self.garbage[id])
maxLen = 5000
if len(objStr) > maxLen:
snip = '<SNIP>'
objStr = '%s%s' % (objStr[:(maxLen-len(snip))], snip)
s.append(format % (id, itype(self.garbage[id]), objStr))
if self._args.findCycles:
s.append('===== Garbage Cycles =====')
for i in xrange(len(self.cycles)):
yield None
s.append('%s' % self.cycles[i])
if self._args.fullReport:
format = '%0' + '%s' % digits + 'i:%s'
s.append('===== Referrers By Number (what is referring to garbage item?) =====')
for i in xrange(numGarbage):
yield None
s.append(format % (i, self.referrersByNumber[i]))
s.append('===== Referents By Number (what is garbage item referring to?) =====')
for i in xrange(numGarbage):
yield None
s.append(format % (i, self.referentsByNumber[i]))
s.append('===== Referrers (what is referring to garbage item?) =====')
for i in xrange(numGarbage):
yield None
s.append(format % (i, self.referrersByReference[i]))
s.append('===== Referents (what is garbage item referring to?) =====')
for i in xrange(numGarbage):
yield None
s.append(format % (i, self.referentsByReference[i]))
self._report = s
if self._args.log:
self.printingBegin()
for i in xrange(len(self._report)):
yield None
self.notify.info(self._report[i])
self.printingEnd()
yield Job.Done
def leakContainer(task=None):
base = getBase()
if not hasattr(base, 'leakContainer'):
base.leakContainer = {}
# use tuples as keys since they can't be weakref'd, and use an instance
# since it can't be repr/eval'd
# that will force the leak detector to hold a normal 'non-weak' reference
class LeakKey:
pass
base.leakContainer[(LeakKey(),)] = {}
# test the non-weakref object reference handling
if random.random() < .01:
key = random.choice(base.leakContainer.keys())
ContainerLeakDetector.notify.debug(
'removing reference to leakContainer key %s so it will be garbage-collected' % safeRepr(key))
del base.leakContainer[key]
taskMgr.doMethodLater(10, leakContainer, 'leakContainer-%s' % serialNum())
if task:
return task.done
def run(self):
ContainerReport.PrivateIds.update(
set([
id(ContainerReport.PrivateIds),
id(self._visitedIds),
id(self._id2pathStr),
id(self._id2container),
id(self._type2id2len),
id(self._queue),
id(self._instanceDictIds),
]))
# push on a few things that we want to give priority
# for the sake of the variable-name printouts
try:
base
except:
pass
else:
self._enqueueContainer(base.__dict__, 'base')
try:
simbase
except:
pass
else:
self._enqueueContainer(simbase.__dict__, 'simbase')
self._queue.push(__builtins__)
self._id2pathStr[id(__builtins__)] = ''
while len(self._queue) > 0:
# yield up here instead of at the end, since we skip back to the
# top of the while loop from various points
yield None
parentObj = self._queue.pop()
#print '%s: %s, %s' % (id(parentObj), type(parentObj), self._id2pathStr[id(parentObj)])
isInstanceDict = False
if id(parentObj) in self._instanceDictIds:
isInstanceDict = True
try:
if parentObj.__class__.__name__ == 'method-wrapper':
continue
except:
pass
if type(parentObj) in (types.StringType, types.UnicodeType):
continue
if type(parentObj) in (types.ModuleType, types.InstanceType):
child = parentObj.__dict__
if self._examine(child):
assert (self._queue.back() is child)
self._instanceDictIds.add(id(child))
self._id2pathStr[id(child)] = str(
self._id2pathStr[id(parentObj)])
continue
if type(parentObj) is types.DictType:
key = None
attr = None
keys = parentObj.keys()
try:
keys.sort()
except TypeError, e:
self.notify.warning(
'non-sortable dict keys: %s: %s' %
(self._id2pathStr[id(parentObj)], repr(e)))
for key in keys:
try:
attr = parentObj[key]
except KeyError, e:
self.notify.warning(
'could not index into %s with key %s' %
(self._id2pathStr[id(parentObj)], key))
if id(attr) not in self._visitedIds:
self._visitedIds.add(id(attr))
if self._examine(attr):
assert (self._queue.back() is attr)
if parentObj is __builtins__:
self._id2pathStr[id(attr)] = key
else:
if isInstanceDict:
self._id2pathStr[id(
attr)] = self._id2pathStr[id(
parentObj)] + '.%s' % key
else:
self._id2pathStr[id(
attr
)] = self._id2pathStr[id(
parentObj)] + '[%s]' % safeRepr(key)
del key
del attr
continue
def run(self):
ContainerReport.PrivateIds.update(set([
id(ContainerReport.PrivateIds),
id(self._visitedIds),
id(self._id2pathStr),
id(self._id2container),
id(self._type2id2len),
id(self._queue),
id(self._instanceDictIds),
]))
# push on a few things that we want to give priority
# for the sake of the variable-name printouts
try:
base
except:
pass
else:
self._enqueueContainer( base.__dict__,
'base')
try:
simbase
except:
pass
else:
self._enqueueContainer( simbase.__dict__,
'simbase')
self._queue.push(__builtins__)
self._id2pathStr[id(__builtins__)] = ''
while len(self._queue) > 0:
# yield up here instead of at the end, since we skip back to the
# top of the while loop from various points
yield None
parentObj = self._queue.pop()
#print '%s: %s, %s' % (id(parentObj), type(parentObj), self._id2pathStr[id(parentObj)])
isInstanceDict = False
if id(parentObj) in self._instanceDictIds:
isInstanceDict = True
try:
if parentObj.__class__.__name__ == 'method-wrapper':
continue
except:
pass
if type(parentObj) in (types.StringType, types.UnicodeType):
continue
if type(parentObj) in (types.ModuleType, types.InstanceType):
child = parentObj.__dict__
if self._examine(child):
assert (self._queue.back() is child)
self._instanceDictIds.add(id(child))
self._id2pathStr[id(child)] = str(self._id2pathStr[id(parentObj)])
continue
if type(parentObj) is types.DictType:
key = None
attr = None
keys = parentObj.keys()
try:
keys.sort()
except TypeError, e:
self.notify.warning('non-sortable dict keys: %s: %s' % (self._id2pathStr[id(parentObj)], repr(e)))
for key in keys:
try:
attr = parentObj[key]
except KeyError, e:
self.notify.warning('could not index into %s with key %s' % (self._id2pathStr[id(parentObj)],
key))
if id(attr) not in self._visitedIds:
self._visitedIds.add(id(attr))
if self._examine(attr):
assert (self._queue.back() is attr)
if parentObj is __builtins__:
self._id2pathStr[id(attr)] = key
else:
if isInstanceDict:
self._id2pathStr[id(attr)] = self._id2pathStr[id(parentObj)] + '.%s' % key
else:
self._id2pathStr[id(attr)] = self._id2pathStr[id(parentObj)] + '[%s]' % safeRepr(key)
del key
del attr
continue
def run(self):
try:
# this yields a different set of start refs every time we start a new traversal
# force creation of a new workingListSelector inside the while loop right off the bat
workingListSelector = nullGen()
# this holds the current step of the current traversal
curObjRef = None
while True:
# yield up here instead of at the end, since we skip back to the
# top of the while loop from various points
yield None
#import pdb;pdb.set_trace()
if curObjRef is None:
# choose an object to start a traversal from
try:
startRefWorkingList = next(workingListSelector)
except StopIteration:
# do relative # of traversals on each set based on how many refs it contains
baseLen = len(self._baseStartRefWorkingList.source)
discLen = len(self._discoveredStartRefWorkingList.source)
minLen = float(max(1, min(baseLen, discLen)))
# this will cut down the traversals of the larger set by 2/3
minLen *= 3.
workingListSelector = flywheel([self._baseStartRefWorkingList, self._discoveredStartRefWorkingList],
[baseLen/minLen, discLen/minLen])
yield None
continue
# grab the next start ref from this sequence and see if it's still valid
while True:
yield None
try:
curObjRef = next(startRefWorkingList.refGen)
break
except StopIteration:
# we've run out of refs, grab a new set
if len(startRefWorkingList.source) == 0:
# ref set is empty, choose another
break
# make a generator that yields containers a # of times that is
# proportional to their length
for fw in makeFlywheelGen(
list(startRefWorkingList.source.values()),
countFunc=lambda x: self.getStartObjAffinity(x),
scale=.05):
yield None
startRefWorkingList.refGen = fw
if curObjRef is None:
# this ref set is empty, choose another
# the base set should never be empty (builtins etc.)
continue
# do we need to go look up the object in _id2ref? sometimes we do that
# to avoid storing multiple redundant refs to a single item
if type(curObjRef) is int:
startId = curObjRef
curObjRef = None
try:
for containerRef in self._leakDetector.getContainerByIdGen(startId):
yield None
except:
# ref is invalid
self.notify.debug('invalid startRef, stored as id %s' % startId)
self._leakDetector.removeContainerById(startId)
continue
curObjRef = containerRef
try:
for curObj in curObjRef.getContainerGen():
yield None
except:
self.notify.debug('lost current container, ref.getContainerGen() failed')
# that container is gone, try again
curObjRef = None
continue
self.notify.debug('--> %s' % curObjRef)
#import pdb;pdb.set_trace()
# store a copy of the current objRef
parentObjRef = curObjRef
# if we hit a dead end, start over from another container
curObjRef = None
if hasattr(curObj, '__dict__'):
child = curObj.__dict__
hasLength = self._hasLength(child)
notDeadEnd = not self._isDeadEnd(child)
if hasLength or notDeadEnd:
# prevent cycles in the references (i.e. base.loader.base)
for goesThrough in parentObjRef.goesThroughGen(child):
# don't yield, container might lose this element
pass
if not goesThrough:
objRef = ObjectRef(Indirection(evalStr='.__dict__'),
id(child), parentObjRef)
yield None
if hasLength:
for i in self._addContainerGen(child, objRef):
yield None
if notDeadEnd:
self._addDiscoveredStartRef(child, objRef)
curObjRef = objRef
continue
if type(curObj) is dict:
key = None
attr = None
keys = list(curObj.keys())
# we will continue traversing the object graph via one key of the dict,
# choose it at random without taking a big chunk of CPU time
numKeysLeft = len(keys) + 1
for key in keys:
yield None
numKeysLeft -= 1
try:
attr = curObj[key]
except KeyError as e:
# this is OK because we are yielding during the iteration
self.notify.debug('could not index into %s with key %s' % (
parentObjRef, safeRepr(key)))
continue
hasLength = self._hasLength(attr)
notDeadEnd = False
# if we haven't picked the next ref, check if this one is a candidate
if curObjRef is None:
notDeadEnd = not self._isDeadEnd(attr, key)
if hasLength or notDeadEnd:
# prevent cycles in the references (i.e. base.loader.base)
for goesThrough in parentObjRef.goesThroughGen(curObj[key]):
# don't yield, container might lose this element
pass
if not goesThrough:
if curObj is builtins.__dict__:
objRef = ObjectRef(Indirection(evalStr='%s' % key),
id(curObj[key]))
else:
objRef = ObjectRef(Indirection(dictKey=key),
id(curObj[key]), parentObjRef)
yield None
if hasLength:
for i in self._addContainerGen(attr, objRef):
yield None
if notDeadEnd:
self._addDiscoveredStartRef(attr, objRef)
if curObjRef is None and random.randrange(numKeysLeft) == 0:
curObjRef = objRef
del key
del attr
continue
try:
childNames = dir(curObj)
except:
pass
else:
try:
index = -1
attrs = []
while 1:
yield None
try:
attr = next(itr)
except:
# some custom classes don't do well when iterated
attr = None
break
attrs.append(attr)
# we will continue traversing the object graph via one attr,
# choose it at random without taking a big chunk of CPU time
numAttrsLeft = len(attrs) + 1
for attr in attrs:
yield None
index += 1
numAttrsLeft -= 1
hasLength = self._hasLength(attr)
notDeadEnd = False
if curObjRef is None:
notDeadEnd = not self._isDeadEnd(attr)
if hasLength or notDeadEnd:
# prevent cycles in the references (i.e. base.loader.base)
for goesThrough in parentObjRef.goesThrough(curObj[index]):
# don't yield, container might lose this element
pass
if not goesThrough:
objRef = ObjectRef(Indirection(evalStr='[%s]' % index),
id(curObj[index]), parentObjRef)
yield None
if hasLength:
for i in self._addContainerGen(attr, objRef):
yield None
if notDeadEnd:
self._addDiscoveredStartRef(attr, objRef)
if curObjRef is None and random.randrange(numAttrsLeft) == 0:
curObjRef = objRef
del attr
except StopIteration as e:
pass
del itr
continue
except Exception as e:
print('FindContainers job caught exception: %s' % e)
if __dev__:
raise
yield Job.Done
def run(self):
ContainerReport.PrivateIds.update(
set([
id(ContainerReport.PrivateIds),
id(self._visitedIds),
id(self._id2pathStr),
id(self._id2container),
id(self._type2id2len),
id(self._queue),
id(self._instanceDictIds),
]))
# push on a few things that we want to give priority
# for the sake of the variable-name printouts
try:
base
except:
pass
else:
self._enqueueContainer(base.__dict__, 'base')
try:
simbase
except:
pass
else:
self._enqueueContainer(simbase.__dict__, 'simbase')
self._queue.push(__builtins__)
self._id2pathStr[id(__builtins__)] = ''
while len(self._queue) > 0:
# yield up here instead of at the end, since we skip back to the
# top of the while loop from various points
yield None
parentObj = self._queue.pop()
#print '%s: %s, %s' % (id(parentObj), type(parentObj), self._id2pathStr[id(parentObj)])
isInstanceDict = False
if id(parentObj) in self._instanceDictIds:
isInstanceDict = True
try:
if parentObj.__class__.__name__ == 'method-wrapper':
continue
except:
pass
if isinstance(parentObj, (str, bytes)):
continue
if type(parentObj) in (types.ModuleType, types.InstanceType):
child = parentObj.__dict__
if self._examine(child):
assert self._queue.back() is child
self._instanceDictIds.add(id(child))
self._id2pathStr[id(child)] = str(
self._id2pathStr[id(parentObj)])
continue
if isinstance(parentObj, dict):
key = None
attr = None
keys = list(parentObj.keys())
try:
keys.sort()
except TypeError as e:
self.notify.warning(
'non-sortable dict keys: %s: %s' %
(self._id2pathStr[id(parentObj)], repr(e)))
for key in keys:
try:
attr = parentObj[key]
except KeyError as e:
self.notify.warning(
'could not index into %s with key %s' %
(self._id2pathStr[id(parentObj)], key))
if id(attr) not in self._visitedIds:
self._visitedIds.add(id(attr))
if self._examine(attr):
assert self._queue.back() is attr
if parentObj is __builtins__:
self._id2pathStr[id(attr)] = key
else:
if isInstanceDict:
self._id2pathStr[id(
attr)] = self._id2pathStr[id(
parentObj)] + '.%s' % key
else:
self._id2pathStr[id(
attr
)] = self._id2pathStr[id(
parentObj)] + '[%s]' % safeRepr(key)
del key
del attr
continue
if type(parentObj) is not types.FileType:
try:
itr = iter(parentObj)
except:
pass
else:
try:
index = 0
while 1:
try:
attr = next(itr)
except:
# some custom classes don't do well when iterated
attr = None
break
if id(attr) not in self._visitedIds:
self._visitedIds.add(id(attr))
if self._examine(attr):
assert self._queue.back() is attr
self._id2pathStr[id(
attr)] = self._id2pathStr[id(
parentObj)] + '[%s]' % index
index += 1
del attr
except StopIteration as e:
pass
del itr
continue
try:
childNames = dir(parentObj)
except:
pass
else:
childName = None
child = None
for childName in childNames:
child = getattr(parentObj, childName)
if id(child) not in self._visitedIds:
self._visitedIds.add(id(child))
if self._examine(child):
assert self._queue.back() is child
self._id2pathStr[id(child)] = self._id2pathStr[id(
parentObj)] + '.%s' % childName
del childName
del child
continue
if self._log:
self.printingBegin()
for i in self._output(limit=self._limit):
yield None
self.printingEnd()
yield Job.Done
def run(self):
try:
self._leakDetector._index2containerId2len[self._index] = {}
ids = self._leakDetector.getContainerIds()
# record the current len of each container
for objId in ids:
yield None
try:
for result in self._leakDetector.getContainerByIdGen(objId):
yield None
container = result
except Exception as e:
# this container no longer exists
if self.notify.getDebug():
for contName in self._leakDetector.getContainerNameByIdGen(objId):
yield None
self.notify.debug(
'%s no longer exists; caught exception in getContainerById (%s)' % (
contName, e))
self._leakDetector.removeContainerById(objId)
continue
if container is None:
# this container no longer exists
if self.notify.getDebug():
for contName in self._leakDetector.getContainerNameByIdGen(objId):
yield None
self.notify.debug('%s no longer exists; getContainerById returned None' %
contName)
self._leakDetector.removeContainerById(objId)
continue
try:
cLen = len(container)
except Exception as e:
# this container no longer exists
if self.notify.getDebug():
for contName in self._leakDetector.getContainerNameByIdGen(objId):
yield None
self.notify.debug(
'%s is no longer a container, it is now %s (%s)' %
(contName, safeRepr(container), e))
self._leakDetector.removeContainerById(objId)
continue
self._leakDetector._index2containerId2len[self._index][objId] = cLen
# compare the current len of each container to past lens
if self._index > 0:
idx2id2len = self._leakDetector._index2containerId2len
for objId in idx2id2len[self._index]:
yield None
if objId in idx2id2len[self._index-1]:
diff = idx2id2len[self._index][objId] - idx2id2len[self._index-1][objId]
"""
# this check is too spammy
if diff > 20:
if diff > idx2id2len[self._index-1][objId]:
minutes = (self._leakDetector._index2delay[self._index] -
self._leakDetector._index2delay[self._index-1]) / 60.
name = self._leakDetector.getContainerNameById(objId)
if idx2id2len[self._index-1][objId] != 0:
percent = 100. * (float(diff) / float(idx2id2len[self._index-1][objId]))
try:
for container in self._leakDetector.getContainerByIdGen(objId):
yield None
except:
# TODO
self.notify.debug('caught exception in getContainerByIdGen (1)')
else:
self.notify.warning(
'%s (%s) grew %.2f%% in %.2f minutes (%s items at last measurement, current contents: %s)' % (
name, itype(container), percent, minutes, idx2id2len[self._index][objId],
fastRepr(container, maxLen=CheckContainers.ReprItems)))
yield None
"""
if (self._index > 2 and
objId in idx2id2len[self._index-2] and
objId in idx2id2len[self._index-3]):
diff2 = idx2id2len[self._index-1][objId] - idx2id2len[self._index-2][objId]
diff3 = idx2id2len[self._index-2][objId] - idx2id2len[self._index-3][objId]
if self._index <= 4:
if diff > 0 and diff2 > 0 and diff3 > 0:
name = self._leakDetector.getContainerNameById(objId)
try:
for container in self._leakDetector.getContainerByIdGen(objId):
yield None
except:
# TODO
self.notify.debug('caught exception in getContainerByIdGen (2)')
else:
msg = ('%s (%s) consistently increased in size over the last '
'3 periods (%s items at last measurement, current contents: %s)' %
(name, itype(container), idx2id2len[self._index][objId],
fastRepr(container, maxLen=CheckContainers.ReprItems)))
self.notify.warning(msg)
yield None
elif (objId in idx2id2len[self._index-4] and
objId in idx2id2len[self._index-5]):
# if size has consistently increased over the last 5 checks,
# send out a warning
diff4 = idx2id2len[self._index-3][objId] - idx2id2len[self._index-4][objId]
diff5 = idx2id2len[self._index-4][objId] - idx2id2len[self._index-5][objId]
if diff > 0 and diff2 > 0 and diff3 > 0 and diff4 > 0 and diff5 > 0:
name = self._leakDetector.getContainerNameById(objId)
try:
for container in self._leakDetector.getContainerByIdGen(objId):
yield None
except:
# TODO
self.notify.debug('caught exception in getContainerByIdGen (3)')
else:
msg = ('leak detected: %s (%s) consistently increased in size over the last '
'5 periods (%s items at last measurement, current contents: %s)' %
(name, itype(container), idx2id2len[self._index][objId],
fastRepr(container, maxLen=CheckContainers.ReprItems)))
self.notify.warning(msg)
yield None
messenger.send(self._leakDetector.getLeakEvent(), [container, name])
if config.GetBool('pdb-on-leak-detect', 0):
import pdb;pdb.set_trace()
pass
except Exception as e:
print('CheckContainers job caught exception: %s' % e)
if __dev__:
raise
yield Job.Done