def flush(self):
"""
Delete each item in the cache then clear all references to them
"""
assert self.checkCache()
CRCache.notify.debug("Flushing the cache")
# NOTE: delayDeleted objects should no longer get into the cache in the first place
# give objects a chance to clean themselves up before checking for DelayDelete leaks
messenger.send('clientCleanup')
# some of these objects might be holding delayDeletes on others
# track each object that is delayDeleted after it gets its chance to delete,
# and check them after all objects have had a chance to delete
delayDeleted = []
for distObj in self.dict.values():
distObj.deleteOrDelay()
if distObj.getDelayDeleteCount() != 0:
delayDeleted.append(distObj)
if distObj.getDelayDeleteCount() <= 0:
# make sure we're not leaking
distObj.detectLeaks()
# now that all objects have had a chance to delete, are there any objects left
# that are still delayDeleted?
delayDeleteLeaks = []
for distObj in delayDeleted:
if distObj.getDelayDeleteCount() != 0:
delayDeleteLeaks.append(distObj)
if len(delayDeleteLeaks) > 0:
s = 'CRCache.flush:'
for obj in delayDeleteLeaks:
s += ('\n could not delete %s (%s), delayDeletes=%s' %
(safeRepr(obj), itype(obj), obj.getDelayDeleteNames()))
self.notify.error(s)
# Null out all references to the objects so they will get gcd
self.dict = {}
self.fifo = []
def printDelayDeletes(self):
print('DelayDeletes:')
print('=============')
for obj in self._delayDeletedDOs.values():
print('%s\t%s (%s)\tdelayDeletes=%s' %
(obj.doId, safeRepr(obj), itype(obj),
obj.getDelayDeleteNames()))
def __init__(self, objects):
self._objs = list(objects)
self._type2objs = {}
self._count2types = {}
self._len2obj = {}
type2count = {}
for obj in self._objs:
typ = itype(obj)
type2count.setdefault(typ, 0)
type2count[typ] += 1
self._type2objs.setdefault(typ, [])
self._type2objs[typ].append(obj)
try:
self._len2obj[len(obj)] = obj
except:
pass
self._count2types = invertDictLossless(type2count)
def run(self):
try:
self._leakDetector._index2containerId2len[self._index] = { }
ids = self._leakDetector.getContainerIds()
for objId in ids:
yield None
try:
for result in self._leakDetector.getContainerByIdGen(objId):
yield None
container = result
except Exception:
e = None
if self.notify.getDebug():
for contName in self._leakDetector.getContainerNameByIdGen(objId):
yield None
self.notify.debug('%s no longer exists; caught exception in getContainerById (%s)' % (contName, e))
self._leakDetector.removeContainerById(objId)
continue
if container is None:
if self.notify.getDebug():
for contName in self._leakDetector.getContainerNameByIdGen(objId):
yield None
self.notify.debug('%s no longer exists; getContainerById returned None' % contName)
self._leakDetector.removeContainerById(objId)
continue
try:
cLen = len(container)
except Exception:
e = None
if self.notify.getDebug():
for contName in self._leakDetector.getContainerNameByIdGen(objId):
yield None
self.notify.debug('%s is no longer a container, it is now %s (%s)' % (contName, safeRepr(container), e))
self._leakDetector.removeContainerById(objId)
continue
self._leakDetector._index2containerId2len[self._index][objId] = cLen
if self._index > 0:
idx2id2len = self._leakDetector._index2containerId2len
for objId in idx2id2len[self._index]:
yield None
if objId in idx2id2len[self._index - 1]:
diff = idx2id2len[self._index][objId] - idx2id2len[self._index - 1][objId]
if self._index > 2 and objId in idx2id2len[self._index - 2] and objId in idx2id2len[self._index - 3]:
diff2 = idx2id2len[self._index - 1][objId] - idx2id2len[self._index - 2][objId]
diff3 = idx2id2len[self._index - 2][objId] - idx2id2len[self._index - 3][objId]
if self._index <= 4:
if diff > 0 and diff2 > 0 and diff3 > 0:
name = self._leakDetector.getContainerNameById(objId)
try:
for container in self._leakDetector.getContainerByIdGen(objId):
yield None
except:
self.notify.debug('caught exception in getContainerByIdGen (2)')
msg = '%s (%s) consistently increased in size over the last 3 periods (%s items at last measurement, current contents: %s)' % (name, itype(container), idx2id2len[self._index][objId], fastRepr(container, maxLen = CheckContainers.ReprItems))
self.notify.warning(msg)
yield None
elif objId in idx2id2len[self._index - 4] and objId in idx2id2len[self._index - 5]:
diff4 = idx2id2len[self._index - 3][objId] - idx2id2len[self._index - 4][objId]
diff5 = idx2id2len[self._index - 4][objId] - idx2id2len[self._index - 5][objId]
if diff > 0 and diff2 > 0 and diff3 > 0 and diff4 > 0 and diff5 > 0:
name = self._leakDetector.getContainerNameById(objId)
try:
for container in self._leakDetector.getContainerByIdGen(objId):
yield None
except:
self.notify.debug('caught exception in getContainerByIdGen (3)')
msg = 'leak detected: %s (%s) consistently increased in size over the last 5 periods (%s items at last measurement, current contents: %s)' % (name, itype(container), idx2id2len[self._index][objId], fastRepr(container, maxLen = CheckContainers.ReprItems))
self.notify.warning(msg)
yield None
messenger.send(self._leakDetector.getLeakEvent(), [
container,
name])
if config.GetBool('pdb-on-leak-detect', 0):
import pdb as pdb
pdb.set_trace()
objId in idx2id2len[self._index - 3]
except Exception:
e = None
print 'CheckContainers job caught exception: %s' % e
if __dev__:
raise
yield Job.Done
def run(self):
try:
self._leakDetector._index2containerId2len[self._index] = {}
ids = self._leakDetector.getContainerIds()
# record the current len of each container
for objId in ids:
yield None
try:
for result in self._leakDetector.getContainerByIdGen(objId):
yield None
container = result
except Exception as e:
# this container no longer exists
if self.notify.getDebug():
for contName in self._leakDetector.getContainerNameByIdGen(objId):
yield None
self.notify.debug(
'%s no longer exists; caught exception in getContainerById (%s)' % (
contName, e))
self._leakDetector.removeContainerById(objId)
continue
if container is None:
# this container no longer exists
if self.notify.getDebug():
for contName in self._leakDetector.getContainerNameByIdGen(objId):
yield None
self.notify.debug('%s no longer exists; getContainerById returned None' %
contName)
self._leakDetector.removeContainerById(objId)
continue
try:
cLen = len(container)
except Exception as e:
# this container no longer exists
if self.notify.getDebug():
for contName in self._leakDetector.getContainerNameByIdGen(objId):
yield None
self.notify.debug(
'%s is no longer a container, it is now %s (%s)' %
(contName, safeRepr(container), e))
self._leakDetector.removeContainerById(objId)
continue
self._leakDetector._index2containerId2len[self._index][objId] = cLen
# compare the current len of each container to past lens
if self._index > 0:
idx2id2len = self._leakDetector._index2containerId2len
for objId in idx2id2len[self._index]:
yield None
if objId in idx2id2len[self._index-1]:
diff = idx2id2len[self._index][objId] - idx2id2len[self._index-1][objId]
"""
# this check is too spammy
if diff > 20:
if diff > idx2id2len[self._index-1][objId]:
minutes = (self._leakDetector._index2delay[self._index] -
self._leakDetector._index2delay[self._index-1]) / 60.
name = self._leakDetector.getContainerNameById(objId)
if idx2id2len[self._index-1][objId] != 0:
percent = 100. * (float(diff) / float(idx2id2len[self._index-1][objId]))
try:
for container in self._leakDetector.getContainerByIdGen(objId):
yield None
except:
# TODO
self.notify.debug('caught exception in getContainerByIdGen (1)')
else:
self.notify.warning(
'%s (%s) grew %.2f%% in %.2f minutes (%s items at last measurement, current contents: %s)' % (
name, itype(container), percent, minutes, idx2id2len[self._index][objId],
fastRepr(container, maxLen=CheckContainers.ReprItems)))
yield None
"""
if (self._index > 2 and
objId in idx2id2len[self._index-2] and
objId in idx2id2len[self._index-3]):
diff2 = idx2id2len[self._index-1][objId] - idx2id2len[self._index-2][objId]
diff3 = idx2id2len[self._index-2][objId] - idx2id2len[self._index-3][objId]
if self._index <= 4:
if diff > 0 and diff2 > 0 and diff3 > 0:
name = self._leakDetector.getContainerNameById(objId)
try:
for container in self._leakDetector.getContainerByIdGen(objId):
yield None
except:
# TODO
self.notify.debug('caught exception in getContainerByIdGen (2)')
else:
msg = ('%s (%s) consistently increased in size over the last '
'3 periods (%s items at last measurement, current contents: %s)' %
(name, itype(container), idx2id2len[self._index][objId],
fastRepr(container, maxLen=CheckContainers.ReprItems)))
self.notify.warning(msg)
yield None
elif (objId in idx2id2len[self._index-4] and
objId in idx2id2len[self._index-5]):
# if size has consistently increased over the last 5 checks,
# send out a warning
diff4 = idx2id2len[self._index-3][objId] - idx2id2len[self._index-4][objId]
diff5 = idx2id2len[self._index-4][objId] - idx2id2len[self._index-5][objId]
if diff > 0 and diff2 > 0 and diff3 > 0 and diff4 > 0 and diff5 > 0:
name = self._leakDetector.getContainerNameById(objId)
try:
for container in self._leakDetector.getContainerByIdGen(objId):
yield None
except:
# TODO
self.notify.debug('caught exception in getContainerByIdGen (3)')
else:
msg = ('leak detected: %s (%s) consistently increased in size over the last '
'5 periods (%s items at last measurement, current contents: %s)' %
(name, itype(container), idx2id2len[self._index][objId],
fastRepr(container, maxLen=CheckContainers.ReprItems)))
self.notify.warning(msg)
yield None
messenger.send(self._leakDetector.getLeakEvent(), [container, name])
if config.GetBool('pdb-on-leak-detect', 0):
import pdb;pdb.set_trace()
pass
except Exception as e:
print('CheckContainers job caught exception: %s' % e)
if __dev__:
raise
yield Job.Done
def run(self):
# set of ids of objects that we know are always attached to builtin;
# if an object is attached to one of these, it's attached to builtin
# this cuts down on the amount of searching that needs to be done
builtinIds = set()
builtinIds.add(id(builtins.__dict__))
try:
builtinIds.add(id(base))
builtinIds.add(id(base.cr))
builtinIds.add(id(base.cr.doId2do))
except:
pass
try:
builtinIds.add(id(simbase))
builtinIds.add(id(simbase.air))
builtinIds.add(id(simbase.air.doId2do))
except:
pass
try:
builtinIds.add(id(uber))
builtinIds.add(id(uber.air))
builtinIds.add(id(uber.air.doId2do))
except:
pass
while True:
yield None
objects = list(messenger._Messenger__objectEvents.keys())
assert self.notify.debug('%s objects in the messenger' %
len(objects))
for object in objects:
yield None
assert self.notify.debug('---> new object: %s' % itype(object))
# try to find a path to builtin that doesn't involve the messenger
# lists of objects for breadth-first search
# iterate through one list while populating other list
objList1 = []
objList2 = []
curObjList = objList1
nextObjList = objList2
visitedObjIds = set()
# add the id of the object, and the messenger containers so that
# the search for builtin will stop at the messenger; we're looking
# for any path to builtin that don't involve the messenger
visitedObjIds.add(id(object))
visitedObjIds.add(id(messenger._Messenger__objectEvents))
visitedObjIds.add(id(messenger._Messenger__callbacks))
nextObjList.append(object)
foundBuiltin = False
# breadth-first search, go until you run out of new objects or you find __builtin__
while len(nextObjList) > 0:
if foundBuiltin:
break
# swap the lists, prepare for the next pass
curObjList = nextObjList
nextObjList = []
assert self.notify.debug(
'next search iteration, num objects: %s' %
len(curObjList))
for curObj in curObjList:
if foundBuiltin:
break
yield None
referrers = gc.get_referrers(curObj)
assert self.notify.debug(
'curObj: %s @ %s, %s referrers, repr=%s' %
(itype(curObj), hex(id(curObj)), len(referrers),
fastRepr(curObj, maxLen=2)))
for referrer in referrers:
#assert self.notify.debug('referrer: %s' % itype(curObj))
yield None
refId = id(referrer)
# don't go in a loop
if refId in visitedObjIds:
#assert self.notify.debug('already visited')
continue
# don't self-reference
if referrer is curObjList or referrer is nextObjList:
continue
if refId in builtinIds:
# not a leak, there is a path to builtin that does not involve the messenger
#assert self.notify.debug('object has another path to __builtin__, it\'s not a messenger leak')
foundBuiltin = True
break
else:
visitedObjIds.add(refId)
nextObjList.append(referrer)
if not foundBuiltin:
self.notify.warning(
'%s is referenced only by the messenger' %
(itype(object)))
def run(self):
# do the garbage collection
oldFlags = gc.get_debug()
if self._args.delOnly:
# do a collect without SAVEALL, to identify the instances that are involved in
# cycles with instances that define __del__
# cycles that do not involve any instances that define __del__ are cleaned up
# automatically by Python, but they also appear in gc.garbage when SAVEALL is set
gc.set_debug(0)
if self._args.collect:
gc.collect()
garbageInstances = gc.garbage[:]
del gc.garbage[:]
# only yield if there's more time-consuming work to do,
# if there's no garbage, give instant feedback
if len(garbageInstances) > 0:
yield None
# don't repr the garbage list if we don't have to
if self.notify.getDebug():
self.notify.debug('garbageInstances == %s' %
fastRepr(garbageInstances))
self.numGarbageInstances = len(garbageInstances)
# grab the ids of the garbage instances (objects with __del__)
self.garbageInstanceIds = set()
for i in range(len(garbageInstances)):
self.garbageInstanceIds.add(id(garbageInstances[i]))
if i % 20 == 0:
yield None
# then release the list of instances so that it doesn't interfere with the gc.collect() below
del garbageInstances
else:
self.garbageInstanceIds = set()
# do a SAVEALL pass so that we have all of the objects involved in legitimate garbage cycles
# without SAVEALL, gc.garbage only contains objects with __del__ methods
gc.set_debug(gc.DEBUG_SAVEALL)
if self._args.collect:
gc.collect()
self.garbage = gc.garbage[:]
del gc.garbage[:]
# only yield if there's more time-consuming work to do,
# if there's no garbage, give instant feedback
if len(self.garbage) > 0:
yield None
# don't repr the garbage list if we don't have to
if self.notify.getDebug():
self.notify.debug('self.garbage == %s' % fastRepr(self.garbage))
gc.set_debug(oldFlags)
self.numGarbage = len(self.garbage)
# only yield if there's more time-consuming work to do,
# if there's no garbage, give instant feedback
if self.numGarbage > 0:
yield None
if self._args.verbose:
self.notify.info('found %s garbage items' % self.numGarbage)
# print the types of the garbage first, in case the repr of an object
# causes a crash
#if self.numGarbage > 0:
# self.notify.info('TYPES ONLY (this is only needed if a crash occurs before GarbageReport finishes):')
# for result in printNumberedTypesGen(self.garbage):
# yield None
# Py obj id -> garbage list index
self._id2index = {}
self.referrersByReference = {}
self.referrersByNumber = {}
self.referentsByReference = {}
self.referentsByNumber = {}
self._id2garbageInfo = {}
self.cycles = []
self.cyclesBySyntax = []
self.uniqueCycleSets = set()
self.cycleIds = set()
# make the id->index table to speed up the next steps
for i in range(self.numGarbage):
self._id2index[id(self.garbage[i])] = i
if i % 20 == 0:
yield None
# grab the referrers (pointing to garbage)
if self._args.fullReport and (self.numGarbage != 0):
if self._args.verbose:
self.notify.info('getting referrers...')
for i in range(self.numGarbage):
yield None
for result in self._getReferrers(self.garbage[i]):
yield None
byNum, byRef = result
self.referrersByNumber[i] = byNum
self.referrersByReference[i] = byRef
# grab the referents (pointed to by garbage)
if self.numGarbage > 0:
if self._args.verbose:
self.notify.info('getting referents...')
for i in range(self.numGarbage):
yield None
for result in self._getReferents(self.garbage[i]):
yield None
byNum, byRef = result
self.referentsByNumber[i] = byNum
self.referentsByReference[i] = byRef
for i in range(self.numGarbage):
if hasattr(self.garbage[i], '_garbageInfo') and callable(
self.garbage[i]._garbageInfo):
try:
info = self.garbage[i]._garbageInfo()
except Exception as e:
info = str(e)
self._id2garbageInfo[id(self.garbage[i])] = info
yield None
else:
if i % 20 == 0:
yield None
# find the cycles
if self._args.findCycles and self.numGarbage > 0:
if self._args.verbose:
self.notify.info('calculating cycles...')
for i in range(self.numGarbage):
yield None
for newCycles in self._getCycles(i, self.uniqueCycleSets):
yield None
self.cycles.extend(newCycles)
# create a representation of the cycle in human-readable form
newCyclesBySyntax = []
for cycle in newCycles:
cycleBySyntax = ''
objs = []
# leave off the last index, it's a repeat of the first index
for index in cycle[:-1]:
objs.append(self.garbage[index])
yield None
# make the list repeat so we can safely iterate off the end
numObjs = len(objs) - 1
objs.extend(objs)
# state variables for our loop below
numToSkip = 0
objAlreadyRepresented = False
# if cycle starts off with an instance dict, start with the instance instead
startIndex = 0
# + 1 to include a reference back to the first object
endIndex = numObjs + 1
if inspect.isclass(objs[-1]) and type(objs[0]) is dict:
startIndex -= 1
endIndex -= 1
for index in range(startIndex, endIndex):
if numToSkip:
numToSkip -= 1
continue
obj = objs[index]
if inspect.isclass(obj):
if not objAlreadyRepresented:
cycleBySyntax += '%s' % obj.__class__.__name__
cycleBySyntax += '.'
# skip past the instance dict and get the member obj
numToSkip += 1
member = objs[index + 2]
for key, value in obj.__dict__.items():
if value is member:
break
yield None
else:
key = '<unknown member name>'
cycleBySyntax += '%s' % key
objAlreadyRepresented = True
elif type(obj) is dict:
cycleBySyntax += '{'
# get object referred to by dict
val = objs[index + 1]
for key, value in obj.items():
if value is val:
break
yield None
else:
key = '<unknown key>'
cycleBySyntax += '%s}' % fastRepr(key)
objAlreadyRepresented = True
elif type(obj) in (tuple, list):
brackets = {
tuple: '()',
list: '[]',
}[type(obj)]
# get object being referenced by container
nextObj = objs[index + 1]
cycleBySyntax += brackets[0]
for index in range(len(obj)):
if obj[index] is nextObj:
index = str(index)
break
yield None
else:
index = '<unknown index>'
cycleBySyntax += '%s%s' % (index, brackets[1])
objAlreadyRepresented = True
else:
cycleBySyntax += '%s --> ' % itype(obj)
objAlreadyRepresented = False
newCyclesBySyntax.append(cycleBySyntax)
yield None
self.cyclesBySyntax.extend(newCyclesBySyntax)
# if we're not doing a full report, add this cycle's IDs to the master set
if not self._args.fullReport:
for cycle in newCycles:
yield None
self.cycleIds.update(set(cycle))
self.numCycles = len(self.cycles)
if self._args.findCycles:
s = [
'===== GarbageReport: \'%s\' (%s %s) =====' %
(self._args.name, self.numCycles,
('cycle' if self.numCycles == 1 else 'cycles'))
]
else:
s = ['===== GarbageReport: \'%s\' =====' % (self._args.name)]
if self.numGarbage > 0:
# make a list of the ids we will actually be printing
if self._args.fullReport:
garbageIndices = range(self.numGarbage)
else:
garbageIndices = list(self.cycleIds)
garbageIndices.sort()
numGarbage = len(garbageIndices)
# log each individual item with a number in front of it
if not self._args.fullReport:
abbrev = '(abbreviated) '
else:
abbrev = ''
s.append('===== Garbage Items %s=====' % abbrev)
digits = 0
n = numGarbage
while n > 0:
yield None
digits += 1
n /= 10
digits = digits
format = '%0' + '%s' % digits + 'i:%s \t%s'
for i in range(numGarbage):
yield None
idx = garbageIndices[i]
if self._args.safeMode:
# in safe mode, don't try to repr any of the objects
objStr = repr(itype(self.garbage[idx]))
else:
objStr = fastRepr(self.garbage[idx])
maxLen = 5000
if len(objStr) > maxLen:
snip = '<SNIP>'
objStr = '%s%s' % (objStr[:(maxLen - len(snip))], snip)
s.append(format % (idx, itype(self.garbage[idx]), objStr))
# also log the types of the objects
s.append('===== Garbage Item Types %s=====' % abbrev)
for i in range(numGarbage):
yield None
idx = garbageIndices[i]
objStr = str(deeptype(self.garbage[idx]))
maxLen = 5000
if len(objStr) > maxLen:
snip = '<SNIP>'
objStr = '%s%s' % (objStr[:(maxLen - len(snip))], snip)
s.append(format % (idx, itype(self.garbage[idx]), objStr))
if self._args.findCycles:
s.append('===== Garbage Cycles (Garbage Item Numbers) =====')
ac = AlphabetCounter()
for i in range(self.numCycles):
yield None
s.append('%s:%s' % (ac.next(), self.cycles[i]))
if self._args.findCycles:
s.append('===== Garbage Cycles (Python Syntax) =====')
ac = AlphabetCounter()
for i in range(len(self.cyclesBySyntax)):
yield None
s.append('%s:%s' % (ac.next(), self.cyclesBySyntax[i]))
if len(self._id2garbageInfo) > 0:
s.append('===== Garbage Custom Info =====')
ac = AlphabetCounter()
for i in range(len(self.cyclesBySyntax)):
yield None
counter = ac.next()
_id = id(self.garbage[i])
if _id in self._id2garbageInfo:
s.append('%s:%s' %
(counter, self._id2garbageInfo[_id]))
if self._args.fullReport:
format = '%0' + '%s' % digits + 'i:%s'
s.append(
'===== Referrers By Number (what is referring to garbage item?) ====='
)
for i in range(numGarbage):
yield None
s.append(format % (i, self.referrersByNumber[i]))
s.append(
'===== Referents By Number (what is garbage item referring to?) ====='
)
for i in range(numGarbage):
yield None
s.append(format % (i, self.referentsByNumber[i]))
s.append(
'===== Referrers (what is referring to garbage item?) ====='
)
for i in range(numGarbage):
yield None
s.append(format % (i, self.referrersByReference[i]))
s.append(
'===== Referents (what is garbage item referring to?) ====='
)
for i in range(numGarbage):
yield None
s.append(format % (i, self.referentsByReference[i]))
self._report = s
if self._args.log:
self.printingBegin()
for i in range(len(self._report)):
if self.numGarbage > 0:
yield None
self.notify.info(self._report[i])
self.notify.info('===== Garbage Report Done =====')
self.printingEnd()
yield Job.Done
def run(self):
try:
self._leakDetector._index2containerId2len[self._index] = {}
ids = self._leakDetector.getContainerIds()
# record the current len of each container
for objId in ids:
yield None
try:
for result in self._leakDetector.getContainerByIdGen(objId):
yield None
container = result
except Exception as e:
# this container no longer exists
if self.notify.getDebug():
for contName in self._leakDetector.getContainerNameByIdGen(objId):
yield None
self.notify.debug(
'%s no longer exists; caught exception in getContainerById (%s)' % (
contName, e))
self._leakDetector.removeContainerById(objId)
continue
if container is None:
# this container no longer exists
if self.notify.getDebug():
for contName in self._leakDetector.getContainerNameByIdGen(objId):
yield None
self.notify.debug('%s no longer exists; getContainerById returned None' %
contName)
self._leakDetector.removeContainerById(objId)
continue
try:
cLen = len(container)
except Exception as e:
# this container no longer exists
if self.notify.getDebug():
for contName in self._leakDetector.getContainerNameByIdGen(objId):
yield None
self.notify.debug(
'%s is no longer a container, it is now %s (%s)' %
(contName, safeRepr(container), e))
self._leakDetector.removeContainerById(objId)
continue
self._leakDetector._index2containerId2len[self._index][objId] = cLen
# compare the current len of each container to past lens
if self._index > 0:
idx2id2len = self._leakDetector._index2containerId2len
for objId in idx2id2len[self._index]:
yield None
if objId in idx2id2len[self._index-1]:
diff = idx2id2len[self._index][objId] - idx2id2len[self._index-1][objId]
"""
# this check is too spammy
if diff > 20:
if diff > idx2id2len[self._index-1][objId]:
minutes = (self._leakDetector._index2delay[self._index] -
self._leakDetector._index2delay[self._index-1]) / 60.
name = self._leakDetector.getContainerNameById(objId)
if idx2id2len[self._index-1][objId] != 0:
percent = 100. * (float(diff) / float(idx2id2len[self._index-1][objId]))
try:
for container in self._leakDetector.getContainerByIdGen(objId):
yield None
except:
# TODO
self.notify.debug('caught exception in getContainerByIdGen (1)')
else:
self.notify.warning(
'%s (%s) grew %.2f%% in %.2f minutes (%s items at last measurement, current contents: %s)' % (
name, itype(container), percent, minutes, idx2id2len[self._index][objId],
fastRepr(container, maxLen=CheckContainers.ReprItems)))
yield None
"""
if (self._index > 2 and
objId in idx2id2len[self._index-2] and
objId in idx2id2len[self._index-3]):
diff2 = idx2id2len[self._index-1][objId] - idx2id2len[self._index-2][objId]
diff3 = idx2id2len[self._index-2][objId] - idx2id2len[self._index-3][objId]
if self._index <= 4:
if diff > 0 and diff2 > 0 and diff3 > 0:
name = self._leakDetector.getContainerNameById(objId)
try:
for container in self._leakDetector.getContainerByIdGen(objId):
yield None
except:
# TODO
self.notify.debug('caught exception in getContainerByIdGen (2)')
else:
msg = ('%s (%s) consistently increased in size over the last '
'3 periods (%s items at last measurement, current contents: %s)' %
(name, itype(container), idx2id2len[self._index][objId],
fastRepr(container, maxLen=CheckContainers.ReprItems)))
self.notify.warning(msg)
yield None
elif (objId in idx2id2len[self._index-4] and
objId in idx2id2len[self._index-5]):
# if size has consistently increased over the last 5 checks,
# send out a warning
diff4 = idx2id2len[self._index-3][objId] - idx2id2len[self._index-4][objId]
diff5 = idx2id2len[self._index-4][objId] - idx2id2len[self._index-5][objId]
if diff > 0 and diff2 > 0 and diff3 > 0 and diff4 > 0 and diff5 > 0:
name = self._leakDetector.getContainerNameById(objId)
try:
for container in self._leakDetector.getContainerByIdGen(objId):
yield None
except:
# TODO
self.notify.debug('caught exception in getContainerByIdGen (3)')
else:
msg = ('leak detected: %s (%s) consistently increased in size over the last '
'5 periods (%s items at last measurement, current contents: %s)' %
(name, itype(container), idx2id2len[self._index][objId],
fastRepr(container, maxLen=CheckContainers.ReprItems)))
self.notify.warning(msg)
yield None
messenger.send(self._leakDetector.getLeakEvent(), [container, name])
if config.GetBool('pdb-on-leak-detect', 0):
import pdb;pdb.set_trace()
pass
except Exception as e:
print('CheckContainers job caught exception: %s' % e)
if __dev__:
raise
yield Job.Done