def flush(self): """ Delete each item in the cache then clear all references to them """ assert self.checkCache() CRCache.notify.debug("Flushing the cache") # NOTE: delayDeleted objects should no longer get into the cache in the first place # give objects a chance to clean themselves up before checking for DelayDelete leaks messenger.send('clientCleanup') # some of these objects might be holding delayDeletes on others # track each object that is delayDeleted after it gets its chance to delete, # and check them after all objects have had a chance to delete delayDeleted = [] for distObj in self.dict.values(): distObj.deleteOrDelay() if distObj.getDelayDeleteCount() != 0: delayDeleted.append(distObj) if distObj.getDelayDeleteCount() <= 0: # make sure we're not leaking distObj.detectLeaks() # now that all objects have had a chance to delete, are there any objects left # that are still delayDeleted? delayDeleteLeaks = [] for distObj in delayDeleted: if distObj.getDelayDeleteCount() != 0: delayDeleteLeaks.append(distObj) if len(delayDeleteLeaks) > 0: s = 'CRCache.flush:' for obj in delayDeleteLeaks: s += ('\n could not delete %s (%s), delayDeletes=%s' % (safeRepr(obj), itype(obj), obj.getDelayDeleteNames())) self.notify.error(s) # Null out all references to the objects so they will get gcd self.dict = {} self.fifo = []
def printDelayDeletes(self): print('DelayDeletes:') print('=============') for obj in self._delayDeletedDOs.values(): print('%s\t%s (%s)\tdelayDeletes=%s' % (obj.doId, safeRepr(obj), itype(obj), obj.getDelayDeleteNames()))
def __init__(self, objects): self._objs = list(objects) self._type2objs = {} self._count2types = {} self._len2obj = {} type2count = {} for obj in self._objs: typ = itype(obj) type2count.setdefault(typ, 0) type2count[typ] += 1 self._type2objs.setdefault(typ, []) self._type2objs[typ].append(obj) try: self._len2obj[len(obj)] = obj except: pass self._count2types = invertDictLossless(type2count)
def run(self): try: self._leakDetector._index2containerId2len[self._index] = { } ids = self._leakDetector.getContainerIds() for objId in ids: yield None try: for result in self._leakDetector.getContainerByIdGen(objId): yield None container = result except Exception: e = None if self.notify.getDebug(): for contName in self._leakDetector.getContainerNameByIdGen(objId): yield None self.notify.debug('%s no longer exists; caught exception in getContainerById (%s)' % (contName, e)) self._leakDetector.removeContainerById(objId) continue if container is None: if self.notify.getDebug(): for contName in self._leakDetector.getContainerNameByIdGen(objId): yield None self.notify.debug('%s no longer exists; getContainerById returned None' % contName) self._leakDetector.removeContainerById(objId) continue try: cLen = len(container) except Exception: e = None if self.notify.getDebug(): for contName in self._leakDetector.getContainerNameByIdGen(objId): yield None self.notify.debug('%s is no longer a container, it is now %s (%s)' % (contName, safeRepr(container), e)) self._leakDetector.removeContainerById(objId) continue self._leakDetector._index2containerId2len[self._index][objId] = cLen if self._index > 0: idx2id2len = self._leakDetector._index2containerId2len for objId in idx2id2len[self._index]: yield None if objId in idx2id2len[self._index - 1]: diff = idx2id2len[self._index][objId] - idx2id2len[self._index - 1][objId] if self._index > 2 and objId in idx2id2len[self._index - 2] and objId in idx2id2len[self._index - 3]: diff2 = idx2id2len[self._index - 1][objId] - idx2id2len[self._index - 2][objId] diff3 = idx2id2len[self._index - 2][objId] - idx2id2len[self._index - 3][objId] if self._index <= 4: if diff > 0 and diff2 > 0 and diff3 > 0: name = self._leakDetector.getContainerNameById(objId) try: for container in self._leakDetector.getContainerByIdGen(objId): yield None except: self.notify.debug('caught exception in getContainerByIdGen (2)') msg = '%s (%s) consistently increased in size over the last 3 periods (%s items at last measurement, current contents: %s)' % (name, itype(container), idx2id2len[self._index][objId], fastRepr(container, maxLen = CheckContainers.ReprItems)) self.notify.warning(msg) yield None elif objId in idx2id2len[self._index - 4] and objId in idx2id2len[self._index - 5]: diff4 = idx2id2len[self._index - 3][objId] - idx2id2len[self._index - 4][objId] diff5 = idx2id2len[self._index - 4][objId] - idx2id2len[self._index - 5][objId] if diff > 0 and diff2 > 0 and diff3 > 0 and diff4 > 0 and diff5 > 0: name = self._leakDetector.getContainerNameById(objId) try: for container in self._leakDetector.getContainerByIdGen(objId): yield None except: self.notify.debug('caught exception in getContainerByIdGen (3)') msg = 'leak detected: %s (%s) consistently increased in size over the last 5 periods (%s items at last measurement, current contents: %s)' % (name, itype(container), idx2id2len[self._index][objId], fastRepr(container, maxLen = CheckContainers.ReprItems)) self.notify.warning(msg) yield None messenger.send(self._leakDetector.getLeakEvent(), [ container, name]) if config.GetBool('pdb-on-leak-detect', 0): import pdb as pdb pdb.set_trace() objId in idx2id2len[self._index - 3] except Exception: e = None print 'CheckContainers job caught exception: %s' % e if __dev__: raise yield Job.Done
def run(self): try: self._leakDetector._index2containerId2len[self._index] = {} ids = self._leakDetector.getContainerIds() # record the current len of each container for objId in ids: yield None try: for result in self._leakDetector.getContainerByIdGen(objId): yield None container = result except Exception as e: # this container no longer exists if self.notify.getDebug(): for contName in self._leakDetector.getContainerNameByIdGen(objId): yield None self.notify.debug( '%s no longer exists; caught exception in getContainerById (%s)' % ( contName, e)) self._leakDetector.removeContainerById(objId) continue if container is None: # this container no longer exists if self.notify.getDebug(): for contName in self._leakDetector.getContainerNameByIdGen(objId): yield None self.notify.debug('%s no longer exists; getContainerById returned None' % contName) self._leakDetector.removeContainerById(objId) continue try: cLen = len(container) except Exception as e: # this container no longer exists if self.notify.getDebug(): for contName in self._leakDetector.getContainerNameByIdGen(objId): yield None self.notify.debug( '%s is no longer a container, it is now %s (%s)' % (contName, safeRepr(container), e)) self._leakDetector.removeContainerById(objId) continue self._leakDetector._index2containerId2len[self._index][objId] = cLen # compare the current len of each container to past lens if self._index > 0: idx2id2len = self._leakDetector._index2containerId2len for objId in idx2id2len[self._index]: yield None if objId in idx2id2len[self._index-1]: diff = idx2id2len[self._index][objId] - idx2id2len[self._index-1][objId] """ # this check is too spammy if diff > 20: if diff > idx2id2len[self._index-1][objId]: minutes = (self._leakDetector._index2delay[self._index] - self._leakDetector._index2delay[self._index-1]) / 60. name = self._leakDetector.getContainerNameById(objId) if idx2id2len[self._index-1][objId] != 0: percent = 100. * (float(diff) / float(idx2id2len[self._index-1][objId])) try: for container in self._leakDetector.getContainerByIdGen(objId): yield None except: # TODO self.notify.debug('caught exception in getContainerByIdGen (1)') else: self.notify.warning( '%s (%s) grew %.2f%% in %.2f minutes (%s items at last measurement, current contents: %s)' % ( name, itype(container), percent, minutes, idx2id2len[self._index][objId], fastRepr(container, maxLen=CheckContainers.ReprItems))) yield None """ if (self._index > 2 and objId in idx2id2len[self._index-2] and objId in idx2id2len[self._index-3]): diff2 = idx2id2len[self._index-1][objId] - idx2id2len[self._index-2][objId] diff3 = idx2id2len[self._index-2][objId] - idx2id2len[self._index-3][objId] if self._index <= 4: if diff > 0 and diff2 > 0 and diff3 > 0: name = self._leakDetector.getContainerNameById(objId) try: for container in self._leakDetector.getContainerByIdGen(objId): yield None except: # TODO self.notify.debug('caught exception in getContainerByIdGen (2)') else: msg = ('%s (%s) consistently increased in size over the last ' '3 periods (%s items at last measurement, current contents: %s)' % (name, itype(container), idx2id2len[self._index][objId], fastRepr(container, maxLen=CheckContainers.ReprItems))) self.notify.warning(msg) yield None elif (objId in idx2id2len[self._index-4] and objId in idx2id2len[self._index-5]): # if size has consistently increased over the last 5 checks, # send out a warning diff4 = idx2id2len[self._index-3][objId] - idx2id2len[self._index-4][objId] diff5 = idx2id2len[self._index-4][objId] - idx2id2len[self._index-5][objId] if diff > 0 and diff2 > 0 and diff3 > 0 and diff4 > 0 and diff5 > 0: name = self._leakDetector.getContainerNameById(objId) try: for container in self._leakDetector.getContainerByIdGen(objId): yield None except: # TODO self.notify.debug('caught exception in getContainerByIdGen (3)') else: msg = ('leak detected: %s (%s) consistently increased in size over the last ' '5 periods (%s items at last measurement, current contents: %s)' % (name, itype(container), idx2id2len[self._index][objId], fastRepr(container, maxLen=CheckContainers.ReprItems))) self.notify.warning(msg) yield None messenger.send(self._leakDetector.getLeakEvent(), [container, name]) if config.GetBool('pdb-on-leak-detect', 0): import pdb;pdb.set_trace() pass except Exception as e: print('CheckContainers job caught exception: %s' % e) if __dev__: raise yield Job.Done
def run(self): # set of ids of objects that we know are always attached to builtin; # if an object is attached to one of these, it's attached to builtin # this cuts down on the amount of searching that needs to be done builtinIds = set() builtinIds.add(id(builtins.__dict__)) try: builtinIds.add(id(base)) builtinIds.add(id(base.cr)) builtinIds.add(id(base.cr.doId2do)) except: pass try: builtinIds.add(id(simbase)) builtinIds.add(id(simbase.air)) builtinIds.add(id(simbase.air.doId2do)) except: pass try: builtinIds.add(id(uber)) builtinIds.add(id(uber.air)) builtinIds.add(id(uber.air.doId2do)) except: pass while True: yield None objects = list(messenger._Messenger__objectEvents.keys()) assert self.notify.debug('%s objects in the messenger' % len(objects)) for object in objects: yield None assert self.notify.debug('---> new object: %s' % itype(object)) # try to find a path to builtin that doesn't involve the messenger # lists of objects for breadth-first search # iterate through one list while populating other list objList1 = [] objList2 = [] curObjList = objList1 nextObjList = objList2 visitedObjIds = set() # add the id of the object, and the messenger containers so that # the search for builtin will stop at the messenger; we're looking # for any path to builtin that don't involve the messenger visitedObjIds.add(id(object)) visitedObjIds.add(id(messenger._Messenger__objectEvents)) visitedObjIds.add(id(messenger._Messenger__callbacks)) nextObjList.append(object) foundBuiltin = False # breadth-first search, go until you run out of new objects or you find __builtin__ while len(nextObjList) > 0: if foundBuiltin: break # swap the lists, prepare for the next pass curObjList = nextObjList nextObjList = [] assert self.notify.debug( 'next search iteration, num objects: %s' % len(curObjList)) for curObj in curObjList: if foundBuiltin: break yield None referrers = gc.get_referrers(curObj) assert self.notify.debug( 'curObj: %s @ %s, %s referrers, repr=%s' % (itype(curObj), hex(id(curObj)), len(referrers), fastRepr(curObj, maxLen=2))) for referrer in referrers: #assert self.notify.debug('referrer: %s' % itype(curObj)) yield None refId = id(referrer) # don't go in a loop if refId in visitedObjIds: #assert self.notify.debug('already visited') continue # don't self-reference if referrer is curObjList or referrer is nextObjList: continue if refId in builtinIds: # not a leak, there is a path to builtin that does not involve the messenger #assert self.notify.debug('object has another path to __builtin__, it\'s not a messenger leak') foundBuiltin = True break else: visitedObjIds.add(refId) nextObjList.append(referrer) if not foundBuiltin: self.notify.warning( '%s is referenced only by the messenger' % (itype(object)))
def run(self): # do the garbage collection oldFlags = gc.get_debug() if self._args.delOnly: # do a collect without SAVEALL, to identify the instances that are involved in # cycles with instances that define __del__ # cycles that do not involve any instances that define __del__ are cleaned up # automatically by Python, but they also appear in gc.garbage when SAVEALL is set gc.set_debug(0) if self._args.collect: gc.collect() garbageInstances = gc.garbage[:] del gc.garbage[:] # only yield if there's more time-consuming work to do, # if there's no garbage, give instant feedback if len(garbageInstances) > 0: yield None # don't repr the garbage list if we don't have to if self.notify.getDebug(): self.notify.debug('garbageInstances == %s' % fastRepr(garbageInstances)) self.numGarbageInstances = len(garbageInstances) # grab the ids of the garbage instances (objects with __del__) self.garbageInstanceIds = set() for i in range(len(garbageInstances)): self.garbageInstanceIds.add(id(garbageInstances[i])) if i % 20 == 0: yield None # then release the list of instances so that it doesn't interfere with the gc.collect() below del garbageInstances else: self.garbageInstanceIds = set() # do a SAVEALL pass so that we have all of the objects involved in legitimate garbage cycles # without SAVEALL, gc.garbage only contains objects with __del__ methods gc.set_debug(gc.DEBUG_SAVEALL) if self._args.collect: gc.collect() self.garbage = gc.garbage[:] del gc.garbage[:] # only yield if there's more time-consuming work to do, # if there's no garbage, give instant feedback if len(self.garbage) > 0: yield None # don't repr the garbage list if we don't have to if self.notify.getDebug(): self.notify.debug('self.garbage == %s' % fastRepr(self.garbage)) gc.set_debug(oldFlags) self.numGarbage = len(self.garbage) # only yield if there's more time-consuming work to do, # if there's no garbage, give instant feedback if self.numGarbage > 0: yield None if self._args.verbose: self.notify.info('found %s garbage items' % self.numGarbage) # print the types of the garbage first, in case the repr of an object # causes a crash #if self.numGarbage > 0: # self.notify.info('TYPES ONLY (this is only needed if a crash occurs before GarbageReport finishes):') # for result in printNumberedTypesGen(self.garbage): # yield None # Py obj id -> garbage list index self._id2index = {} self.referrersByReference = {} self.referrersByNumber = {} self.referentsByReference = {} self.referentsByNumber = {} self._id2garbageInfo = {} self.cycles = [] self.cyclesBySyntax = [] self.uniqueCycleSets = set() self.cycleIds = set() # make the id->index table to speed up the next steps for i in range(self.numGarbage): self._id2index[id(self.garbage[i])] = i if i % 20 == 0: yield None # grab the referrers (pointing to garbage) if self._args.fullReport and (self.numGarbage != 0): if self._args.verbose: self.notify.info('getting referrers...') for i in range(self.numGarbage): yield None for result in self._getReferrers(self.garbage[i]): yield None byNum, byRef = result self.referrersByNumber[i] = byNum self.referrersByReference[i] = byRef # grab the referents (pointed to by garbage) if self.numGarbage > 0: if self._args.verbose: self.notify.info('getting referents...') for i in range(self.numGarbage): yield None for result in self._getReferents(self.garbage[i]): yield None byNum, byRef = result self.referentsByNumber[i] = byNum self.referentsByReference[i] = byRef for i in range(self.numGarbage): if hasattr(self.garbage[i], '_garbageInfo') and callable( self.garbage[i]._garbageInfo): try: info = self.garbage[i]._garbageInfo() except Exception as e: info = str(e) self._id2garbageInfo[id(self.garbage[i])] = info yield None else: if i % 20 == 0: yield None # find the cycles if self._args.findCycles and self.numGarbage > 0: if self._args.verbose: self.notify.info('calculating cycles...') for i in range(self.numGarbage): yield None for newCycles in self._getCycles(i, self.uniqueCycleSets): yield None self.cycles.extend(newCycles) # create a representation of the cycle in human-readable form newCyclesBySyntax = [] for cycle in newCycles: cycleBySyntax = '' objs = [] # leave off the last index, it's a repeat of the first index for index in cycle[:-1]: objs.append(self.garbage[index]) yield None # make the list repeat so we can safely iterate off the end numObjs = len(objs) - 1 objs.extend(objs) # state variables for our loop below numToSkip = 0 objAlreadyRepresented = False # if cycle starts off with an instance dict, start with the instance instead startIndex = 0 # + 1 to include a reference back to the first object endIndex = numObjs + 1 if inspect.isclass(objs[-1]) and type(objs[0]) is dict: startIndex -= 1 endIndex -= 1 for index in range(startIndex, endIndex): if numToSkip: numToSkip -= 1 continue obj = objs[index] if inspect.isclass(obj): if not objAlreadyRepresented: cycleBySyntax += '%s' % obj.__class__.__name__ cycleBySyntax += '.' # skip past the instance dict and get the member obj numToSkip += 1 member = objs[index + 2] for key, value in obj.__dict__.items(): if value is member: break yield None else: key = '<unknown member name>' cycleBySyntax += '%s' % key objAlreadyRepresented = True elif type(obj) is dict: cycleBySyntax += '{' # get object referred to by dict val = objs[index + 1] for key, value in obj.items(): if value is val: break yield None else: key = '<unknown key>' cycleBySyntax += '%s}' % fastRepr(key) objAlreadyRepresented = True elif type(obj) in (tuple, list): brackets = { tuple: '()', list: '[]', }[type(obj)] # get object being referenced by container nextObj = objs[index + 1] cycleBySyntax += brackets[0] for index in range(len(obj)): if obj[index] is nextObj: index = str(index) break yield None else: index = '<unknown index>' cycleBySyntax += '%s%s' % (index, brackets[1]) objAlreadyRepresented = True else: cycleBySyntax += '%s --> ' % itype(obj) objAlreadyRepresented = False newCyclesBySyntax.append(cycleBySyntax) yield None self.cyclesBySyntax.extend(newCyclesBySyntax) # if we're not doing a full report, add this cycle's IDs to the master set if not self._args.fullReport: for cycle in newCycles: yield None self.cycleIds.update(set(cycle)) self.numCycles = len(self.cycles) if self._args.findCycles: s = [ '===== GarbageReport: \'%s\' (%s %s) =====' % (self._args.name, self.numCycles, ('cycle' if self.numCycles == 1 else 'cycles')) ] else: s = ['===== GarbageReport: \'%s\' =====' % (self._args.name)] if self.numGarbage > 0: # make a list of the ids we will actually be printing if self._args.fullReport: garbageIndices = range(self.numGarbage) else: garbageIndices = list(self.cycleIds) garbageIndices.sort() numGarbage = len(garbageIndices) # log each individual item with a number in front of it if not self._args.fullReport: abbrev = '(abbreviated) ' else: abbrev = '' s.append('===== Garbage Items %s=====' % abbrev) digits = 0 n = numGarbage while n > 0: yield None digits += 1 n /= 10 digits = digits format = '%0' + '%s' % digits + 'i:%s \t%s' for i in range(numGarbage): yield None idx = garbageIndices[i] if self._args.safeMode: # in safe mode, don't try to repr any of the objects objStr = repr(itype(self.garbage[idx])) else: objStr = fastRepr(self.garbage[idx]) maxLen = 5000 if len(objStr) > maxLen: snip = '<SNIP>' objStr = '%s%s' % (objStr[:(maxLen - len(snip))], snip) s.append(format % (idx, itype(self.garbage[idx]), objStr)) # also log the types of the objects s.append('===== Garbage Item Types %s=====' % abbrev) for i in range(numGarbage): yield None idx = garbageIndices[i] objStr = str(deeptype(self.garbage[idx])) maxLen = 5000 if len(objStr) > maxLen: snip = '<SNIP>' objStr = '%s%s' % (objStr[:(maxLen - len(snip))], snip) s.append(format % (idx, itype(self.garbage[idx]), objStr)) if self._args.findCycles: s.append('===== Garbage Cycles (Garbage Item Numbers) =====') ac = AlphabetCounter() for i in range(self.numCycles): yield None s.append('%s:%s' % (ac.next(), self.cycles[i])) if self._args.findCycles: s.append('===== Garbage Cycles (Python Syntax) =====') ac = AlphabetCounter() for i in range(len(self.cyclesBySyntax)): yield None s.append('%s:%s' % (ac.next(), self.cyclesBySyntax[i])) if len(self._id2garbageInfo) > 0: s.append('===== Garbage Custom Info =====') ac = AlphabetCounter() for i in range(len(self.cyclesBySyntax)): yield None counter = ac.next() _id = id(self.garbage[i]) if _id in self._id2garbageInfo: s.append('%s:%s' % (counter, self._id2garbageInfo[_id])) if self._args.fullReport: format = '%0' + '%s' % digits + 'i:%s' s.append( '===== Referrers By Number (what is referring to garbage item?) =====' ) for i in range(numGarbage): yield None s.append(format % (i, self.referrersByNumber[i])) s.append( '===== Referents By Number (what is garbage item referring to?) =====' ) for i in range(numGarbage): yield None s.append(format % (i, self.referentsByNumber[i])) s.append( '===== Referrers (what is referring to garbage item?) =====' ) for i in range(numGarbage): yield None s.append(format % (i, self.referrersByReference[i])) s.append( '===== Referents (what is garbage item referring to?) =====' ) for i in range(numGarbage): yield None s.append(format % (i, self.referentsByReference[i])) self._report = s if self._args.log: self.printingBegin() for i in range(len(self._report)): if self.numGarbage > 0: yield None self.notify.info(self._report[i]) self.notify.info('===== Garbage Report Done =====') self.printingEnd() yield Job.Done
def run(self): try: self._leakDetector._index2containerId2len[self._index] = {} ids = self._leakDetector.getContainerIds() # record the current len of each container for objId in ids: yield None try: for result in self._leakDetector.getContainerByIdGen(objId): yield None container = result except Exception as e: # this container no longer exists if self.notify.getDebug(): for contName in self._leakDetector.getContainerNameByIdGen(objId): yield None self.notify.debug( '%s no longer exists; caught exception in getContainerById (%s)' % ( contName, e)) self._leakDetector.removeContainerById(objId) continue if container is None: # this container no longer exists if self.notify.getDebug(): for contName in self._leakDetector.getContainerNameByIdGen(objId): yield None self.notify.debug('%s no longer exists; getContainerById returned None' % contName) self._leakDetector.removeContainerById(objId) continue try: cLen = len(container) except Exception as e: # this container no longer exists if self.notify.getDebug(): for contName in self._leakDetector.getContainerNameByIdGen(objId): yield None self.notify.debug( '%s is no longer a container, it is now %s (%s)' % (contName, safeRepr(container), e)) self._leakDetector.removeContainerById(objId) continue self._leakDetector._index2containerId2len[self._index][objId] = cLen # compare the current len of each container to past lens if self._index > 0: idx2id2len = self._leakDetector._index2containerId2len for objId in idx2id2len[self._index]: yield None if objId in idx2id2len[self._index-1]: diff = idx2id2len[self._index][objId] - idx2id2len[self._index-1][objId] """ # this check is too spammy if diff > 20: if diff > idx2id2len[self._index-1][objId]: minutes = (self._leakDetector._index2delay[self._index] - self._leakDetector._index2delay[self._index-1]) / 60. name = self._leakDetector.getContainerNameById(objId) if idx2id2len[self._index-1][objId] != 0: percent = 100. * (float(diff) / float(idx2id2len[self._index-1][objId])) try: for container in self._leakDetector.getContainerByIdGen(objId): yield None except: # TODO self.notify.debug('caught exception in getContainerByIdGen (1)') else: self.notify.warning( '%s (%s) grew %.2f%% in %.2f minutes (%s items at last measurement, current contents: %s)' % ( name, itype(container), percent, minutes, idx2id2len[self._index][objId], fastRepr(container, maxLen=CheckContainers.ReprItems))) yield None """ if (self._index > 2 and objId in idx2id2len[self._index-2] and objId in idx2id2len[self._index-3]): diff2 = idx2id2len[self._index-1][objId] - idx2id2len[self._index-2][objId] diff3 = idx2id2len[self._index-2][objId] - idx2id2len[self._index-3][objId] if self._index <= 4: if diff > 0 and diff2 > 0 and diff3 > 0: name = self._leakDetector.getContainerNameById(objId) try: for container in self._leakDetector.getContainerByIdGen(objId): yield None except: # TODO self.notify.debug('caught exception in getContainerByIdGen (2)') else: msg = ('%s (%s) consistently increased in size over the last ' '3 periods (%s items at last measurement, current contents: %s)' % (name, itype(container), idx2id2len[self._index][objId], fastRepr(container, maxLen=CheckContainers.ReprItems))) self.notify.warning(msg) yield None elif (objId in idx2id2len[self._index-4] and objId in idx2id2len[self._index-5]): # if size has consistently increased over the last 5 checks, # send out a warning diff4 = idx2id2len[self._index-3][objId] - idx2id2len[self._index-4][objId] diff5 = idx2id2len[self._index-4][objId] - idx2id2len[self._index-5][objId] if diff > 0 and diff2 > 0 and diff3 > 0 and diff4 > 0 and diff5 > 0: name = self._leakDetector.getContainerNameById(objId) try: for container in self._leakDetector.getContainerByIdGen(objId): yield None except: # TODO self.notify.debug('caught exception in getContainerByIdGen (3)') else: msg = ('leak detected: %s (%s) consistently increased in size over the last ' '5 periods (%s items at last measurement, current contents: %s)' % (name, itype(container), idx2id2len[self._index][objId], fastRepr(container, maxLen=CheckContainers.ReprItems))) self.notify.warning(msg) yield None messenger.send(self._leakDetector.getLeakEvent(), [container, name]) if config.GetBool('pdb-on-leak-detect', 0): import pdb;pdb.set_trace() pass except Exception as e: print('CheckContainers job caught exception: %s' % e) if __dev__: raise yield Job.Done