def build_detectors(self, inbox): logs.context['step'] = 'build_detectors' detectors = [Detector(inbox) for Detector in self.features] verbose = self.should_enable_extra_debugging(inbox, detectors) for i, detector in enumerate(detectors): logs.context['detector'] = type(detector).__name__ detector.create_sender_profile(self.sender_profile_num_emails) logs.Watchdog.reset() if verbose: progress_logger.info('Finished creating {} sender profile, RSS = {}'.format(type(detector).__name__, MemTracker.cur_mem_usage())) MemTracker.logMemory('finished creating {} sender profile'.format(type(detector).__name__)) else: logs.RateLimitedMemTracker.checkmem('finished creating {} sender profile'.format(type(detector).__name__)) del logs.context['step'] del logs.context['detector'] return detectors
def should_enable_extra_debugging(self, inbox, detectors): if len(inbox) < 10000: return False detector_names = ', '.join([type(d).__name__ for d in detectors]) progress_logger.info('Enabling extra debugging for large inbox with {} messages, {}; RSS = {}, creating sender profiles for {}'.format(len(inbox), logs.context, MemTracker.cur_mem_usage(), detector_names)) return True
def build_detectors(self, inbox): logs.context['step'] = 'build_detectors' detectors = [Detector(inbox) for Detector in self.features] verbose = self.should_enable_extra_debugging(inbox, detectors) for i, detector in enumerate(detectors): logs.context['detector'] = type(detector).__name__ detector.create_sender_profile(self.sender_profile_indeces) logs.Watchdog.reset() if verbose: progress_logger.info('Finished creating {} sender profile, RSS = {}'.format(type(detector).__name__, MemTracker.cur_mem_usage())) MemTracker.logMemory('finished creating {} sender profile'.format(type(detector).__name__)) else: logs.RateLimitedMemTracker.checkmem('finished creating {} sender profile'.format(type(detector).__name__)) del logs.context['step'] del logs.context['detector'] return detectors
def checkmem_rate_limited(self): cur = MemTracker.cur_mem_usage() if cur >= 2*self.max_mem_logged: self.max_mem_logged = cur MemTracker.logMemory(self.task + "; " + str(context))
def checkmem_rate_limited(self): cur = MemTracker.cur_mem_usage() if cur >= 2 * self.max_mem_logged: self.max_mem_logged = cur MemTracker.logMemory(self.task + "; " + str(context))