def sendLoadAverageWarning(interval, limit, load): cpu_count = multiprocessing.cpu_count() sendAdministratorMessage("watchdog", "%d-minute load average" % interval, ("The current %d-minute load average is %.2f!\n" % (interval, load)) + ("The configured limit is %.2f (%.2f x %d CPUs).\n" % (limit, limit / cpu_count, cpu_count)) + "\n" + "-- critic\n")
def run(self): soft_restart_attempted = set() previous = {} while not self.terminated: self.interrupted = False pidfile_dir = configuration.paths.WSGI_PIDFILE_DIR if os.path.isdir(pidfile_dir): pids = set(map(int, os.listdir(pidfile_dir))) else: pids = set() for pid in pids: try: rss = getRSS(pid) except IOError, error: if error.errno == errno.ENOENT: self.warning("unlinking stale pid-file: %s" % os.path.join(pidfile_dir, str(pid))) os.unlink(os.path.join(pidfile_dir, str(pid))) continue else: raise if previous.get(pid) != rss: self.debug("pid=%d, rss=%d bytes" % (pid, rss)) previous[pid] = rss if rss > configuration.services.WATCHDOG["rss_hard_limit"]: sendAdministratorMessage( "watchdog", "pid(%d): hard memory limit exceeded" % pid, ("Current RSS: %d kilobytes\nSending process SIGKILL (%d).\n\n-- critic" % (rss, signal.SIGKILL))) self.info( "Killing pid(%d): hard memory limit exceeded, RSS: %d kilobytes" % (pid, rss)) os.kill(pid, signal.SIGKILL) elif rss > configuration.services.WATCHDOG[ "rss_soft_limit"] and pid not in soft_restart_attempted: sendAdministratorMessage( "watchdog", "pid(%d): soft memory limit exceeded" % pid, ("Current RSS: %d kilobytes\nSending process SIGINT (%d).\n\n" % (rss, signal.SIGINT))) self.info( "Killing pid(%d): soft memory limit exceeded, RSS: %d kilobytes" % (pid, rss)) os.kill(pid, signal.SIGINT) soft_restart_attempted.add(pid) for pid in previous.keys(): if pid not in pids: del previous[pid] soft_restart_attempted = soft_restart_attempted & pids time.sleep(10)
def sendLoadAverageWarning(interval, limit, load): cpu_count = multiprocessing.cpu_count() mailutils.sendAdministratorMessage( "watchdog", "%d-minute load average" % interval, ("The current %d-minute load average is %.2f!\n" % (interval, load)) + ("The configured limit is %.2f (%.2f x %d CPUs).\n" % (limit, limit / cpu_count, cpu_count)) + "\n-- critic\n")
def run(self): soft_restart_attempted = set() previous = {} while not self.terminated: self.interrupted = False pidfile_dir = configuration.paths.WSGI_PIDFILE_DIR if os.path.isdir(pidfile_dir): pids = set(map(int, os.listdir(pidfile_dir))) else: pids = [] for pid in pids: try: rss = getRSS(pid) except IOError, error: if error.errno == errno.ENOENT: self.warning("unlinking stale pid-file: %s" % os.path.join(pidfile_dir, str(pid))) os.unlink(os.path.join(pidfile_dir, str(pid))) continue else: raise if previous.get(pid) != rss: self.debug("pid=%d, rss=%d bytes" % (pid, rss)) previous[pid] = rss if rss > configuration.services.WATCHDOG["rss_hard_limit"]: sendAdministratorMessage("watchdog", "pid(%d): hard memory limit exceeded" % pid, ("Current RSS: %d kilobytes\nSending process SIGKILL (%d).\n\n-- critic" % (rss, signal.SIGKILL))) self.info("Killing pid(%d): hard memory limit exceeded, RSS: %d kilobytes" % (pid, rss)) os.kill(pid, signal.SIGKILL) elif rss > configuration.services.WATCHDOG["rss_soft_limit"] and pid not in soft_restart_attempted: sendAdministratorMessage("watchdog", "pid(%d): soft memory limit exceeded" % process.pid, ("Current RSS: %d kilobytes\nSending process SIGINT (%d).\n\n%s" % (rss, signal.SIGINT))) self.info("Killing pid(%d): soft memory limit exceeded, RSS: %d kilobytes" % (pid, rss)) os.kill(pid, signal.SIGINT) soft_restart_attempted.add(pid) for pid in previous.keys(): if pid not in pids: del previous[pid] soft_restart_attempted = soft_restart_attempted & pids time.sleep(10)
def sendUserCreatedMail(self, source, external=None): import mailutils if self.email_verified is False: email_status = " (pending verification)" else: email_status = "" message = """\ A new user has been created: User name: %(username)r Full name: %(fullname)r Email: %(email)r%(email_status)s """ % { "username": self.name, "fullname": self.fullname, "email": self.email, "email_status": email_status } if external: import auth provider = auth.PROVIDERS[external["provider"]] message += """\ External: %(provider)s %(account)r """ % { "provider": provider.getTitle(), "account": external["account"] } message += """\ -- critic """ mailutils.sendAdministratorMessage(source, "User '%s' registered" % self.name, message)
def sendUserCreatedMail(self, source, external=None): import mailutils if self.email_verified is False: email_status = " (pending verification)" else: email_status = "" message = """\ A new user has been created: User name: %(username)r Full name: %(fullname)r Email: %(email)r%(email_status)s """ % { "username": self.name, "fullname": self.fullname, "email": self.email, "email_status": email_status, } if external: import auth provider = auth.PROVIDERS[external["provider"]] message += """\ External: %(provider)s %(account)r """ % { "provider": provider.getTitle(), "account": external["account"], } message += """\ -- critic """ mailutils.sendAdministratorMessage(source, "User '%s' registered" % self.name, message)
def run(self): soft_restart_attempted = set() previous = {} getloadavg_failed = False load1_has_warned = 0 load1_last_time = 0 load5_has_warned = 0 load5_last_time = 0 load15_has_warned = 0 load15_last_time = 0 while not self.terminated: self.interrupted = False def sendLoadAverageWarning(interval, limit, load): cpu_count = multiprocessing.cpu_count() mailutils.sendAdministratorMessage( "watchdog", "%d-minute load average" % interval, ("The current %d-minute load average is %.2f!\n" % (interval, load)) + ("The configured limit is %.2f (%.2f x %d CPUs).\n" % (limit, limit / cpu_count, cpu_count)) + "\n-- critic\n") try: load1, load5, load15 = os.getloadavg() self.debug("load average: %r, %r, %r" % (load1, load5, load15)) except OSError: load1, load5, load15 = 0, 0, 0 if not getloadavg_failed: self.exception("failed to detect system load average") getloadavg_failed = True now = time.time() if self.load1_limit and load1 > self.load1_limit: if load1 > load1_has_warned * 1.2 or now - load1_last_time > 60: sendLoadAverageWarning(1, self.load1_limit, load1) load1_has_warned = load1 load1_last_time = now else: load1_has_warned = 0 load1_last_time = 0 if self.load5_limit and load5 > self.load5_limit: if load5 > load5_has_warned * 1.2 or now - load5_last_time > 5 * 60: sendLoadAverageWarning(5, self.load5_limit, load5) load5_has_warned = load5 load5_last_time = now else: load5_has_warned = 0 load5_last_time = 0 if self.load15_limit and load15 > self.load15_limit: if load15 > load15_has_warned * 1.2 or now - load15_last_time > 15 * 60: sendLoadAverageWarning(15, self.load15_limit, load15) load15_has_warned = load15 load15_last_time = now else: load15_has_warned = 0 load15_last_time = 0 pidfile_dir = configuration.paths.WSGI_PIDFILE_DIR if os.path.isdir(pidfile_dir): pids = set(map(int, os.listdir(pidfile_dir))) else: pids = set() for pid in pids: try: rss = getRSS(pid) except IOError as error: if error.errno == errno.ENOENT: self.warning("unlinking stale pid-file: %s" % os.path.join(pidfile_dir, str(pid))) os.unlink(os.path.join(pidfile_dir, str(pid))) continue else: raise if previous.get(pid) != rss: self.debug("pid=%d, rss=%d bytes" % (pid, rss)) previous[pid] = rss if rss > configuration.services.WATCHDOG["rss_hard_limit"]: mailutils.sendAdministratorMessage( "watchdog", "pid(%d): hard memory limit exceeded" % pid, ("Current RSS: %d kilobytes\nSending process SIGKILL (%d).\n\n-- critic" % (rss, signal.SIGKILL))) self.info("Killing pid(%d): hard memory limit exceeded, RSS: %d kilobytes" % (pid, rss)) os.kill(pid, signal.SIGKILL) elif rss > configuration.services.WATCHDOG["rss_soft_limit"] and pid not in soft_restart_attempted: mailutils.sendAdministratorMessage( "watchdog", "pid(%d): soft memory limit exceeded" % pid, ("Current RSS: %d kilobytes\nSending process SIGINT (%d).\n\n" % (rss, signal.SIGINT))) self.info("Killing pid(%d): soft memory limit exceeded, RSS: %d kilobytes" % (pid, rss)) os.kill(pid, signal.SIGINT) soft_restart_attempted.add(pid) for pid in previous.keys(): if pid not in pids: del previous[pid] soft_restart_attempted = soft_restart_attempted & pids time.sleep(10)
def emit(self, record): import mailutils mailutils.sendAdministratorMessage(self.__logfile_name, record.message.splitlines()[0], self.formatter.format(record))
def run(self): soft_restart_attempted = set() previous = {} getloadavg_failed = False load1_has_warned = 0 load1_last_time = 0 load5_has_warned = 0 load5_last_time = 0 load15_has_warned = 0 load15_last_time = 0 while not self.terminated: self.interrupted = False def sendLoadAverageWarning(interval, limit, load): cpu_count = multiprocessing.cpu_count() mailutils.sendAdministratorMessage( "watchdog", "%d-minute load average" % interval, ("The current %d-minute load average is %.2f!\n" % (interval, load)) + ("The configured limit is %.2f (%.2f x %d CPUs).\n" % (limit, limit / cpu_count, cpu_count)) + "\n-- critic\n") try: load1, load5, load15 = os.getloadavg() self.debug("load average: %r, %r, %r" % (load1, load5, load15)) except OSError: load1, load5, load15 = 0, 0, 0 if not getloadavg_failed: self.exception("failed to detect system load average") getloadavg_failed = True now = time.time() if self.load1_limit and load1 > self.load1_limit: if load1 > load1_has_warned * 1.2 or now - load1_last_time > 60: sendLoadAverageWarning(1, self.load1_limit, load1) load1_has_warned = load1 load1_last_time = now else: load1_has_warned = 0 load1_last_time = 0 if self.load5_limit and load5 > self.load5_limit: if load5 > load5_has_warned * 1.2 or now - load5_last_time > 5 * 60: sendLoadAverageWarning(5, self.load5_limit, load5) load5_has_warned = load5 load5_last_time = now else: load5_has_warned = 0 load5_last_time = 0 if self.load15_limit and load15 > self.load15_limit: if load15 > load15_has_warned * 1.2 or now - load15_last_time > 15 * 60: sendLoadAverageWarning(15, self.load15_limit, load15) load15_has_warned = load15 load15_last_time = now else: load15_has_warned = 0 load15_last_time = 0 pidfile_dir = configuration.paths.WSGI_PIDFILE_DIR if os.path.isdir(pidfile_dir): pids = set(map(int, os.listdir(pidfile_dir))) else: pids = set() for pid in pids: try: rss = getRSS(pid) except IOError as error: if error.errno == errno.ENOENT: self.warning("unlinking stale pid-file: %s" % os.path.join(pidfile_dir, str(pid))) os.unlink(os.path.join(pidfile_dir, str(pid))) continue else: raise if previous.get(pid) != rss: self.debug("pid=%d, rss=%d bytes" % (pid, rss)) previous[pid] = rss if rss > configuration.services.WATCHDOG["rss_hard_limit"]: mailutils.sendAdministratorMessage( "watchdog", "pid(%d): hard memory limit exceeded" % pid, ("Current RSS: %d kilobytes\nSending process SIGKILL (%d).\n\n-- critic" % (rss, signal.SIGKILL))) self.info( "Killing pid(%d): hard memory limit exceeded, RSS: %d kilobytes" % (pid, rss)) os.kill(pid, signal.SIGKILL) elif rss > configuration.services.WATCHDOG[ "rss_soft_limit"] and pid not in soft_restart_attempted: mailutils.sendAdministratorMessage( "watchdog", "pid(%d): soft memory limit exceeded" % pid, ("Current RSS: %d kilobytes\nSending process SIGINT (%d).\n\n" % (rss, signal.SIGINT))) self.info( "Killing pid(%d): soft memory limit exceeded, RSS: %d kilobytes" % (pid, rss)) os.kill(pid, signal.SIGINT) soft_restart_attempted.add(pid) for pid in previous.keys(): if pid not in pids: del previous[pid] soft_restart_attempted = soft_restart_attempted & pids self.signal_idle_state() time.sleep(10)