def sendLoadAverageWarning(interval, limit, load):
cpu_count = multiprocessing.cpu_count()
sendAdministratorMessage("watchdog", "%d-minute load average" % interval,
("The current %d-minute load average is %.2f!\n" % (interval, load)) +
("The configured limit is %.2f (%.2f x %d CPUs).\n" % (limit, limit / cpu_count, cpu_count)) +
"\n" +
"-- critic\n")
def run(self):
soft_restart_attempted = set()
previous = {}
while not self.terminated:
self.interrupted = False
pidfile_dir = configuration.paths.WSGI_PIDFILE_DIR
if os.path.isdir(pidfile_dir):
pids = set(map(int, os.listdir(pidfile_dir)))
else:
pids = set()
for pid in pids:
try:
rss = getRSS(pid)
except IOError, error:
if error.errno == errno.ENOENT:
self.warning("unlinking stale pid-file: %s" %
os.path.join(pidfile_dir, str(pid)))
os.unlink(os.path.join(pidfile_dir, str(pid)))
continue
else:
raise
if previous.get(pid) != rss:
self.debug("pid=%d, rss=%d bytes" % (pid, rss))
previous[pid] = rss
if rss > configuration.services.WATCHDOG["rss_hard_limit"]:
sendAdministratorMessage(
"watchdog",
"pid(%d): hard memory limit exceeded" % pid,
("Current RSS: %d kilobytes\nSending process SIGKILL (%d).\n\n-- critic"
% (rss, signal.SIGKILL)))
self.info(
"Killing pid(%d): hard memory limit exceeded, RSS: %d kilobytes"
% (pid, rss))
os.kill(pid, signal.SIGKILL)
elif rss > configuration.services.WATCHDOG[
"rss_soft_limit"] and pid not in soft_restart_attempted:
sendAdministratorMessage(
"watchdog",
"pid(%d): soft memory limit exceeded" % pid,
("Current RSS: %d kilobytes\nSending process SIGINT (%d).\n\n"
% (rss, signal.SIGINT)))
self.info(
"Killing pid(%d): soft memory limit exceeded, RSS: %d kilobytes"
% (pid, rss))
os.kill(pid, signal.SIGINT)
soft_restart_attempted.add(pid)
for pid in previous.keys():
if pid not in pids: del previous[pid]
soft_restart_attempted = soft_restart_attempted & pids
time.sleep(10)
def sendLoadAverageWarning(interval, limit, load):
cpu_count = multiprocessing.cpu_count()
mailutils.sendAdministratorMessage(
"watchdog", "%d-minute load average" % interval,
("The current %d-minute load average is %.2f!\n" %
(interval, load)) +
("The configured limit is %.2f (%.2f x %d CPUs).\n" %
(limit, limit / cpu_count, cpu_count)) + "\n-- critic\n")
def run(self):
soft_restart_attempted = set()
previous = {}
while not self.terminated:
self.interrupted = False
pidfile_dir = configuration.paths.WSGI_PIDFILE_DIR
if os.path.isdir(pidfile_dir):
pids = set(map(int, os.listdir(pidfile_dir)))
else:
pids = []
for pid in pids:
try: rss = getRSS(pid)
except IOError, error:
if error.errno == errno.ENOENT:
self.warning("unlinking stale pid-file: %s" % os.path.join(pidfile_dir, str(pid)))
os.unlink(os.path.join(pidfile_dir, str(pid)))
continue
else: raise
if previous.get(pid) != rss:
self.debug("pid=%d, rss=%d bytes" % (pid, rss))
previous[pid] = rss
if rss > configuration.services.WATCHDOG["rss_hard_limit"]:
sendAdministratorMessage("watchdog", "pid(%d): hard memory limit exceeded" % pid,
("Current RSS: %d kilobytes\nSending process SIGKILL (%d).\n\n-- critic"
% (rss, signal.SIGKILL)))
self.info("Killing pid(%d): hard memory limit exceeded, RSS: %d kilobytes" % (pid, rss))
os.kill(pid, signal.SIGKILL)
elif rss > configuration.services.WATCHDOG["rss_soft_limit"] and pid not in soft_restart_attempted:
sendAdministratorMessage("watchdog", "pid(%d): soft memory limit exceeded" % process.pid,
("Current RSS: %d kilobytes\nSending process SIGINT (%d).\n\n%s"
% (rss, signal.SIGINT)))
self.info("Killing pid(%d): soft memory limit exceeded, RSS: %d kilobytes" % (pid, rss))
os.kill(pid, signal.SIGINT)
soft_restart_attempted.add(pid)
for pid in previous.keys():
if pid not in pids: del previous[pid]
soft_restart_attempted = soft_restart_attempted & pids
time.sleep(10)
def sendUserCreatedMail(self, source, external=None):
import mailutils
if self.email_verified is False:
email_status = " (pending verification)"
else:
email_status = ""
message = """\
A new user has been created:
User name: %(username)r
Full name: %(fullname)r
Email: %(email)r%(email_status)s
""" % {
"username": self.name,
"fullname": self.fullname,
"email": self.email,
"email_status": email_status
}
if external:
import auth
provider = auth.PROVIDERS[external["provider"]]
message += """\
External: %(provider)s %(account)r
""" % {
"provider": provider.getTitle(),
"account": external["account"]
}
message += """\
-- critic
"""
mailutils.sendAdministratorMessage(source,
"User '%s' registered" % self.name,
message)
def sendUserCreatedMail(self, source, external=None):
import mailutils
if self.email_verified is False:
email_status = " (pending verification)"
else:
email_status = ""
message = """\
A new user has been created:
User name: %(username)r
Full name: %(fullname)r
Email: %(email)r%(email_status)s
""" % {
"username": self.name,
"fullname": self.fullname,
"email": self.email,
"email_status": email_status,
}
if external:
import auth
provider = auth.PROVIDERS[external["provider"]]
message += """\
External: %(provider)s %(account)r
""" % {
"provider": provider.getTitle(),
"account": external["account"],
}
message += """\
-- critic
"""
mailutils.sendAdministratorMessage(source, "User '%s' registered" % self.name, message)
def run(self):
soft_restart_attempted = set()
previous = {}
getloadavg_failed = False
load1_has_warned = 0
load1_last_time = 0
load5_has_warned = 0
load5_last_time = 0
load15_has_warned = 0
load15_last_time = 0
while not self.terminated:
self.interrupted = False
def sendLoadAverageWarning(interval, limit, load):
cpu_count = multiprocessing.cpu_count()
mailutils.sendAdministratorMessage(
"watchdog", "%d-minute load average" % interval,
("The current %d-minute load average is %.2f!\n"
% (interval, load)) +
("The configured limit is %.2f (%.2f x %d CPUs).\n"
% (limit, limit / cpu_count, cpu_count)) +
"\n-- critic\n")
try:
load1, load5, load15 = os.getloadavg()
self.debug("load average: %r, %r, %r" % (load1, load5, load15))
except OSError:
load1, load5, load15 = 0, 0, 0
if not getloadavg_failed:
self.exception("failed to detect system load average")
getloadavg_failed = True
now = time.time()
if self.load1_limit and load1 > self.load1_limit:
if load1 > load1_has_warned * 1.2 or now - load1_last_time > 60:
sendLoadAverageWarning(1, self.load1_limit, load1)
load1_has_warned = load1
load1_last_time = now
else:
load1_has_warned = 0
load1_last_time = 0
if self.load5_limit and load5 > self.load5_limit:
if load5 > load5_has_warned * 1.2 or now - load5_last_time > 5 * 60:
sendLoadAverageWarning(5, self.load5_limit, load5)
load5_has_warned = load5
load5_last_time = now
else:
load5_has_warned = 0
load5_last_time = 0
if self.load15_limit and load15 > self.load15_limit:
if load15 > load15_has_warned * 1.2 or now - load15_last_time > 15 * 60:
sendLoadAverageWarning(15, self.load15_limit, load15)
load15_has_warned = load15
load15_last_time = now
else:
load15_has_warned = 0
load15_last_time = 0
pidfile_dir = configuration.paths.WSGI_PIDFILE_DIR
if os.path.isdir(pidfile_dir):
pids = set(map(int, os.listdir(pidfile_dir)))
else:
pids = set()
for pid in pids:
try: rss = getRSS(pid)
except IOError as error:
if error.errno == errno.ENOENT:
self.warning("unlinking stale pid-file: %s" % os.path.join(pidfile_dir, str(pid)))
os.unlink(os.path.join(pidfile_dir, str(pid)))
continue
else: raise
if previous.get(pid) != rss:
self.debug("pid=%d, rss=%d bytes" % (pid, rss))
previous[pid] = rss
if rss > configuration.services.WATCHDOG["rss_hard_limit"]:
mailutils.sendAdministratorMessage(
"watchdog", "pid(%d): hard memory limit exceeded" % pid,
("Current RSS: %d kilobytes\nSending process SIGKILL (%d).\n\n-- critic"
% (rss, signal.SIGKILL)))
self.info("Killing pid(%d): hard memory limit exceeded, RSS: %d kilobytes" % (pid, rss))
os.kill(pid, signal.SIGKILL)
elif rss > configuration.services.WATCHDOG["rss_soft_limit"] and pid not in soft_restart_attempted:
mailutils.sendAdministratorMessage(
"watchdog", "pid(%d): soft memory limit exceeded" % pid,
("Current RSS: %d kilobytes\nSending process SIGINT (%d).\n\n"
% (rss, signal.SIGINT)))
self.info("Killing pid(%d): soft memory limit exceeded, RSS: %d kilobytes" % (pid, rss))
os.kill(pid, signal.SIGINT)
soft_restart_attempted.add(pid)
for pid in previous.keys():
if pid not in pids: del previous[pid]
soft_restart_attempted = soft_restart_attempted & pids
time.sleep(10)
def emit(self, record):
import mailutils
mailutils.sendAdministratorMessage(self.__logfile_name, record.message.splitlines()[0], self.formatter.format(record))
def run(self):
soft_restart_attempted = set()
previous = {}
getloadavg_failed = False
load1_has_warned = 0
load1_last_time = 0
load5_has_warned = 0
load5_last_time = 0
load15_has_warned = 0
load15_last_time = 0
while not self.terminated:
self.interrupted = False
def sendLoadAverageWarning(interval, limit, load):
cpu_count = multiprocessing.cpu_count()
mailutils.sendAdministratorMessage(
"watchdog", "%d-minute load average" % interval,
("The current %d-minute load average is %.2f!\n" %
(interval, load)) +
("The configured limit is %.2f (%.2f x %d CPUs).\n" %
(limit, limit / cpu_count, cpu_count)) + "\n-- critic\n")
try:
load1, load5, load15 = os.getloadavg()
self.debug("load average: %r, %r, %r" % (load1, load5, load15))
except OSError:
load1, load5, load15 = 0, 0, 0
if not getloadavg_failed:
self.exception("failed to detect system load average")
getloadavg_failed = True
now = time.time()
if self.load1_limit and load1 > self.load1_limit:
if load1 > load1_has_warned * 1.2 or now - load1_last_time > 60:
sendLoadAverageWarning(1, self.load1_limit, load1)
load1_has_warned = load1
load1_last_time = now
else:
load1_has_warned = 0
load1_last_time = 0
if self.load5_limit and load5 > self.load5_limit:
if load5 > load5_has_warned * 1.2 or now - load5_last_time > 5 * 60:
sendLoadAverageWarning(5, self.load5_limit, load5)
load5_has_warned = load5
load5_last_time = now
else:
load5_has_warned = 0
load5_last_time = 0
if self.load15_limit and load15 > self.load15_limit:
if load15 > load15_has_warned * 1.2 or now - load15_last_time > 15 * 60:
sendLoadAverageWarning(15, self.load15_limit,
load15)
load15_has_warned = load15
load15_last_time = now
else:
load15_has_warned = 0
load15_last_time = 0
pidfile_dir = configuration.paths.WSGI_PIDFILE_DIR
if os.path.isdir(pidfile_dir):
pids = set(map(int, os.listdir(pidfile_dir)))
else:
pids = set()
for pid in pids:
try:
rss = getRSS(pid)
except IOError as error:
if error.errno == errno.ENOENT:
self.warning("unlinking stale pid-file: %s" %
os.path.join(pidfile_dir, str(pid)))
os.unlink(os.path.join(pidfile_dir, str(pid)))
continue
else:
raise
if previous.get(pid) != rss:
self.debug("pid=%d, rss=%d bytes" % (pid, rss))
previous[pid] = rss
if rss > configuration.services.WATCHDOG["rss_hard_limit"]:
mailutils.sendAdministratorMessage(
"watchdog",
"pid(%d): hard memory limit exceeded" % pid,
("Current RSS: %d kilobytes\nSending process SIGKILL (%d).\n\n-- critic"
% (rss, signal.SIGKILL)))
self.info(
"Killing pid(%d): hard memory limit exceeded, RSS: %d kilobytes"
% (pid, rss))
os.kill(pid, signal.SIGKILL)
elif rss > configuration.services.WATCHDOG[
"rss_soft_limit"] and pid not in soft_restart_attempted:
mailutils.sendAdministratorMessage(
"watchdog",
"pid(%d): soft memory limit exceeded" % pid,
("Current RSS: %d kilobytes\nSending process SIGINT (%d).\n\n"
% (rss, signal.SIGINT)))
self.info(
"Killing pid(%d): soft memory limit exceeded, RSS: %d kilobytes"
% (pid, rss))
os.kill(pid, signal.SIGINT)
soft_restart_attempted.add(pid)
for pid in previous.keys():
if pid not in pids: del previous[pid]
soft_restart_attempted = soft_restart_attempted & pids
self.signal_idle_state()
time.sleep(10)
def emit(self, record):
import mailutils
mailutils.sendAdministratorMessage(self.__logfile_name,
record.message.splitlines()[0],
self.formatter.format(record))
