def main(args):
logging.basicConfig(stream=sys.stderr, level=logging.DEBUG, format='%(asctime)s %(levelname)s %(filename)s: %(message)s')
logger = logging.getLogger("supervisord-watchdog")
debug_mode = True if 'DEBUG' in os.environ else False
while True:
logger.info("Listening for events...")
headers, body = listener.wait(sys.stdin, sys.stdout)
body = dict([pair.split(":") for pair in body.split(" ")])
logger.debug("Headers: %r", repr(headers))
logger.debug("Body: %r", repr(body))
logger.debug("Args: %r", repr(args))
if debug_mode: continue
try:
if headers["eventname"] == "PROCESS_STATE_FATAL":
logger.info("Process entered FATAL state...")
if not args or body["processname"] in args:
logger.error("Sending TERM signal to supervisord instance ...")
subprocess.call(["/bin/kill", "-15", "1"], stdout=sys.stderr)
time.sleep(5)
logger.critical("Still alive! Sending KILL to all processes...")
subprocess.call(["/bin/kill", "-9", "-1"], stdout=sys.stderr)
except Exception as e:
logger.critical("Unexpected Exception: %s", str(e))
listener.fail(sys.stdout)
exit(1)
else:
listener.ok(sys.stdout)
def run():
db_conn = create_connect_db()
db_cursor = db_conn.cursor()
while True:
headers, body = listener.wait(sys.stdin, sys.stdout)
body = dict([pair.split(":") for pair in body.split(" ")])
# write_stderr('{} | {}'.format(headers, body))
if headers["eventname"] in ("PROCESS_STATE_RUNNING",
"PROCESS_STATE_STOPPED"):
process = body["processname"]
if process != "listener":
new_state = headers["eventname"] == "PROCESS_STATE_RUNNING"
while True:
try:
write_stderr("{} -> {}".format(process, new_state))
db_cursor.execute(query, (process, new_state))
db_conn.commit()
break
except (psycopg2.InterfaceError,
psycopg2.OperationalError):
db_conn = create_connect_db()
db_cursor = db_conn.cursor()
except BaseException:
db_conn.rollback()
break
# acknowledge the event
write_stdout("RESULT 2\nOK")
def main():
parser = argparse.ArgumentParser(
description="Supervisor event listener to notify on process events.")
parser.add_argument("--apikey",
help="Specify the pushbullet api key",
required=True)
args = parser.parse_args()
pb_api_key = Pushbullet(args.apikey)
while True:
headers, body = listener.wait(sys.stdin, sys.stdout)
body = dict([pair.split(":") for pair in body.split(" ")])
write_stderr("Headers: %r\n" % repr(headers))
write_stderr("Body: %r\n" % repr(body))
#Check if the process is in "STOPPED STATE" and send a pushbullet notification
if headers["eventname"] == "PROCESS_STATE_EXITED":
write_stderr("Process state stopping...\n")
process_name = body.get('processname')
push = pb_api_key.push_note(
"Supervisord Alert",
"Process in stopped state: " + process_name)
push
#Supervisor acknowledge
write_stdout("RESULT 2\nOK")
def main(args):
logging.basicConfig(
stream=sys.stdout,
level=logging.INFO,
format='%(asctime)s %(levelname)s %(filename)s: %(message)s')
logger = logging.getLogger('socket-server')
while True:
logger.info('Listening for events...')
headers, body = listener.wait(sys.stdin, sys.stdout)
body = dict([pair.split(':') for pair in body.split(' ')])
logger.debug('Headers: %r', repr(headers))
logger.debug('Body: %r', repr(body))
try:
if headers['eventname'] == 'PROCESS_STATE_FATAL':
logger.info('Process entered FATAL state...')
if not args or body['processname'] in args:
logger.error('Killing off supervisord instance ...')
call(['/bin/kill', '-15', '1'], stdout=sys.stderr)
logger.info('Sent TERM signal to init process')
time.sleep(5)
logger.critical(
'Why am I still alive? Send KILL to all processes...')
call(['/bin/kill', '-9', '-1'], stdout=sys.stderr)
except Exception as e:
logger.critical('Unexpected Exception: %s', str(e))
listener.fail(sys.stdout)
exit(1)
else:
listener.ok(sys.stdout)
def main():
while True:
headers, body = listener.wait()
body = dict([pair.split(":") for pair in body.split(" ")])
if (headers["eventname"] == "PROCESS_STATE_RUNNING"
and body["processname"] == "dbus"):
write_stderr("dbus is running, starting avahi\n")
call(["supervisorctl", "start", "avahi"])
listener.ok()
def main():
while True:
headers, body = listener.wait(sys.stdin, sys.stdout)
body = dict([pair.split(":") for pair in body.split(" ")])
write_stdout("Headers: %r\n" % repr(headers))
write_stdout("Body: %r\n" % repr(body))
listener.ok(sys.stdout)
write_stdout("eventname: %s\n" % headers["eventname"])
if headers["eventname"] == "PROCESS_STATE_STOPPED":
write_stdout("Process state stopped...\n")
def main():
while True:
headers, body = listener.wait(sys.stdin, sys.stdout)
body = dict([pair.split(":") for pair in body.split(" ")])
write_stderr("Headers: %r\n" % repr(headers))
write_stderr("Body: %r\n" % repr(body))
try:
if body["processname"] == program_name:
if headers["eventname"] != "PROCESS_STATE_RUNNING":
i = 0
check_program_pid = False
time.sleep(5) # first timeout 5s
while i < timeout:
time.sleep(1)
if os.path.exists(program_pid) and os.path.isfile(
program_pid):
pidfile = open(program_pid, 'rt')
pid = int(pidfile.readline())
if check_pid(pid):
check_program_pid = True
break
i += 1
if not check_program_pid:
try:
pidfile = open(supervisord_pid, 'rt')
pid = int(pidfile.readline())
kill_pid(pid)
except Exception as e:
write_stderr("Unexpected Exception: " + str(e))
sys.exit(0)
except Exception as e:
write_stderr("Unexpected Exception: " + str(e))
listener.fail(sys.stdout)
sys.exit(1)
else:
listener.ok(sys.stdout)
def main():
while True:
try:
msg_hdr, msg_payload = listener.wait(sys.stdin, sys.stdout)
if "eventname" in msg_hdr:
process_event(msg_hdr, msg_payload)
listener.ok(sys.stdout)
except Exception as e:
write_stderr(str(e))
def main(args):
logging.basicConfig(stream=sys.stderr, level=logging.DEBUG, format='%(asctime)s %(levelname)s %(filename)s: %(message)s')
logger = logging.getLogger("supervisord-eventlistener")
debug_mode = True if 'DEBUG' in os.environ else False
if 'PROCESSNAME' in os.environ:
processName = os.environ["PROCESSNAME"]
else:
logger.critical("Set PROCESSNAME in environment!");
exit(1)
if 'EVENT' in os.environ:
eventName = os.environ["EVENT"]
else:
logger.critical("Set EVENT in environment!")
exit(1)
if 'EXECUTE' in os.environ:
executeCommand = os.environ["EXECUTE"].split(" ")
else:
logger.critical("Set EXECUTE in environment!")
exit(1)
if 'DELAY' in os.environ:
sleepTime = int(os.environ["DELAY"])
else:
logger.critical("Set DELAY in environment!")
exit(1)
while True:
headers, body = listener.wait(sys.stdin, sys.stdout)
body = dict([pair.split(":") for pair in body.split(" ")])
if debug_mode:
logger.debug("ENV: %r", repr(os.environ))
logger.debug("Headers: %r", repr(headers))
logger.debug("Body: %r", repr(body))
logger.debug("Args: %r", repr(args))
try:
if headers["eventname"] == eventName and body["processname"] == processName:
if debug_mode:
logger.debug("Process %s entered RUNNING state...", processName)
time.sleep(sleepTime);
if debug_mode:
logger.debug("Execute %s after %s (sec)...", os.environ["EXECUTE"], os.environ["DELAY"])
res = subprocess.call(executeCommand, stdout=sys.stderr)
except Exception as e:
logger.critical("Unexpected Exception: %s", str(e))
listener.fail(sys.stdout)
exit(1)
else:
listener.ok(sys.stdout)
def test_wait(self):
from supervisor.childutils import listener
class Dummy:
def readline(self):
return 'len:5'
def read(self, *ignored):
return 'hello'
stdin = Dummy()
stdout = StringIO()
headers, payload = listener.wait(stdin, stdout)
self.assertEqual(headers, {'len':'5'})
self.assertEqual(payload, 'hello')
self.assertEqual(stdout.getvalue(), 'READY\n')
def test_wait(self):
from supervisor.childutils import listener
class Dummy:
def readline(self):
return 'len:5'
def read(self, *ignored):
return 'hello'
stdin = Dummy()
stdout = StringIO()
headers, payload = listener.wait(stdin, stdout)
self.assertEqual(headers, {'len':'5'})
self.assertEqual(payload, 'hello')
self.assertEqual(stdout.getvalue(), 'READY\n')
def main():
# Telegraf needs to start after the influxdb instance starts listening
# otherwise telegraf is not able to connect to the database
while True:
headers, body = listener.wait(sys.stdin, sys.stdout)
# 'Listening for signals' is logged once the db server starts listening
# @see cmd/influxd/main.go
if re.search(r'Listening for signals', body):
call(["supervisorctl", "start", "telegraf"])
call(["supervisorctl", "start", "kapacitor"])
sys.exit(0)
else:
listener.ok(sys.stdout)
def main():
parser = argparse.ArgumentParser(
description="Supervisor event listener to notify on process events.",
epilog="Homepage: https://github.com/rahiel/supervisor-alert")
parser.add_argument(
"-c",
"--command",
help="Specify the command to process the event messages.")
parser.add_argument("--telegram",
help="Use telegram-send to send event messages.",
action="store_true")
parser.add_argument("--configure",
help="configure %(prog)s",
action="store_true")
parser.add_argument("--show-hostname",
help="show hostname in messages",
action="store_true")
parser.add_argument("--version",
action="version",
version="%(prog)s {}".format(__version__))
args = parser.parse_args()
if args.configure:
return configure()
s = "PROCESS_STATE_"
hostname = gethostname()
if args.telegram:
alert = telegram
elif args.command:
alert = partial(send, command=shlex.split(args.command))
else:
raise Exception("No command specified.")
while True:
headers, payload = listener.wait()
event_name = headers["eventname"]
if event_name.startswith(s):
event_name = event_name[len(s):].lower()
data = get_headers(payload) # keys: from_state, pid, processname
process_name = data["processname"]
message = process_name + " has entered state " + event_name
if args.show_hostname:
message = hostname + ": " + message
alert(message=message)
else:
listener.ok()
def main():
while True:
headers, body = listener.wait(sys.stdin, sys.stdout)
body = dict([pair.split(":") for pair in body.split(" ")])
if body["groupname"] == "concierge":
try:
pidfile = open('/run/supervisor.pid', 'r')
pid = int(pidfile.readline())
os.kill(pid, signal.SIGQUIT)
except Exception as e:
write_stdout('could not kill supervisor: %s\n' % e.strerror)
write_stdout('RESULT 2\nOK')
def main():
# run main loop
while True:
try:
msg_hdr, msg_payload = listener.wait(sys.stdin, sys.stdout)
if "eventname" in msg_hdr:
process_event(msg_hdr, msg_payload)
except KeyboardInterrupt:
break
except Exception as e:
traceback.print_exc(file=sys.stderr)
err(str(e))
listener.ok(sys.stdout)
def test_wait(self):
from supervisor.childutils import listener
from supervisor.dispatchers import PEventListenerDispatcher
token = PEventListenerDispatcher.READY_FOR_EVENTS_TOKEN
class Dummy:
def readline(self):
return 'len:5'
def read(self, *ignored):
return 'hello'
stdin = Dummy()
stdout = StringIO()
headers, payload = listener.wait(stdin, stdout)
self.assertEqual(headers, {'len':'5'})
self.assertEqual(payload, 'hello')
self.assertEqual(stdout.getvalue(), 'READY\n')
def main():
while True:
headers, body = listener.wait(sys.stdin, sys.stdout)
body = dict([pair.split(":") for pair in body.split(" ")])
#write_stderr("Headers: %r\n" % repr(headers))
#write_stderr("Body: %r\n" % repr(body))
if headers["eventname"] == "PROCESS_STATE_STOPPED" or \
headers["eventname"] == "PROCESS_STATE_EXITED" or \
headers["eventname"] == "PROCESS_STATE_FATAL":
os.kill(1, 15)
write_stderr("Terminating...\n")
# acknowledge the event
write_stdout("RESULT 2\nOK")
def test_wait(self):
from supervisor.childutils import listener
from supervisor.dispatchers import PEventListenerDispatcher
token = PEventListenerDispatcher.READY_FOR_EVENTS_TOKEN
class Dummy:
def readline(self):
return 'len:5'
def read(self, *ignored):
return 'hello'
stdin = Dummy()
stdout = StringIO()
headers, payload = listener.wait(stdin, stdout)
self.assertEqual(headers, {'len': '5'})
self.assertEqual(payload, 'hello')
self.assertEqual(stdout.getvalue(), 'READY\n')
def main(args):
# logging.basicConfig(stream=sys.stderr, level=logging.DEBUG, format='%(asctime)s %(levelname)s %(filename)s: %(message)s')
# logger = logging.getLogger("supervisord-eventlistener")
storage = Storage()
while True:
headers, body = listener.wait(sys.stdin, sys.stdout)
body = dict([pair.split(":") for pair in body.split(" ")])
try:
storage.query("INSERT INTO messages (process_name, content) VALUES (?, ?)", [
body["processname"], headers["eventname"]])
except Exception as e:
# logger.critical("Unexpected Exception: %s", str(e))
listener.fail(sys.stdout)
exit(1)
else:
listener.ok(sys.stdout)
def main():
while True:
headers, body = listener.wait(sys.stdin, sys.stdout)
body = dict([pair.split(":") for pair in body.split(" ")])
# write_stderr("Headers: %r\n" % repr(headers))
# write_stderr("Body: %r\n" % repr(body))
listener.ok(sys.stdout)
write_stderr("==========>\t%s\t===>\t%s\t ++++++\n" % (body["processname"] , headers["eventname"]) );
if body["processname"] != "rtorrent":
continue
write_stderr("==========> most probalom kigyilkolni..." );
pidfile = open('/run/supervisord.pid','r')
pid = int( pidfile.readline() );
os.kill( pid,signal.SIGQUIT )
os.kill( 1 ,signal.SIGQUIT )
def run():
db_conn = create_connect_db()
db_cursor = db_conn.cursor()
while True:
headers, body = listener.wait(sys.stdin, sys.stdout)
body = dict([pair.split(":") for pair in body.split(" ")])
# write_stderr('{} | {}'.format(headers, body))
if headers['eventname'] in ('PROCESS_STATE_RUNNING',
'PROCESS_STATE_STOPPED'):
process = body['processname']
if process != 'listener':
new_state = headers['eventname'] == 'PROCESS_STATE_RUNNING'
try:
write_stderr('{} -> {}'.format(process, new_state))
db_cursor.execute(query, (process, new_state))
db_conn.commit()
except Exception:
db_conn.rollback()
# acknowledge the event
write_stdout("RESULT 2\nOK")
def main():
parser = argparse.ArgumentParser(description="Supervisor event listener to notify on process events.",
epilog="Homepage: https://github.com/rahiel/supervisor-alert")
parser.add_argument("-c", "--command", help="Specify the command to process the event messages.")
parser.add_argument("--telegram", help="Use telegram-send to send event messages.", action="store_true")
parser.add_argument("--configure", help="configure %(prog)s", action="store_true")
parser.add_argument("--show-hostname", help="show hostname in messages", action="store_true")
parser.add_argument("--version", action="version", version="%(prog)s {}".format(__version__))
args = parser.parse_args()
if args.configure:
return configure()
s = "PROCESS_STATE_"
hostname = gethostname()
if args.telegram:
alert = telegram
elif args.command:
alert = partial(send, command=shlex.split(args.command))
else:
raise Exception("No command specified.")
while True:
headers, payload = listener.wait()
event_name = headers["eventname"]
if event_name.startswith(s):
event_name = event_name[len(s):].lower()
data = get_headers(payload) # keys: from_state, pid, processname
process_name = data["processname"]
message = process_name + " has entered state " + event_name
if args.show_hostname:
message = hostname + ": " + message
alert(message=message)
else:
listener.ok()
def main():
rpci = getRPCInterface(os.environ)
while True:
h, p = listener.wait()
if not h['eventname'] == 'PROCESS_STATE_EXITED':
listener.ok()
continue
ph, _pd = eventdata(p + '\n')
if ph['processname'] == 'mongodb':
if int(ph['expected']):
listener.ok()
continue
listener.send("MONGODB HAS BEEN RESTARTED. SO WILL THE APP!")
rpci.supervisor.stopProcessGroup('app')
rpci.supervisor.startProcessGroup('app', True)
listener.ok()
now = datetime.now()
send_twitter("[FATAL] %s - mongodb has been restarted" % now.isoformat())
return
listener.ok()
#!/usr/bin/env python
import smtplib
import yaml
import sys
from supervisor.childutils import listener
import os
localdir = os.path.dirname(__file__)
config_file = os.path.join(localdir, 'conf', 'listener.yaml')
config = yaml.load(file(config_file, 'r'))
while 1:
(header, payload) = listener.wait()
if "processname:gogogon-consumer" in payload:
server = smtplib.SMTP(config['server']['host'])
server.starttls()
server.login(config['server']['username'],
config['server']['password'])
server.sendmail(config['message']['from_addr'],
config['message']['to_addrs'],
config['message']['body'])
server.quit()
listener.ok()
def run(self):
self._debug("Starting")
"""
The main run loop - evaluates all incoming events and never exits
"""
while True:
headers, payload = listener.wait(self.stdin, self.stdout)
self._debug("Headers: %r\n" % repr(headers))
self._debug("Body: %r\n" % repr(payload))
event_name = headers.get('eventname', None)
self._debug("Received event %s" % event_name)
### ignore TICK events
if re.match('TICK', event_name):
self._debug("Ignoring TICK event '%s'" % event_name)
### some sort of process related event - worth capturing
elif re.match('PROCESS', event_name):
### true for all process events:
event_data = self._parse_payload(payload, event_name)
process_name = event_data.get('processname', None)
group_name = event_data.get('groupname', None)
### if you didn't specify a 'processname' explicitly, it'll
### be the same as groupname. otherwise, they differ and
### both should be in the key. So, check for that and decide.
if process_name != group_name:
process_name += "_" + group_name
### in the case you used a . in your name, we'll convert that to a _
### here because influx renders those as seperators.
### Also remove any spaces as those are not supported in keys either.
### not sure if there are any other things that should be filtered out
### the docs are not conclusive and couldn't find the code section
### either. I'm pretty sure the key is used as the FS name for the
### whisper DB, so probably filesystem reserved characters are bad,
### however, that'd hold true for supervisor as well and I don't think
### we'd get here with such keys.
process_name = process_name.replace('.', '_')
process_name = process_name.replace(' ', '_')
### stdout/stderr capturing
if re.match('PROCESS_LOG', event_name):
event = "process_log, group=%s, name=%s, event=%s, value=1" % \
( self.influx_group, process_name, event_name.lower() )
self._send_to_influx(event)
### state change
elif re.match('PROCESS_STATE', event_name):
event = "process_state, group=%s, name=%s, from_state=%s, to_state=%s, value=1" % \
( self.influx_group, process_name, event_data.get('from_state', 'unknown').lower(), event_name.lower() )
self._send_to_influx(event)
### ignore IPC for now
elif re.match('PROCESS_COMMUNICATION', event_name):
self._debug("Ignoring PROCESS event: '%s'" % event_name)
### unknown process event..?
else:
self._debug("Unknown PROCESS event: '%s'" % event_name)
### completely unknown event
else:
self._debug("Unknown event: '%s'" % event_name)
listener.ok(self.stdout)
#!/usr/bin/env python
import smtplib
import yaml
import sys
from supervisor.childutils import listener
import os
localdir = os.path.dirname(__file__)
config_file = os.path.join(localdir, 'conf', 'listener.yaml')
config = yaml.load(file(config_file, 'r'))
while 1:
(header, payload) = listener.wait()
if "processname:gogogon-consumer" in payload:
server = smtplib.SMTP(config['server']['host'])
server.starttls()
server.login(config['server']['username'], config['server']['password'])
server.sendmail(config['message']['from_addr'], config['message']['to_addrs'], config['message']['body'])
server.quit()
listener.ok()