def main():
r"""This is where it happens \o/"""
parser = OptionParser()
parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False, help=SUPPRESS_HELP)
parser.add_option("-q", "--quiet", action="store_true", dest="quiet", default=False, help=SUPPRESS_HELP)
parser.add_option("-t", "--test", action="store_true", dest="test", default=False, help="Test config and exit")
parser.add_option("-p", "--pidfile", dest="pidfile", default=None, help="Write PID into this file")
parser.add_option("-N", "--no-network", dest="no_network", default=False, action="store_true", help="Disable network listening socket")
parser.add_option("-d", "--debug", dest="debug", default=False, action="store_true", help=SUPPRESS_HELP)
parser.add_option("-f", "--config", dest="config", default="monitor.ini", help="configuration file")
parser.add_option("-H", "--no-heartbeat", action="store_true", dest="no_heartbeat", default=False, help="Omit printing the '.' character when running checks")
parser.add_option('-1', '--one-shot', action='store_true', dest='one_shot', default=False, help='Run the monitors once only, without alerting. Require monitors without "fail" in the name to succeed. Exit zero or non-zero accordingly.')
parser.add_option('--loops', dest='loops', default=-1, help=SUPPRESS_HELP, type=int)
parser.add_option('-l', '--log-level', dest="loglevel", default="warn", help="Log level: critical, error, warn, info, debug")
parser.add_option('-C', '--no-colour', '--no-color', action='store_true', dest='no_colour', default=False, help='Do not colourise log output')
parser.add_option('--no-timestamps', action='store_true', dest='no_timestamps', default=False, help='Do not prefix log output with timestamps')
(options, _) = parser.parse_args()
if options.quiet:
print('Warning: --quiet is deprecated; use --log-level=critical')
options.loglevel = 'critical'
if options.verbose:
print('Warning: --verbose is deprecated; use --log-level=info')
options.loglevel = 'info'
if options.debug:
print('Warning: --debug is deprecated; use --log-level=debug')
options.loglevel = 'debug'
if options.no_timestamps:
logging_timestamp = ''
else:
logging_timestamp = '%(asctime)s '
try:
log_level = getattr(logging, options.loglevel.upper())
except AttributeError:
print('Log level {0} is unknown'.format(options.loglevel))
sys.exit(1)
log_datefmt = '%Y-%m-%d %H:%M:%S'
log_plain_format = logging_timestamp + '%(levelname)8s (%(name)s) %(message)s'
if not options.no_colour:
try:
handler = colorlog.StreamHandler()
handler.setFormatter(colorlog.ColoredFormatter(logging_timestamp + '%(log_color)s%(levelname)8s%(reset)s (%(name)s) %(message)s', datefmt='%Y-%m-%d %H:%M:%S'))
main_logger.addHandler(handler)
except NameError:
logging.basicConfig(format=log_plain_format, datefmt=log_datefmt)
main_logger.error('Could not enable colorlog')
else:
logging.basicConfig(format=log_plain_format, datefmt=log_datefmt)
main_logger.setLevel(log_level)
if not options.quiet:
main_logger.info('=== SimpleMonitor v%s', VERSION)
main_logger.info('Loading main config from %s', options.config)
config = EnvironmentAwareConfigParser()
if not os.path.exists(options.config):
main_logger.critical('Configuration file "%s" does not exist!', options.config)
sys.exit(1)
try:
config.read(options.config)
except Exception as e:
main_logger.critical('Unable to read configuration file')
main_logger.critical(e)
sys.exit(1)
try:
interval = config.getint("monitor", "interval")
except Exception:
main_logger.critical('Missing [monitor] section from config file, or missing the "interval" setting in it')
sys.exit(1)
pidfile = None
try:
pidfile = config.get("monitor", "pidfile")
except Exception:
pass
if options.pidfile:
pidfile = options.pidfile
if pidfile:
my_pid = os.getpid()
try:
with open(pidfile, "w") as file_handle:
file_handle.write("%d\n" % my_pid)
except Exception:
main_logger.error("Couldn't write to pidfile!")
pidfile = None
if config.has_option("monitor", "monitors"):
monitors_file = config.get("monitor", "monitors")
else:
monitors_file = "monitors.ini"
main_logger.info("Loading monitor config from %s", monitors_file)
try:
allow_pickle = config.getboolean("monitor", "allow_pickle",
fallback='true')
except ValueError:
main_logger.critical('allow_pickle should be "true" or "false".')
sys.exit(1)
m = SimpleMonitor(allow_pickle=allow_pickle)
m = load_monitors(m, monitors_file)
count = m.count_monitors()
if count == 0:
main_logger.critical("No monitors loaded :(")
sys.exit(2)
m = load_loggers(m, config)
m = load_alerters(m, config)
try:
if config.get("monitor", "remote") == "1":
if not options.no_network:
enable_remote = True
remote_port = int(config.get("monitor", "remote_port"))
else:
enable_remote = False
else:
enable_remote = False
except Exception:
enable_remote = False
if not m.verify_dependencies():
sys.exit(1)
if options.test:
main_logger.warning("Config test complete. Exiting.")
sys.exit(0)
if options.one_shot:
main_logger.warning("One-shot mode: expecting monitors without 'fail' in the name to succeed, and with to fail. Will exit zero or non-zero accordingly.")
try:
key = config.get("monitor", "key")
except Exception:
key = None
if enable_remote:
if not options.quiet:
if not allow_pickle:
allowing_pickle = "not "
else:
allowing_pickle = ""
main_logger.info("Starting remote listener thread ({0}allowing pickle data)".format(allowing_pickle))
remote_listening_thread = Loggers.network.Listener(
m, remote_port, key, allow_pickle=allow_pickle)
remote_listening_thread.daemon = True
remote_listening_thread.start()
if not options.quiet:
main_logger.info("=== Starting... (loop runs every %ds) Hit ^C to stop", interval)
loop = True
heartbeat = 0
loops = int(options.loops)
while loop:
try:
if loops > 0:
loops -= 1
if loops == 0:
main_logger.warning('Ran out of loop counter, will stop after this one')
loop = False
m.run_loop()
if options.loglevel in ['error', 'critical', 'warn'] and not options.no_heartbeat:
heartbeat += 1
if heartbeat == 2:
sys.stdout.write(".")
sys.stdout.flush()
heartbeat = 0
except KeyboardInterrupt:
main_logger.info('Received ^C')
loop = False
except Exception:
sys.exc_info()
main_logger.exception("Caught unhandled exception during main loop")
if loop and enable_remote:
if not remote_listening_thread.isAlive():
main_logger.error("Listener thread died :(")
remote_listening_thread = Loggers.network.Listener(
m, remote_port, key, allow_pickle=allow_pickle)
remote_listening_thread.start()
if options.one_shot:
break
try:
if loop:
time.sleep(interval)
except Exception:
main_logger.info("Quitting")
loop = False
if enable_remote:
remote_listening_thread.running = False
main_logger.info('Waiting for listener thread to exit')
remote_listening_thread.join(0)
if pidfile:
try:
os.unlink(pidfile)
except Exception:
main_logger.error("Couldn't remove pidfile!")
if not options.quiet:
main_logger.info("Finished.")
if options.one_shot: # pragma: no cover
ok = True
print('\n--> One-shot results:')
for monitor in sorted(m.monitors.keys()):
if "fail" in monitor:
if m.monitors[monitor].error_count == 0:
print(" Monitor {0} should have failed".format(monitor))
ok = False
else:
print(" Monitor {0} was ok (failed)".format(monitor))
elif "skip" in monitor:
if m.monitors[monitor].skipped():
print(" Monitor {0} was ok (skipped)".format(monitor))
else:
print(" Monitor {0} should have been skipped".format(monitor))
ok = False
else:
if m.monitors[monitor].error_count > 0:
print(" Monitor {0} failed and shouldn't have".format(monitor))
ok = False
else:
print(" Monitor {0} was ok".format(monitor))
if not ok:
print("Not all non-'fail' succeeded, or not all 'fail' monitors failed.")
sys.exit(1)
logging.shutdown()
def main():
"""This is where it happens \o/"""
parser = OptionParser()
parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False, help="Be more verbose")
parser.add_option("-q", "--quiet", action="store_true", dest="quiet", default=False, help="Don't output anything except errors")
parser.add_option("-t", "--test", action="store_true", dest="test", default=False, help="Test config and exit")
parser.add_option("-p", "--pidfile", dest="pidfile", default="", help="Write PID into this file")
parser.add_option("-N", "--no-network", dest="no_network", default=False, action="store_true", help="Disable network listening socket")
parser.add_option("-d", "--debug", dest="debug", default=False, action="store_true", help="Enable debug output")
parser.add_option("-f", "--config", dest="config", default="monitor.ini", help="configuration file")
(options, args) = parser.parse_args()
if options.quiet and options.verbose:
options.verbose = False
if options.quiet and options.debug:
options.debug = False
if options.debug and not options.verbose:
options.verbose = True
if not options.quiet:
print "SimpleMonitor v%s" % VERSION
print "--> Loading main config from %s" % options.config
config = ConfigParser()
config.read(options.config)
interval = config.getint("monitor", "interval")
pidfile = ""
try:
pidfile = config.get("monitor", "pidfile")
except:
pass
if options.pidfile != "":
pidfile = options.pidfile
if pidfile != "":
my_pid = os.getpid()
try:
with open(pidfile, "w") as file_handle:
file_handle.write("%d\n" % my_pid)
except:
sys.stderr.write("Couldn't write to pidfile!")
if config.has_option("monitor", "monitors"):
monitors_file = config.get("monitor", "monitors")
else:
monitors_file = "monitors.ini"
if not options.quiet:
print "--> Loading monitor config from %s" % monitors_file
m = SimpleMonitor()
m = load_monitors(m, monitors_file, options.quiet)
count = m.count_monitors()
if count == 0:
sys.stderr.write("No monitors loaded :(\n")
sys.exit(2)
if not options.quiet:
print "--> Loaded %d monitors.\n" % count
m = load_loggers(m, config, options.quiet)
m = load_alerters(m, config, options.quiet)
try:
if config.get("monitor", "remote") == "1":
if not options.no_network:
enable_remote = True
remote_port = int(config.get("monitor", "remote_port"))
else:
enable_remote = False
else:
enable_remote = False
except:
enable_remote = False
if not m.verify_dependencies():
sys.exit(1)
if options.test:
print "--> Config test complete. Exiting."
sys.exit(0)
if not options.quiet:
print
try:
key = config.get("monitor", "key")
except:
key = None
if enable_remote:
if not options.quiet:
print "--> Starting remote listener thread"
remote_listening_thread = Loggers.network.Listener(m, remote_port, options.verbose, key)
remote_listening_thread.daemon = True
remote_listening_thread.start()
if not options.quiet:
print "--> Starting... (loop runs every %ds) Hit ^C to stop" % interval
loop = True
heartbeat = 0
m.set_verbosity(options.verbose, options.debug)
while loop:
try:
m.run_tests()
m.do_recovery()
m.do_alerts()
m.do_logs()
if not options.quiet and not options.verbose:
heartbeat += 1
if heartbeat == 2:
sys.stdout.write(".")
sys.stdout.flush()
heartbeat = 0
except KeyboardInterrupt:
if not options.quiet:
print "\n--> EJECT EJECT"
loop = False
except Exception, e:
sys.exc_info()
sys.stderr.write("Caught unhandled exception during main loop: %s\n" % e)
if loop and enable_remote:
if not remote_listening_thread.isAlive():
print "Listener thread died :("
remote_listening_thread = Loggers.network.Listener(m, remote_port, options.verbose)
remote_listening_thread.start()
try:
time.sleep(interval)
except:
print "\n--> Quitting."
loop = False
def main():
"""This is where it happens \o/"""
parser = OptionParser()
parser.add_option("-v",
"--verbose",
action="store_true",
dest="verbose",
default=False,
help="Be more verbose")
parser.add_option("-q",
"--quiet",
action="store_true",
dest="quiet",
default=False,
help="Don't output anything except errors")
parser.add_option("-t",
"--test",
action="store_true",
dest="test",
default=False,
help="Test config and exit")
parser.add_option("-p",
"--pidfile",
dest="pidfile",
default="",
help="Write PID into this file")
parser.add_option("-N",
"--no-network",
dest="no_network",
default=False,
action="store_true",
help="Disable network listening socket")
parser.add_option("-d",
"--debug",
dest="debug",
default=False,
action="store_true",
help="Enable debug output")
parser.add_option("-f",
"--config",
dest="config",
default="monitor.ini",
help="configuration file")
parser.add_option(
"-H",
"--no-heartbeat",
action="store_true",
dest="no_heartbeat",
default=False,
help="Omit printing the '.' character when running checks")
(options, args) = parser.parse_args()
if options.quiet and options.verbose:
options.verbose = False
if options.quiet and options.debug:
options.debug = False
if options.debug and not options.verbose:
options.verbose = True
if not options.quiet:
print "SimpleMonitor v%s" % VERSION
print "--> Loading main config from %s" % options.config
config = EnvironmentAwareConfigParser()
if not os.path.exists(options.config):
print '--> Configuration file "%s" does not exist!' % options.config
sys.exit(1)
try:
config.read(options.config)
except Exception as e:
print '--> Unable to read configuration file "%s"' % options.config
print 'The config parser reported:'
print e
sys.exit(1)
try:
interval = config.getint("monitor", "interval")
except NoOptionError:
print '--> Missing [monitor] section from config file, or missing the "interval" setting in it'
sys.exit(1)
pidfile = ""
try:
pidfile = config.get("monitor", "pidfile")
except:
pass
if options.pidfile != "":
pidfile = options.pidfile
if pidfile != "":
my_pid = os.getpid()
try:
with open(pidfile, "w") as file_handle:
file_handle.write("%d\n" % my_pid)
except:
sys.stderr.write("Couldn't write to pidfile!")
if config.has_option("monitor", "monitors"):
monitors_file = config.get("monitor", "monitors")
else:
monitors_file = "monitors.ini"
if not options.quiet:
print "--> Loading monitor config from %s" % monitors_file
m = SimpleMonitor()
m = load_monitors(m, monitors_file, options.quiet)
count = m.count_monitors()
if count == 0:
sys.stderr.write("No monitors loaded :(\n")
sys.exit(2)
if not options.quiet:
print "--> Loaded %d monitors.\n" % count
m = load_loggers(m, config, options.quiet)
m = load_alerters(m, config, options.quiet)
try:
if config.get("monitor", "remote") == "1":
if not options.no_network:
enable_remote = True
remote_port = int(config.get("monitor", "remote_port"))
else:
enable_remote = False
else:
enable_remote = False
except:
enable_remote = False
if not m.verify_dependencies():
sys.exit(1)
if options.test:
print "--> Config test complete. Exiting."
sys.exit(0)
if not options.quiet:
print
try:
key = config.get("monitor", "key")
except:
key = None
if enable_remote:
if not options.quiet:
print "--> Starting remote listener thread"
remote_listening_thread = Loggers.network.Listener(
m, remote_port, options.verbose, key)
remote_listening_thread.daemon = True
remote_listening_thread.start()
if not options.quiet:
print "--> Starting... (loop runs every %ds) Hit ^C to stop" % interval
loop = True
heartbeat = 0
m.set_verbosity(options.verbose, options.debug)
while loop:
try:
m.run_tests()
m.do_recovery()
m.do_alerts()
m.do_logs()
if not options.quiet and not options.verbose and not options.no_heartbeat:
heartbeat += 1
if heartbeat == 2:
sys.stdout.write(".")
sys.stdout.flush()
heartbeat = 0
except KeyboardInterrupt:
if not options.quiet:
print "\n--> EJECT EJECT"
loop = False
except Exception, e:
sys.exc_info()
sys.stderr.write(
"Caught unhandled exception during main loop: %s\n" % e)
if loop and enable_remote:
if not remote_listening_thread.isAlive():
print "Listener thread died :("
remote_listening_thread = Loggers.network.Listener(
m, remote_port, options.verbose)
remote_listening_thread.start()
try:
time.sleep(interval)
except:
print "\n--> Quitting."
loop = False
def main():
"""This is where it happens \o/"""
parser = OptionParser()
parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False, help="Be more verbose")
parser.add_option("-q", "--quiet", action="store_true", dest="quiet", default=False, help="Don't output anything except errors")
parser.add_option("-t", "--test", action="store_true", dest="test", default=False, help="Test config and exit")
parser.add_option("-p", "--pidfile", dest="pidfile", default=None, help="Write PID into this file")
parser.add_option("-N", "--no-network", dest="no_network", default=False, action="store_true", help="Disable network listening socket")
parser.add_option("-d", "--debug", dest="debug", default=False, action="store_true", help="Enable debug output")
parser.add_option("-f", "--config", dest="config", default="monitor.ini", help="configuration file")
parser.add_option("-H", "--no-heartbeat", action="store_true", dest="no_heartbeat", default=False, help="Omit printing the '.' character when running checks")
parser.add_option('-1', '--one-shot', action='store_true', dest='one_shot', default=False, help='Run the monitors once only, without alerting. Require monitors without "fail" in the name to succeed. Exit zero or non-zero accordingly.')
parser.add_option('--loops', dest='loops', default=-1, help=SUPPRESS_HELP, type=int)
(options, args) = parser.parse_args()
if options.quiet and options.verbose:
options.verbose = False
if options.quiet and options.debug:
options.debug = False
if options.debug and not options.verbose:
options.verbose = True
if not options.quiet:
print("SimpleMonitor v%s" % VERSION)
print("--> Loading main config from %s" % options.config)
config = EnvironmentAwareConfigParser()
if not os.path.exists(options.config):
print('--> Configuration file "%s" does not exist!' % options.config)
sys.exit(1)
try:
config.read(options.config)
except Exception as e:
print('--> Unable to read configuration file "%s"' % options.config)
print('The config parser reported:')
print(e)
sys.exit(1)
try:
interval = config.getint("monitor", "interval")
except Exception:
print('--> Missing [monitor] section from config file, or missing the "interval" setting in it')
sys.exit(1)
pidfile = None
try:
pidfile = config.get("monitor", "pidfile")
except Exception:
pass
if options.pidfile:
pidfile = options.pidfile
if pidfile:
my_pid = os.getpid()
try:
with open(pidfile, "w") as file_handle:
file_handle.write("%d\n" % my_pid)
except Exception:
sys.stderr.write("Couldn't write to pidfile!")
pidfile = None
if config.has_option("monitor", "monitors"):
monitors_file = config.get("monitor", "monitors")
else:
monitors_file = "monitors.ini"
if not options.quiet:
print("--> Loading monitor config from %s" % monitors_file)
m = SimpleMonitor()
m = load_monitors(m, monitors_file, options.quiet)
count = m.count_monitors()
if count == 0:
sys.stderr.write("No monitors loaded :(\n")
sys.exit(2)
if not options.quiet:
print("--> Loaded %d monitors.\n" % count)
m = load_loggers(m, config, options.quiet)
m = load_alerters(m, config, options.quiet)
try:
if config.get("monitor", "remote") == "1":
if not options.no_network:
enable_remote = True
remote_port = int(config.get("monitor", "remote_port"))
else:
enable_remote = False
else:
enable_remote = False
except Exception:
enable_remote = False
if not m.verify_dependencies():
sys.exit(1)
if options.test:
print("--> Config test complete. Exiting.")
sys.exit(0)
if options.one_shot:
print("--> One-shot mode: expecting monitors without 'fail' in the name to succeed,\n and with to fail. Will exit zero or non-zero accordingly.")
if not options.quiet:
print()
try:
key = config.get("monitor", "key")
except Exception:
key = None
if enable_remote:
if not options.quiet:
print("--> Starting remote listener thread")
remote_listening_thread = Loggers.network.Listener(m, remote_port, options.verbose, key)
remote_listening_thread.daemon = True
remote_listening_thread.start()
if not options.quiet:
print("--> Starting... (loop runs every %ds) Hit ^C to stop" % interval)
loop = True
heartbeat = 0
m.set_verbosity(options.verbose, options.debug)
loops = int(options.loops)
while loop:
try:
if loops > 0:
loops -= 1
if loops == 0:
print('Ran out of loop counter, will stop after this one')
loop = False
m.run_tests()
m.do_recovery()
m.do_alerts()
m.do_logs()
if not options.quiet and not options.verbose and not options.no_heartbeat:
heartbeat += 1
if heartbeat == 2:
sys.stdout.write(".")
sys.stdout.flush()
heartbeat = 0
except KeyboardInterrupt:
if not options.quiet:
print("\n--> EJECT EJECT")
loop = False
except Exception as e:
sys.exc_info()
sys.stderr.write("Caught unhandled exception during main loop: %s\n" % e)
if loop and enable_remote:
if not remote_listening_thread.isAlive():
print("Listener thread died :(")
remote_listening_thread = Loggers.network.Listener(m, remote_port, options.verbose)
remote_listening_thread.start()
if options.one_shot:
break
try:
time.sleep(interval)
except Exception:
print("\n--> Quitting.")
loop = False
if enable_remote:
remote_listening_thread.running = False
remote_listening_thread.join(0)
if pidfile:
try:
os.unlink(pidfile)
except Exception as e:
print("Couldn't remove pidfile!")
print(e)
if not options.quiet:
print("--> Finished.")
if options.one_shot:
ok = True
print('\n--> One-shot results:')
for monitor in sorted(m.monitors.keys()):
if "fail" in monitor:
if m.monitors[monitor].error_count == 0:
print(" Monitor {0} should have failed".format(monitor))
ok = False
else:
print(" Monitor {0} was ok (failed)".format(monitor))
elif "skip" in monitor:
if m.monitors[monitor].skipped():
print(" Monitor {0} was ok (skipped)".format(monitor))
else:
print(" Monitor {0} should have been skipped".format(monitor))
ok = False
else:
if m.monitors[monitor].error_count > 0:
print(" Monitor {0} failed and shouldn't have".format(monitor))
ok = False
else:
print(" Monitor {0} was ok".format(monitor))
if not ok:
print("Not all non-'fail' succeeded, or not all 'fail' monitors failed.")
sys.exit(1)