def __init__(self, options=None):
self.handler = StampedNotificationHandler()
self.status = {}
self._info = None
if options is not None:
self.options = options
else:
self.options = utils.AttributeDict({"stack": None, "time": 10, "noop": False, "verbose": False})
class Monitor(object):
def __init__(self, options=None):
self.handler = StampedNotificationHandler()
self.status = {}
self._info = None
if options is not None:
self.options = options
else:
self.options = utils.AttributeDict({"stack": None, "time": 10, "noop": False, "verbose": False})
def update(self, force=False):
if self._info is None or force:
self._info = libs.ec2_utils.get_stack(stack=self.options.stack)
if self._info is None:
utils.log("error retrieving stack data from AWS")
def run(self):
while True:
self.update(force=True)
self.ping()
time.sleep(self.options.time)
def ping(self):
self.update()
if self.options.stack:
mich = None
else:
mich = self._info.instance.instance_id
for node in self._info.nodes:
node_status = 1
# don't try to monitor myself
if node.instance_id == mich:
continue
try:
utils.logRaw("pinging node '%s.%s'..." % (node.stack, node.name), True)
if "apiServer" in node.roles:
self._try_ping_apiServer(node)
if "webServer" in node.roles:
self._try_ping_webServer(node)
if "db" in node.roles:
self._try_ping_db(node)
utils.logRaw("success!\n")
except Exception, e:
node_status = -1
unexpected = False
detail = None
if isinstance(e, MonitorException):
logs.error("monitor error: %s" % e)
utils.log("monitor error: %s" % e)
detail = e.detail
else:
logs.error("unexpected error: %s" % e)
utils.log("unexpected error: %s" % e)
unexpected = True
# only send a notification if this node's status has changed since
# the last time we checked, so we don't get duplicate notifications
# related to the same incident.
try:
notify = -1 != self.status[node.instance_id]
except KeyError:
notify = False
if notify and not self.options.noop:
subject = "%s.%s error" % (node.stack, node.name)
message = str(e)
if unexpected or e.email:
if detail is not None:
message = "%s\n\n--- detail ---\n\n%s" % (message, detail)
self.handler.email(subject, message)
if unexpected or e.sms:
self.handler.sms(subject, message)
self.status[node.instance_id] = node_status
