def initialize(self):
"""
Set up the PagerDuty API client and add the notification callback.
"""
self.pager = PagerDuty(self.config['Subdomain'], self.config['APIKey'])
self.service_keys = dict((
(collectd.NOTIF_FAILURE, self.config['FailureServiceKey']),
(collectd.NOTIF_WARNING, self.config['WarningServiceKey']),
))
self.add_notification_callback(self.notify)
def checkForPages(self):
if self.args.verbose:
print "Connecting to PagerDuty..."
pager = PagerDuty(self.config.get("settings","pagerdutyHost"),self.config.get("settings","pagerdutyApiKey"))
currentTime = datetime.utcnow()
cronInterval = self.config.getint("settings","cronInterval")
pastTime = currentTime - timedelta(minutes=cronInterval)
currentTimeStr = currentTime.strftime('%Y-%m-%dT%X')
pastTimeStr= pastTime.strftime('%Y-%m-%dT%X')
argList = {}
argList['since'] = pastTimeStr
argList['until'] = currentTimeStr
argList['fields'] = "incident_number,status,trigger_summary_data,last_status_change_by,service"
if self.config.getboolean("settings","triggeredOnly"):
argList['status'] = 'triggered'
if self.args.verbose:
print "Fetching incidents..."
print argList
incidents = pager.incidents.list(**argList)
if self.args.verbose:
print "Looping incidents..."
for incident in incidents:
if self.args.verbose:
print incident
incident_json = incident.trigger_summary_data.to_json()
subject = incident.service.name
message = " - ".join(item[1] for item in incident_json.items())
if incident.status != 'triggered':
if incident.last_status_change_by is not None:
message = incident.last_status_change_by.name + " " + incident.status + " " + message
else:
message = "API " + incident.status + " " + message
if self.args.verbose:
print "Sending incident popup (%s - %s)" % (subject,message)
self.sendIncident(subject,message)
time.sleep(self.config.getfloat("settings","notificationDelay"))
class PagerDutyNotifier(CollectDPlugin):
"""
Collectd plugin for sending notifications to PagerDuty.
"""
def configure(self, config, **kwargs):
"""
Ensure the required configuration options are set in collectd's config.
"""
super(PagerDutyNotifier, self).configure(config, **kwargs)
for key in ('APIKey', 'Subdomain',
'WarningServiceKey', 'FailureServiceKey'):
if key not in self.config.keys():
message = 'Required configuration key %s missing!' % key
self.error(message)
raise PluginError(message)
def initialize(self):
"""
Set up the PagerDuty API client and add the notification callback.
"""
self.pager = PagerDuty(self.config['Subdomain'], self.config['APIKey'])
self.service_keys = dict((
(collectd.NOTIF_FAILURE, self.config['FailureServiceKey']),
(collectd.NOTIF_WARNING, self.config['WarningServiceKey']),
))
self.add_notification_callback(self.notify)
def services(self, trigger_type, severity):
"""
Return the service keys to send the event to. For trigger events, this
is a 1-item list containing the configured service key for the
given severity. For resolve events, this is a list of all service
keys, because we need to ensure the event is resolved everywhere it
was triggered.
"""
if trigger_type == 'resolve':
keys = self.service_keys.values()
else:
keys = [self.service_keys[severity]]
return keys
def notify(self, notification):
"""
Send the notification to PagerDuty.
"""
# Use a friendly string instead of a number.
severity = {
collectd.NOTIF_FAILURE: 'FAILURE',
collectd.NOTIF_WARNING: 'WARNING',
collectd.NOTIF_OKAY: 'OKAY',
}.get(notification.severity)
# Short description - this ends up in SMS and email subject, so
# keep it short!
description = '%s on %s (from %s plugin)' % (
severity,
notification.host,
notification.plugin,
)
# This is the details of the incident, will end up in the email
# body and the PagerDuty interface. Go crazy.
details = {
'host': notification.host,
'plugin': notification.plugin,
'plugin_instance': notification.plugin_instance,
'type': notification.type,
'type_instance': notification.type_instance,
'message': notification.message,
'severity': severity,
}
# We construct an incident key from any of these components that
# are not None.
components = filter(None, [
notification.host,
notification.plugin,
notification.plugin_instance,
notification.type,
notification.type_instance,
])
incident_key = '.'.join(components)
# Figure out what kind of trigger this is.
trigger_type = {
collectd.NOTIF_FAILURE: 'trigger',
collectd.NOTIF_WARNING: 'trigger',
collectd.NOTIF_OKAY: 'resolve',
}.get(notification.severity)
# Finally, trigger the event in PagerDuty
service_keys = self.services(trigger_type, notification.severity)
for service in service_keys:
self.info('Triggering %r event in PagerDuty service %s: %r' %
(trigger_type, service, details))
self.pager.create_event(
service,
description,
trigger_type,
details,
incident_key
)