示例#1
0
class EventHandler(object):

    # Setup logger
    log = setupLogger('eventhandler')

    sessionKey = None
    nh = None

    def __init__(self, sessionKey):
        self.sessionKey = sessionKey

        self.nh = NotificationHandler(self.sessionKey)

    def handleEvent(self, alert, event, incident, context):
        self.log.info(
            "event={} from alert={} incident_id={} has been fired. Calling custom event handlers."
            .format(event, alert, context.get('incident_id')))
        context.update({"event": event})
        try:
            # TODO: Custom event handlers
            self.nh.handleEvent(event, alert, incident, context)

        except Exception as e:
            self.log.error(
                "Error occured during event handling. Error: {}".format(
                    traceback.format_exc()))

        return True

    def setSessionKey(self, sessionKey):
        self.sessionKey = sessionKey
        if self.nh != None:
            self.nh.setSessionKey(sessionKey)
示例#2
0
    incident = getRestData(uri, sessionKey)
    incident["duplicate_count"] = duplicate_count
    if "_user" in incident:
        del (incident["_user"])
    if "_key" in incident:
        del (incident["_key"])
    getRestData(uri, sessionKey, json.dumps(incident))

    log.info("Duplicate count: {}".format(duplicate_count))


if __name__ == "__main__":
    if len(sys.argv) > 1 and sys.argv[1] == "--execute":
        start = time.time()

        log = setupLogger('alert_manager')

        log.debug("Python Version: {}".format(sys.version))

        #
        # BEGING Setup
        #
        payload = json.loads(sys.stdin.read())
        #log.debug("Payload: {}".format(json.dumps(payload)))

        sessionKey = payload.get('session_key')
        job_id = payload.get('sid')
        search_name = payload.get('search_name')
        # Support for manually running the alert action using the 'sendalert' search command
        if search_name == '':
            search_name = 'adhoc'
示例#3
0
import splunk.appserver.mrsparkle.lib.util as util
dir = os.path.join(util.get_apps_dir(), 'alert_manager', 'bin', 'lib')
if not dir in sys.path:
    sys.path.append(dir)

from CsvLookup import CsvLookup
from ApiManager import ApiManager

from AlertManagerLogger import setupLogger

if __name__ == "__main__":
    start = time.time()

    # Setup logger
    log = setupLogger('migration')

    sessionKey = sys.stdin.readline().strip()
    splunk.setDefault('sessionKey', sessionKey)

    # Setup ApiManager
    am = ApiManager(sessionKey=sessionKey)

    #eh = EventHandler(sessionKey=sessionKey)
    #sh = SuppressionHelper(sessionKey=sessionKey)
    #sessionKey     = urllib.unquote(sessionKey[11:]).decode('utf8')

    log.debug("Alert Manager migration started.")

    # By default, don't disable myself
    disableInput = False
示例#4
0
    inherited_roles.append(role)

    for inherited_role in inherited_roles:
        if inherited_role != role:
            new_roles = resolve_roles(inherited_role, roles)
            if len(new_roles) > 0:
                inherited_roles = inherited_roles + list(set(new_roles) - set(inherited_roles))

    return inherited_roles

if __name__ == "__main__":
    start = time.time()

    # Setup logger
    log = setupLogger('scheduler')

    sessionKey     = sys.stdin.readline().strip()
    splunk.setDefault('sessionKey', sessionKey)

    # Setup Helpers
    am = ApiManager(sessionKey = sessionKey)
    eh = EventHandler(sessionKey=sessionKey)
    sh = SuppressionHelper(sessionKey=sessionKey)
    #sessionKey     = urllib.unquote(sessionKey[11:]).decode('utf8')

    log.debug("Scheduler started.")

    # Check KV Store availability
    while not am.checkKvStore():
        log.warn("KV Store is not yet available, sleeping for 1s.")
import splunk
import splunk.appserver.mrsparkle.lib.util as util
import splunk.rest as rest
import splunk.entity as entity
import splunk.input as input

dir = os.path.join(util.get_apps_dir(), 'alert_manager', 'bin', 'lib')
if not dir in sys.path:
    sys.path.append(dir)

from AlertManagerUsers import AlertManagerUsers
from CsvLookup import CsvLookup

from AlertManagerLogger import setupLogger

logger = setupLogger('rest_handler')

if sys.platform == "win32":
    import msvcrt  # pylint: disable=import-error
    # Binary mode is required for persistent mode on Windows.
    msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY)  # pylint: disable=maybe-no-member
    msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)  # pylint: disable=maybe-no-member
    msvcrt.setmode(sys.stderr.fileno(), os.O_BINARY)  # pylint: disable=maybe-no-member

from splunk.persistconn.application import PersistentServerConnectionApplication


class ExternalWorkflowActionsHandler(PersistentServerConnectionApplication):
    def __init__(self, command_line, command_arg):
        PersistentServerConnectionApplication.__init__(self)
示例#6
0
import csv
import os
import json
import sys

import splunk.rest as rest

import splunk.appserver.mrsparkle.lib.util as util
dir = os.path.join(util.get_apps_dir(), 'alert_manager', 'bin', 'lib')
if not dir in sys.path:
    sys.path.append(dir)

from AlertManagerLogger import setupLogger
log = setupLogger('csvlookup')

class CsvLookup(object):

    csv_data    = []

    def __init__(self, file_path = '', lookup_name = '', sessionKey = ''):
        # Reset on init to avoid strange caching effects
        self.csv_data = []

        log.debug("file_path: '{}', lookup_name: '{}'".format(file_path, lookup_name))

        if file_path == '':
            if lookup_name == '':
                raise Exception("No file_path or lookup_name specified.")
            else:
                if sessionKey == '':
                    raise Exception("No sessionKey provided, unable to query REST API.")
class SuppressionHelper(object):

    # Setup logger
    log = setupLogger('suppression_helper')

    sessionKey  = None

    def __init__(self, sessionKey):
        self.sessionKey = sessionKey

    def compareValue(self, test_value, comparator, pattern_value):
        self.log.debug("compareValue(testvalue=\"{}\", comparator=\"{}\", pattern_value=\"{}\")".format(test_value, comparator, pattern_value))

        if type(test_value) is list:
            test_value = test_value[0]

        if type(pattern_value) is list:
            pattern_value = pattern_value[0]

        if comparator == ">":
            return float(test_value) > float(pattern_value)
        elif comparator == "<":
            return float(test_value) < float(pattern_value)
        elif comparator == "=" or comparator == "==" or comparator == "is":
            return test_value == pattern_value
        elif comparator == "!=" or comparator == "is not":
            return test_value != pattern_value
        elif comparator == "<=":
            return float(test_value) <= float(pattern_value)
        elif comparator == ">=":
            return float(test_value) >= float(pattern_value)
        elif comparator == "contains":
            return pattern_value in test_value
        elif comparator == "does not contain":
            return pattern_value not in test_value
        elif comparator == "starts with":
            return bool(re.match("^" + pattern_value + ".*", test_value))
        elif comparator == "ends with":
            return bool(re.match(".*" + pattern_value + "$", test_value))
        else:
            return False

    def checkSuppression(self, alert, context):
        self.log.info("Checking for matching suppression rules for alert={}".format(alert))
        #query = '{  "disabled": false, "$or": [ { "scope": "*" } , { "scope": "'+ alert +'" } ] }'
        query = '{{ "disabled": false, "$or": [{{ "scope" : "{}"}}, {{ "scope": {{ "$regex": "\\\*"}}  }} ]}}'.format(alert)
        self.log.debug("Query: {}".format(query))
        uri = '/servicesNS/nobody/alert_manager/storage/collections/data/suppression_rules?query={}'.format(urllib.parse.quote(query))
        serverResponse, serverContent = rest.simpleRequest(uri, sessionKey=self.sessionKey)

        if serverResponse['status'] == "200" and len(serverContent.decode('utf-8')) > 0:
            suppression_rules = json.loads(serverContent.decode('utf-8'))
            self.log.debug("Got {} suppression rule(s) matching the scope ('*' or '{}').".format(len(suppression_rules), alert))
            self.log.debug("Context: {}".format(json.dumps(context)))

            matching_rules = []
            unmatching_rules = []

            for suppression_rule in suppression_rules:

                # check if scope matches alert <-> suppression_rule['scope']
                if fnmatch.fnmatch(alert, suppression_rule['scope']):

                    match_type = 'all'
                    if 'match_type' in suppression_rule and suppression_rule['match_type'] != '':
                        match_type = suppression_rule['match_type']

                    self.log.debug("Match type: {}".format(match_type))

                    # iterate over rules of suppressions
                    ruleset_suppression_all = True
                    ruleset_suppression_any = False

                    if "rules" in suppression_rule:
                        for rule in suppression_rule["rules"]:
                            rule_suppression = False

                            self.log.debug("Rule: suppression_title=\"{}\" field=\"{}\" condition=\"{}\" value=\"{}\"".format(suppression_rule['suppression_title'], rule["field"], rule["condition"], rule["value"]))

                            # Parse value from results
                            value_match = re.match("^\$(.*)\$$", rule["value"])
                            if bool(value_match):
                                value_field_name = value_match.group(1)
                                if len(context["result"]) > 0 and value_field_name in context["result"][0]:
                                    rule["value"] =  context["result"][0][value_field_name]
                                else:
                                    self.log.warn("Invalid suppression rule: value field {} not found in results.".format(value_field_name))

                            # Parse special case "time"
                            if rule["field"] == "_time" or rule["field"] == "time":
                                # FIXME: Change timestamp to real timestamp from incident
                                match = self.compareValue(int(time.time()), rule["condition"], rule["value"])
                                if not match:
                                    rule_suppression = False
                                    self.log.debug("Rule {} didn't match.".format(json.dumps(rule)))
                                else:
                                    rule_suppression = True
                                    self.log.debug("Rule {} matched.".format(json.dumps(rule)))

                            # Parse rules refering to fields
                            else:
                                #field_match = re.match("^\$(.*)\$$", rule["field"])
                                #field_match_result = re.match("^\$result\.(.*)\$$", rule["field"])
                                field_match = re.match("^\$(result\.)?(.*)\$$", rule["field"])

                                if bool(field_match):
                                    field_name = field_match.group(2)
                                    self.log.debug("Field name: {}".format(field_name))

                                    if 'result' in context and field_name in context["result"]:
                                        match = self.compareValue(context["result"][field_name], rule["condition"], rule["value"])
                                        if not match:
                                            rule_suppression = False
                                            self.log.debug("Rule {} didn't match.".format(json.dumps(rule)))
                                        else:
                                            rule_suppression = True
                                            self.log.debug("Rule {} matched.".format(json.dumps(rule)))
                                    else:
                                        self.log.warn("Invalid suppression rule: field {} not found in result.".format(field_name))

                                else:
                                    self.log.warn("Suppression rule has an invalid field content format.")

                            # Apply suppression state for this specific rule
                            if rule_suppression:
                                ruleset_suppression_any = True

                            if ruleset_suppression_all and rule_suppression:
                                ruleset_suppression_all = True
                            else:
                                ruleset_suppression_all = False


                        # Check if suppression for this ruleset was successful
                        if match_type == "all":
                            if ruleset_suppression_all:
                                matching_rules.append(suppression_rule['suppression_title'])
                                self.log.info("Suppression for rule with suppression_title='{}' was successful (match_type={}).".format(suppression_rule['suppression_title'], match_type))
                            else:
                                unmatching_rules.append(suppression_rule['suppression_title'])
                                self.log.info("Suppression for rule with suppression_title='{}' was NOT successful (match_type={}).".format(suppression_rule['suppression_title'], match_type))

                        if match_type == "any":
                            if ruleset_suppression_any:
                                matching_rules.append(suppression_rule['suppression_title'])
                                self.log.info("Suppression for rule with suppression_title='{}' was successful (match_type={}).".format(suppression_rule['suppression_title'], match_type))


                else:
                    self.log.info("Scope from rule ({}) didn't match to alert name ({}), skipping...".format(suppression_rule['scope'], alert))

            # Check if suppression was successful
            if len(matching_rules) > 0:
                self.log.info("Suppression successful: At least one matching suppression rule(s) was found. Matching rules : {}. Unmatching rules: {}".format(', '.join(matching_rules), ', '.join(unmatching_rules)))
                return True, matching_rules
            else:
                self.log.info("Suppression failed: No matching rules found. Unmatching rules: {}".format(', '.join(unmatching_rules)))
                return False, []

        else:
            self.log.debug("Failed to get suppression rules with query={}. Maybe no matching rules found? (status={})".format(query, serverResponse['status']))
            return False, []
示例#8
0
class IncidentContext(object):

	log = setupLogger('incidentcontext')

	sessionKey = None
	context = { }

	incident = {}

	def __init__(self, sessionKey, incident_id):
		self.sessionKey = sessionKey

		query = {}
		query['incident_id'] = incident_id

		uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incidents?query={}'.format(urllib.parse.quote(json.dumps(query)))
		serverResponse, serverContent = rest.simpleRequest(uri, sessionKey=sessionKey)
		incident = json.loads(serverContent.decode('utf-8'))
		incident = incident[0]

		query_incident_settings = {}
		query_incident_settings['alert'] = incident["alert"]
		uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incident_settings?query={}'.format(urllib.parse.quote(json.dumps(query_incident_settings)))
		serverResponse, serverContent = rest.simpleRequest(uri, sessionKey=sessionKey)
		incident_settings = json.loads(serverContent.decode('utf-8'))
		if len(incident_settings) > 0:
			incident_settings = incident_settings[0]

		uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incident_results?query={}'.format(urllib.parse.quote(json.dumps(query)))
		serverResponse, serverContent = rest.simpleRequest(uri, sessionKey=sessionKey)
		results = json.loads(serverContent.decode('utf-8'))
		if len(results) > 0:
			results = results[0]

		uri = '/services/server/settings?output_mode=json'
		serverResponse, serverContent = rest.simpleRequest(uri, sessionKey=sessionKey)
		server_settings = json.loads(serverContent.decode('utf-8'))
		if len(server_settings) > 0:
			server_settings = server_settings["entry"][0]["content"]

		uri = '/services/server/info?output_mode=json'
		serverResponse, serverContent = rest.simpleRequest(uri, sessionKey=sessionKey)
		server_info = json.loads(serverContent.decode('utf-8'))
		if len(server_info) > 0:
			server_info = server_info["entry"][0]["content"]

		self.setContext(incident, incident_settings, results, server_info, server_settings)

	def setContext(self, incident, incident_settings, results, server_info, server_settings):
		context = self.context
		try:
			http_port = "8000"
			if 'httpport' in server_settings:
				http_port = str(server_settings['httpport'])

			protocol = 'http'
			if 'enableSplunkWebSSL' in server_settings and self.normalize_bool(str(server_settings['enableSplunkWebSSL'])):
				protocol = 'https'

			context.update({ "_key": incident['_key']})
			context.update({ "incident_id": incident['incident_id']})
			context.update({ "job_id": incident['job_id']})
			context.update({ "title": incident['title']})
			context.update({ "alert_time" : incident["alert_time"] })
			context.update({ "owner" : incident["owner"] })
			context.update({ "name" : incident["alert"] })
			context.update({ "alert" : { "impact": incident["impact"], "urgency": incident["urgency"], "priority": incident["priority"], "expires": incident["ttl"] } })
			context.update({ "app" : incident["app"] })
			context.update({ "external_reference_id": incident["external_reference_id"] })
			if 'category' in incident_settings:
				context.update({ "category" : incident_settings['category'] })
			if 'subcategory' in incident_settings:
				context.update({ "subcategory" : incident_settings['subcategory'] })
			if 'tags' in incident_settings:
				context.update({ "tags" : incident_settings['tags'] })
			context.update({ "results_link" : protocol + "://"+server_info["host_fqdn"] + ":"+ http_port +"/app/" + incident["app"] + "/@go?sid=" + incident["job_id"] })
			context.update({ "view_link" : protocol + "://"+server_info["host_fqdn"] + ":" + http_port + "/app/" + incident["app"] + "/alert?s=" + urllib.parse.quote("/servicesNS/nobody/"+incident["app"]+"/saved/searches/" + incident["alert"] ) })
			context.update({ "server" : { "version": server_info["version"], "build": server_info["build"], "serverName": server_info["serverName"] } })

			if 'status' in incident:
				context.update({ "status" : incident["status"] })

			if "fields" in results:
				result_context = { "result" : results["fields"][0] }
				context.update(result_context)
				results_context = { "results" : results["fields"] }
				context.update(results_context)

		except Exception as e:
			#exc_type, exc_obj, exc_tb = sys.exc_info()
			self.log.error("Error occured during event handling. Error: {}".format((traceback.format_exc())))
			return "Error occured during event handling. Error: {}".format(traceback.format_exc())

		self.context = context

	def update(self, key, value):
		self.context.update({ key : value })
		return True

	def get(self, key):
		return self.context.get(key, False)

	def getContext(self):
		return self.context

	def normalize_bool(self, value):
		return True if value.lower() in ('1', 'true') else False
示例#9
0
 def __init__(self, sessionKey):
     self.sessionKey = sessionKey
     self.log = setupLogger('apimanager')
示例#10
0
class NotificationHandler(object):

    # Setup logger
    log = setupLogger('notifications')

    sessionKey = None
    env = None
    default_sender = None
    settings = {}

    def __init__(self, sessionKey):
        self.sessionKey = sessionKey

        # Setup template paths
        local_dir = os.path.join(util.get_apps_dir(), "alert_manager",
                                 "default", "templates")
        default_dir = os.path.join(util.get_apps_dir(), "alert_manager",
                                   "local", "templates")
        loader = FileSystemLoader([default_dir, local_dir])
        self.env = Environment(loader=loader,
                               variable_start_string='$',
                               variable_end_string='$')

        # TODO: Add support for custom filters
        self.env.filters['get_type'] = get_type

        # Get mailserver settings from splunk
        uri = '/servicesNS/nobody/system/configs/conf-alert_actions/email?output_mode=json'
        serverResponse, serverContent = rest.simpleRequest(
            uri, sessionKey=self.sessionKey)
        server_settings = json.loads(serverContent.decode('utf-8'))
        server_settings = server_settings["entry"][0]["content"]
        #self.log.debug("server settings from splunk: {}".format(json.dumps(server_settings)))

        self.default_sender = server_settings['from']

        use_ssl = False
        if server_settings['use_ssl']:
            use_ssl = True

        use_tls = False
        if server_settings['use_tls']:
            use_tls = True

        # Configure django settings
        clear_pass = ''
        if 'clear_password' in server_settings:
            clear_pass = server_settings['clear_password']

        auth_username = ""
        if 'auth_username' in server_settings:
            auth_username = server_settings['auth_username']

        mail_server = "localhost"
        if 'mailserver' in server_settings:
            mail_server = server_settings['mailserver']

        self.settings = {
            "MAIL_SERVER": mail_server,
            "EMAIL_HOST_USER": auth_username,
            "EMAIL_HOST_PASSWORD": clear_pass,
            "EMAIL_USE_TLS": use_tls,
            "EMAIL_USE_SSL": use_ssl
        }

    def handleEvent(self, event, alert, incident, context):
        self.log.debug("Start handleEvent")
        self.log.debug("Incident: {}".format(incident))
        self.log.debug("Context: {}".format(context))

        notificationSchemeName = self.getNotificationSchemeName(alert)
        notificationScheme = NotificationScheme(self.sessionKey,
                                                notificationSchemeName)
        notifications = notificationScheme.getNotifications(event)

        if len(notifications) > 0:
            for notification in notifications:
                # Parse template
                template_match = re.search("^\$(.+)\.(.+)\$$",
                                           notification["template"])
                if template_match != None:
                    result_type = template_match.group(1)
                    field_name = template_match.group(2)
                    self.log.debug(
                        "Template ({}) references to a field name, starting to parse"
                        .format(notification["template"]))
                    if result_type == 'result' and "result" in context and field_name in context[
                            "result"]:
                        notification["template"] = context["result"][
                            field_name]
                        self.log.debug(
                            "{} found in result. Parsed value {} as template name."
                            .format(field_name, notification["template"]))
                    else:
                        self.log.warn(
                            "Field {} not found in '{}'. Won't send a notification."
                            .format(field_name, result_type))

                # Parse sender
                if notification["sender"] == "default_sender":
                    notification["sender"] = self.default_sender

                # Parse recipients
                recipients = []
                recipients_cc = []
                recipients_bcc = []

                notification_recipients = notification["recipients"]

                # Test if manual notification overwrites recipients
                if context.get("recipients_overwrite") == "true":
                    self.log.debug("Overwriting recipients: true")
                    notification_recipients = context.get("recipients").split(
                        ",")

                self.log.debug("notification_recipients: {}".format(
                    notification_recipients))

                for recipient in notification_recipients:
                    recipient_ok = True

                    # Parse recipient mode
                    if ":" in recipient:
                        search = re.search("(mailto|mailcc|mailbcc)\:(.+)",
                                           recipient)
                        mode = search.group(1)
                        recipient = search.group(2)
                    else:
                        mode = "mailto"

                    # Parse recipient string
                    if recipient == "current_owner":
                        users = AlertManagerUsers(sessionKey=self.sessionKey)
                        user = users.getUser(incident["owner"])
                        if incident["owner"] != "unassigned":
                            recipient = user["email"]
                        else:
                            self.log.info(
                                "Can't send a notification to 'unassigned' or a user who is configured to not receive notifications. alert={} owner={} event={}"
                                .format(alert, incident["owner"], event))
                            recipient_ok = False

                    else:
                        # Check if recipient is a crosslink to a result field and parse
                        field_recipient = re.search("\$(.+)\.(.+)\$",
                                                    recipient)
                        if field_recipient != None:
                            result_type = field_recipient.group(1)
                            field_name = field_recipient.group(2)
                            self.log.debug(
                                "Should use a recipient from array '{}'. field: {}."
                                .format(result_type, field_name))
                            if result_type == 'result' and "result" in context and field_name in context[
                                    "result"]:
                                recipient = context["result"][
                                    field_name].split(",")
                                self.log.debug(
                                    "{} found in result. Parsed value {}.".
                                    format(field_name, recipient))
                            else:
                                self.log.warn(
                                    "Field {} not found in '{}'. Won't send a notification."
                                    .format(field_name, result_type))
                                recipient_ok = False

                    if recipient_ok:
                        if mode == "mailto":
                            if isinstance(recipient, list):
                                recipients = recipients + recipient
                            else:
                                recipients.append(recipient)
                        elif mode == "mailcc":
                            if isinstance(recipient, list):
                                recipients_cc = recipients_cc + recipient
                            else:
                                recipients_cc.append(recipient)
                        elif mode == "mailbcc":
                            if isinstance(recipient, list):
                                recipients_bcc = recipients_bcc + recipient
                            else:
                                recipients_bcc.append(recipient)

                if len(recipients) > 0 or len(recipients_cc) > 0 or len(
                        recipients_bcc) > 0:
                    self.log.info(
                        "Prepared notification. event={}, alert={}, template={}, sender={}, recipients={}, recipients_cc={}, recipients_bcc={}"
                        .format(event, alert, notification["template"],
                                notification["sender"], recipients,
                                recipients_cc, recipients_bcc))
                    self.send_notification(event, alert,
                                           notification["template"],
                                           notification["sender"], recipients,
                                           recipients_cc, recipients_bcc,
                                           context)
                else:
                    self.log.info(
                        "Done parsing notifications but will stop here since no recipients are present."
                    )

        return True

    def send_notification(self,
                          event,
                          alert,
                          template_name,
                          sender,
                          recipients,
                          recipients_cc=[],
                          recipients_bcc=[],
                          context={}):
        all_recipients = recipients + recipients_cc + recipients_bcc
        self.log.info(
            "Start trying to send notification to {} with event={} of alert {}"
            .format(str(all_recipients), event, alert))

        mail_template = self.get_email_template(template_name)
        if mail_template != False:
            self.log.debug(
                "Found template file ({}). Ready to send notification.".format(
                    json.dumps(mail_template)))

            # Parse html template with django
            try:
                # Parse body as django template
                template = self.env.get_template(
                    mail_template['template_file'])
                content = template.render(context)

                #self.log.debug("Parsed message body. Context was: {}".format(json.dumps(context)))
                text_content = strip_tags(content)

                # Parse subject as django template
                subject_template = Template(source=mail_template['subject'],
                                            variable_start_string='$',
                                            variable_end_string='$')
                subject = subject_template.render(context)
                self.log.debug("Parsed message subject: {}".format(subject))

                # Prepare message
                self.log.debug("Preparing SMTP message...")
                message = MIMEMultipart('mixed')
                message['Subject'] = subject
                message['From'] = sender
                message['Date'] = formatdate(localtime=True)

                smtpRecipients = []

                if len(recipients) > 0:
                    smtpRecipients = smtpRecipients + recipients
                    message['To'] = COMMASPACE.join(recipients)
                if len(recipients_cc) > 0:
                    smtpRecipients = smtpRecipients + recipients_cc
                    message['CC'] = COMMASPACE.join(recipients_cc)

                if len(recipients_bcc) > 0:
                    smtpRecipients = smtpRecipients + recipients_bcc
                    message['BCC'] = COMMASPACE.join(recipients_bcc)

                message_alternative = MIMEMultipart('alternative')
                message_related = MIMEMultipart('related')

                # Add message body
                if mail_template['content_type'] == "html":
                    message_alternative.attach(MIMEText(text_content, 'plain'))
                    message_related.attach(MIMEText(content, 'html', 'utf-8'))
                else:
                    message_alternative.attach(MIMEText(text_content, 'plain'))

                message_alternative.attach(message_related)
                message.attach(message_alternative)

                # Add attachments
                if 'attachments' in mail_template and mail_template[
                        'attachments'] != None and mail_template[
                            'attachments'] != "":
                    attachment_list = mail_template['attachments'].split(" ")
                    self.log.debug(
                        "Have to add attachments to this notification. Attachment list: {}"
                        .format(json.dumps(attachment_list)))

                    for attachment in attachment_list or []:
                        local_file = os.path.join(util.get_apps_dir(),
                                                  "alert_manager", "local",
                                                  "templates", "attachments",
                                                  attachment)
                        default_file = os.path.join(util.get_apps_dir(),
                                                    "alert_manager", "default",
                                                    "templates", "attachments",
                                                    attachment)

                        attachment_file = None
                        if os.path.isfile(local_file):
                            attachment_file = local_file
                            self.log.debug(
                                "{} exists in local, using this one...".format(
                                    attachment))
                        else:
                            self.log.debug(
                                "{} not found in local folder, checking if there's one in default..."
                                .format(attachment))
                            if os.path.isfile(default_file):
                                attachment_file = default_file
                                self.log.debug(
                                    "{} exists in default, using this one...".
                                    format(attachment))
                            else:
                                self.log.warn(
                                    "{} doesn't exist, won't add it to the message."
                                    .format(attachment))

                        if attachment_file != None:
                            ctype, encoding = mimetypes.guess_type(
                                attachment_file)
                            if ctype is None or encoding is not None:
                                ctype = "application/octet-stream"
                            maintype, subtype = ctype.split("/", 1)

                            message_attachment = None
                            if maintype == "text":
                                try:
                                    fp = open(attachment_file)
                                    # Note: we should handle calculating the charset
                                    message_attachment = MIMEText(
                                        fp.read(), _subtype=subtype)
                                finally:
                                    fp.close()
                            elif maintype == "image":
                                try:
                                    fp = open(attachment_file, "rb")
                                    message_attachment = MIMEImage(
                                        fp.read(), _subtype=subtype)
                                finally:
                                    fp.close()
                            elif maintype == "audio":
                                try:
                                    fp = open(attachment_file, "rb")
                                    message_attachment = MIMEAudio(
                                        fp.read(), _subtype=subtype)
                                finally:
                                    fp.close()
                            else:
                                try:
                                    fp = open(attachment_file, "rb")
                                    message_attachment = MIMEBase(
                                        maintype, subtype)
                                    message_attachment.set_payload(fp.read())
                                    encoders.encode_base64(message_attachment)
                                finally:
                                    fp.close()

                            if message_attachment != None:
                                message_attachment.add_header(
                                    "Content-ID", "<" +
                                    basename(attachment_file) + "@splunk>")
                                message_attachment.add_header(
                                    "Content-Disposition",
                                    "attachment",
                                    filename=basename(attachment_file))
                                message_related.attach(message_attachment)

                #self.log.debug("Mail message: {}".format(msg.as_string()))
                #self.log.debug("Settings: {}".format(json.dumps(self.settings)))
                self.log.debug("smtpRecipients: {} type: {}".format(
                    smtpRecipients, type(smtpRecipients)))
                self.log.info(
                    "Connecting to mailserver={} ssl={} tls={}".format(
                        self.settings["MAIL_SERVER"],
                        self.settings["EMAIL_USE_SSL"],
                        self.settings["EMAIL_USE_TLS"]))
                if not self.settings["EMAIL_USE_SSL"]:
                    s = smtplib.SMTP(self.settings["MAIL_SERVER"])
                else:
                    s = smtplib.SMTP_SSL(self.settings["MAIL_SERVER"])

                if self.settings["EMAIL_USE_TLS"]:
                    s.starttls()

                if len(self.settings["EMAIL_HOST_USER"]) > 0:
                    s.login(str(self.settings["EMAIL_HOST_USER"]),
                            str(self.settings["EMAIL_HOST_PASSWORD"]))

                self.log.info("Sending emails....")
                s.sendmail(sender, smtpRecipients, message.as_string())
                s.quit()

                self.log.info("Notifications sent successfully")

            #except TemplateSyntaxError, e:
            #    self.log.error("Unable to parse template {}. Error: {}. Continuing without sending notification...".format(mail_template['template_file'], e)))
            #except smtplib.SMTPServerDisconnected, e:
            #    self.log.error("SMTP server disconnected the connection. Error: {}".format(e))
            except socket.error as e:
                self.log.error(
                    "Wasn't able to connect to mailserver. Reason: {}".format(
                        e))
            #except TemplateDoesNotExist, e:
            #    self.log.error("Template {} not found in {} nor {}. Continuing without sending notification...".format(mail_template['template_file'], local_dir, default_dir)))
            except Exception as e:
                self.log.error(
                    "Unable to send notification. Continuing without sending notification. Unexpected Error: {}"
                    .format(traceback.format_exc()))
        else:
            self.log.warn("Unable to find template file ({}).".format(
                json.dumps(mail_template)))

    def getNotificationSchemeName(self, alert):
        # Retrieve notification scheme from KV store
        query_filter = {}
        query_filter["alert"] = alert
        uri = '/servicesNS/nobody/alert_manager/storage/collections/data/incident_settings/?query={}'.format(
            urllib.parse.quote(json.dumps(query_filter)))
        serverResponse, serverContent = rest.simpleRequest(
            uri, sessionKey=self.sessionKey)

        entries = json.loads(serverContent.decode('utf-8'))

        try:
            return entries[0]["notification_scheme"]

        except Exception as e:
            # TODO: Check response, fall back to default notification scheme
            return None

    def get_email_template(self, template_name):
        query = {}
        query["template_name"] = template_name
        uri = '/servicesNS/nobody/alert_manager/storage/collections/data/email_templates?output_mode=json&query={}'.format(
            urllib.parse.quote(json.dumps(query)))
        serverResponse, serverContent = rest.simpleRequest(
            uri, sessionKey=self.sessionKey)
        self.log.debug("Response for template listing: {}".format(
            serverContent.decode('utf-8')))
        entries = json.loads(serverContent.decode('utf-8'))

        if len(entries) > 0:
            return entries[0]
        else:
            self.log.error(
                "Template {} not found in email_templates! Aborting...".format(
                    template_name))
            return False

    def get_template_file(self, template_file_name):

        self.log.debug("Parsed template file from settings: {}".format(
            template_file_name))

        local_file = os.path.join(util.get_apps_dir(), "alert_manager",
                                  "local", "templates", template_file_name)
        default_file = os.path.join(util.get_apps_dir(), "alert_manager",
                                    "default", "templates", template_file_name)

        if os.path.isfile(local_file):
            self.log.debug("{} exists in local, using this one...".format(
                template_file_name))
            return local_file
        else:
            self.log.debug(
                "{} not found in local folder, checking if there's one in default..."
                .format(template_file_name))
            if os.path.isfile(default_file):
                self.log.debug(
                    "{} exists in default, using this one...".format(
                        template_file_name))
                return default_file
            else:
                self.log.debug(
                    "{} doesn't exist at all, stopping here.".format(
                        template_file_name))
                return False

    def setSessionKey(self, sessionKey):
        self.sessionKey = sessionKey