def do_notify(self, event):
if event.type == "digest":
subject = event.tip_info["body"]
body = event.tip_info["title"]
else:
subject, body = self.get_mail_subject_and_body(event)
receiver_mail = event.receiver_info["mail_address"]
# If the receiver has encryption enabled (for notification), encrypt the mail body
if event.receiver_info["pgp_key_status"] == u"enabled":
gpob = GLBPGP()
try:
gpob.load_key(event.receiver_info["pgp_key_public"])
body = gpob.encrypt_message(event.receiver_info["pgp_key_fingerprint"], body)
except Exception as excep:
log.err(
"Error in PGP interface object (for %s: %s)! (notification+encryption)"
% (event.receiver_info["username"], str(excep))
)
# On this condition (PGP enabled but key invalid) the only
# thing to do is to return None;
# It will be duty of the PGP check schedule will disable the key
# and advise the user and the admin about that action.
return fail(None)
finally:
# the finally statement is always called also if
# except contains a return or a raise
gpob.destroy_environment()
return sendmail(receiver_mail, subject, body)
def do_notify(self, event):
if event.type == 'digest':
subject = event.tip_info['body']
body = event.tip_info['title']
else:
subject, body = self.get_mail_subject_and_body(event)
receiver_mail = event.receiver_info['mail_address']
# If the receiver has encryption enabled (for notification), encrypt the mail body
if event.receiver_info['pgp_key_status'] == u'enabled':
gpob = GLBPGP()
try:
gpob.load_key(event.receiver_info['pgp_key_public'])
body = gpob.encrypt_message(event.receiver_info['pgp_key_fingerprint'], body)
except Exception as excep:
log.err("Error in PGP interface object (for %s: %s)! (notification+encryption)" %
(event.receiver_info['username'], str(excep)))
# On this condition (PGP enabled but key invalid) the only
# thing to do is to return None;
# It will be duty of the PGP check schedule will disable the key
# and advise the user and the admin about that action.
return fail(None)
finally:
# the finally statement is always called also if
# except contains a return or a raise
gpob.destroy_environment()
return sendmail(receiver_mail, subject, body)
def send_pgp_alerts(self, node_desc, receiver_desc, notification_settings):
fakeevent = OD()
fakeevent.type = u'pgp_expiration_alert'
fakeevent.node_info = node_desc
fakeevent.context_info = None
fakeevent.steps_info = None
fakeevent.receiver_info = receiver_desc
fakeevent.tip_info = None
fakeevent.subevent_info = None
body = Templating().format_template(
notification_settings['pgp_alert_mail_template'], fakeevent)
title = Templating().format_template(
notification_settings['pgp_alert_mail_title'], fakeevent)
to_address = receiver_desc['mail_address']
message = MIME_mail_build(GLSetting.memory_copy.notif_source_name,
GLSetting.memory_copy.notif_source_email,
to_address,
to_address,
title,
body)
yield sendmail(authentication_username=GLSetting.memory_copy.notif_username,
authentication_password=GLSetting.memory_copy.notif_password,
from_address=GLSetting.memory_copy.notif_source_email,
to_address=to_address,
message_file=message,
smtp_host=GLSetting.memory_copy.notif_server,
smtp_port=GLSetting.memory_copy.notif_port,
security=GLSetting.memory_copy.notif_security,
event=None)
def post(self):
user = yield get_user_settings(self.current_user.user_id,
GLSettings.memory_copy.default_language)
language = user['language']
yield get_notification(language)
data = {}
data['type'] = 'admin_test_static'
data['node'] = yield admin_serialize_node(language)
data['notification'] = yield get_notification(language)
subject, body = Templating().get_mail_subject_and_body(data)
send_to = user['mail_address']
log.debug("Attempting to send test email to: %s" % send_to)
# If sending the email fails the exception mail address will be mailed.
# If the failure is due to a bad SMTP config that will fail too, but it
# doesn't hurt to try!
try:
yield sendmail(send_to, subject, body)
except Exception as e:
log.debug("Sending to admin failed. Trying an exception mail")
raise e
def send_pgp_alerts(self, node_desc, receiver_desc, notification_settings):
fakeevent = OD()
fakeevent.type = u'pgp_expiration_alert'
fakeevent.node_info = node_desc
fakeevent.context_info = None
fakeevent.steps_info = None
fakeevent.receiver_info = receiver_desc
fakeevent.tip_info = None
fakeevent.subevent_info = None
body = Templating().format_template(
notification_settings['pgp_alert_mail_template'], fakeevent)
title = Templating().format_template(
notification_settings['pgp_alert_mail_title'], fakeevent)
to_address = receiver_desc['mail_address']
message = MIME_mail_build(GLSetting.memory_copy.notif_source_name,
GLSetting.memory_copy.notif_source_email,
to_address,
to_address,
title,
body)
yield sendmail(authentication_username=GLSetting.memory_copy.notif_username,
authentication_password=GLSetting.memory_copy.notif_password,
from_address=GLSetting.memory_copy.notif_source_email,
to_address=to_address,
message_file=message,
smtp_host=GLSetting.memory_copy.notif_server,
smtp_port=GLSetting.memory_copy.notif_port,
security=GLSetting.memory_copy.notif_security,
event=None)
def operation(self):
@transact
def delete_sent_mail(store, mail_id):
store.find(models.Mail, models.Mail.id == mail_id).remove()
@inlineCallbacks
def _success_callback(result, mail_id):
yield delete_sent_mail(mail_id)
def _failure_callback(failure, mail_id):
pass
mail_generator = MailGenerator()
for trigger in ['ReceiverTip', 'Comment', 'Message', 'ReceiverFile']:
yield mail_generator.process_data(trigger)
mails = yield self.get_mails_from_the_pool()
for mail in mails:
sendmail_deferred = sendmail(mail['address'], mail['subject'],
mail['body'])
sendmail_deferred.addCallbacks(_success_callback,
_failure_callback,
callbackArgs=(mail['id'], ),
errbackArgs=(mail['id'], ))
yield sendmail_deferred
def operation():
try:
(two_weeks, three_days, gone) = yield check_expiration_date()
messages = dict({})
for username, sincepoch in two_weeks.iteritems():
messages.update({ username : untranslated_template % "expire in two weeks" })
for username, sincepoch in three_days.iteritems():
messages.update({ username : untranslated_template % "expire in three days" })
for username, sincepoch in gone.iteritems():
messages.update({ username : untranslated_template % "it's already expired" })
for recipient, message in messages.iteritems():
mail_building = ["Date: %s" % rfc822_date(),
"From: \"%s\" <%s>" %
( GLSetting.memory_copy.notif_source_name,
GLSetting.memory_copy.notif_source_email ),
"To: %s" % recipient,
"Subject: Your PGP key expiration date is coming",
"Content-Type: text/plain; charset=ISO-8859-1",
"Content-Transfer-Encoding: 8bit", None,
message]
mail_content = collapse_mail_content(mail_building)
if not mail_content:
log.err("Unable to format (and then notify!) PGP key incoming expiration for %s" % recipient)
log.debug(mail_building)
return
sendmail(GLSetting.memory_copy.notif_username,
GLSetting.memory_copy.notif_password,
GLSetting.memory_copy.notif_username,
[ recipient ],
mail_content,
GLSetting.memory_copy.notif_server,
GLSetting.memory_copy.notif_port,
GLSetting.memory_copy.notif_security)
except Exception as excep:
log.err("Error in PGP key expiration check: %s (failure ignored)" % excep)
return
def send_anomaly_email(admin_email, message):
Alarm.last_alarm_email = datetime_now()
yield sendmail(authentication_username=GLSettings.memory_copy.notif_username,
authentication_password=GLSettings.memory_copy.notif_password,
from_address=GLSettings.memory_copy.notif_username,
to_address=admin_email,
message_file=message,
smtp_host=GLSettings.memory_copy.notif_server,
smtp_port=GLSettings.memory_copy.notif_port,
security=GLSettings.memory_copy.notif_security,
event=None)
def send_anomaly_email(admin_email, message):
Alarm.last_alarm_email = datetime_now()
yield sendmail(
authentication_username=GLSettings.memory_copy.notif_username,
authentication_password=GLSettings.memory_copy.notif_password,
from_address=GLSettings.memory_copy.notif_username,
to_address=admin_email,
message_file=message,
smtp_host=GLSettings.memory_copy.notif_server,
smtp_port=GLSettings.memory_copy.notif_port,
security=GLSettings.memory_copy.notif_security,
event=None)
def ping_mail_flush(self, notification_settings, receivers_syntesis):
"""
TODO This function should be implemented as a clean and testable plugin in the
way defined in plugin/base.py and plugin/notification.py, and/or is the opportunity
to review these classes, at the moment is a simplified version that just create a
ping email and send it via sendmail.
"""
for _, data in receivers_syntesis.iteritems():
receiver_dict, winks = data
receiver_name = receiver_dict['name']
receiver_email = receiver_dict['ping_mail_address']
fakeevent = OD()
fakeevent.type = u'ping_mail'
fakeevent.node_info = None
fakeevent.context_info = None
fakeevent.steps_info = None
fakeevent.receiver_info = receiver_dict
fakeevent.tip_info = None
fakeevent.subevent_info = {'counter': winks}
body = Templating().format_template(
notification_settings['ping_mail_template'], fakeevent)
title = Templating().format_template(
notification_settings['ping_mail_title'], fakeevent)
# so comfortable for a developer!! :)
source_mail_name = GLSetting.developer_name if GLSetting.devel_mode \
else GLSetting.memory_copy.notif_source_name
message = MIME_mail_build(source_mail_name,
GLSetting.memory_copy.notif_source_email,
receiver_name,
receiver_email,
title,
body)
fakeevent2 = OD()
fakeevent2.type = "Ping mail for %s (%d info)" % (receiver_email, winks)
return sendmail(authentication_username=GLSetting.memory_copy.notif_username,
authentication_password=GLSetting.memory_copy.notif_password,
from_address= GLSetting.memory_copy.notif_source_email,
to_address= [receiver_email],
message_file=message,
smtp_host=GLSetting.memory_copy.notif_server,
smtp_port=GLSetting.memory_copy.notif_port,
security=GLSetting.memory_copy.notif_security,
event=fakeevent2)
def ping_mail_flush(self, notification_settings, receivers_synthesis):
"""
TODO This function should be implemented as a clean and testable plugin in the
way defined in plugin/base.py and plugin/notification.py, and/or is the opportunity
to review these classes, at the moment is a simplified version that just create a
ping email and send it via sendmail.
"""
for _, data in receivers_synthesis.iteritems():
receiver_dict, winks = data
receiver_name = receiver_dict['name']
receiver_email = receiver_dict['ping_mail_address']
fakeevent = OD()
fakeevent.type = u'ping_mail'
fakeevent.node_info = None
fakeevent.context_info = None
fakeevent.receiver_info = receiver_dict
fakeevent.tip_info = None
fakeevent.subevent_info = {'counter': winks}
body = Templating().format_template(
notification_settings['ping_mail_template'], fakeevent)
title = Templating().format_template(
notification_settings['ping_mail_title'], fakeevent)
# so comfortable for a developer!! :)
source_mail_name = GLSettings.developer_name if GLSettings.devel_mode \
else GLSettings.memory_copy.notif_source_name
message = MIME_mail_build(source_mail_name,
GLSettings.memory_copy.notif_source_email,
receiver_name,
receiver_email,
title,
body)
fakeevent2 = OD()
fakeevent2.type = "Ping mail for %s (%d info)" % (receiver_email, winks)
return sendmail(authentication_username=GLSettings.memory_copy.notif_username,
authentication_password=GLSettings.memory_copy.notif_password,
from_address= GLSettings.memory_copy.notif_source_email,
to_address= [receiver_email],
message_file=message,
smtp_host=GLSettings.memory_copy.notif_server,
smtp_port=GLSettings.memory_copy.notif_port,
security=GLSettings.memory_copy.notif_security,
event=fakeevent2)
def sendmail(self, tid, to_address, subject, body):
if self.settings.testing:
# during unit testing do not try to send the mail
return defer.succeed(True)
return sendmail(tid, self.tenant_cache[tid].notification.smtp_username,
self.tenant_cache[tid].notification.smtp_password,
self.tenant_cache[tid].notification.smtp_server,
self.tenant_cache[tid].notification.smtp_port,
self.tenant_cache[tid].notification.smtp_security,
self.tenant_cache[tid].notification.smtp_source_name,
self.tenant_cache[tid].notification.smtp_source_email,
to_address, subject, body,
self.tenant_cache[tid].anonymize_outgoing_connections,
self.settings.socks_host, self.settings.socks_port)
def post(self):
user = yield get_user_settings(self.current_user.user_id,
State.tenant_cache[1].default_language)
language = user['language']
data = {
'type': 'admin_test',
'node': (yield admin_serialize_node(language)),
'notification': (yield get_notification(language)),
'user': user,
}
subject, body = Templating().get_mail_subject_and_body(data)
yield sendmail(user['mail_address'], subject, body)
def send_pgp_alerts(self, receiver_desc):
user_language = receiver_desc["language"]
node_desc = yield admin_serialize_node(user_language)
notification_settings = yield get_notification(user_language)
fakeevent = OD()
fakeevent.type = u"pgp_expiration_alert"
fakeevent.node_info = node_desc
fakeevent.context_info = None
fakeevent.receiver_info = receiver_desc
fakeevent.tip_info = None
fakeevent.subevent_info = None
subject = Templating().format_template(notification_settings["pgp_alert_mail_title"], fakeevent)
body = Templating().format_template(notification_settings["pgp_alert_mail_template"], fakeevent)
yield sendmail(receiver_desc["mail_address"], subject, body)
def mail_flush(from_address, to_address, message_file, event):
"""
This function just wrap the sendmail call, using the system memory variables.
"""
log.debug('Email: connecting to [%s:%d] to notify %s using [%s]' %
(GLSettings.memory_copy.notif_server,
GLSettings.memory_copy.notif_port,
to_address[0], GLSettings.memory_copy.notif_security))
return sendmail(authentication_username=GLSettings.memory_copy.notif_username,
authentication_password=GLSettings.memory_copy.notif_password,
from_address=from_address,
to_address=to_address,
message_file=message_file,
smtp_host=GLSettings.memory_copy.notif_server,
smtp_port=GLSettings.memory_copy.notif_port,
security=GLSettings.memory_copy.notif_security,
event=event)
def mail_flush(from_address, to_address, message_file, event):
"""
This function just wrap the sendmail call, using the system memory variables.
"""
log.debug('Email: connecting to [%s:%d] to notify %s using [%s]' %
(GLSetting.memory_copy.notif_server,
GLSetting.memory_copy.notif_port,
to_address[0], GLSetting.memory_copy.notif_security))
return sendmail(authentication_username=GLSetting.memory_copy.notif_username,
authentication_password=GLSetting.memory_copy.notif_password,
from_address= from_address,
to_address= to_address,
message_file=message_file,
smtp_host=GLSetting.memory_copy.notif_server,
smtp_port=GLSetting.memory_copy.notif_port,
security=GLSetting.memory_copy.notif_security,
event=event)
def post(self):
user = yield get_user_settings(self.current_user.user_id,
GLSettings.memory_copy.default_language)
language = user['language']
yield get_notification(language)
data = {}
data['type'] = 'admin_test_static'
data['node'] = yield admin_serialize_node(language)
data['notification'] = yield get_notification(language)
subject, body = Templating().get_mail_subject_and_body(data)
send_to = user['mail_address']
yield sendmail(send_to, subject, body)
def send_pgp_alerts(self, receiver_desc):
user_language = receiver_desc['language']
node_desc = yield admin_serialize_node(user_language)
notification_settings = yield get_notification(user_language)
fakeevent = OD()
fakeevent.type = u'pgp_expiration_alert'
fakeevent.node_info = node_desc
fakeevent.context_info = None
fakeevent.receiver_info = receiver_desc
fakeevent.tip_info = None
fakeevent.subevent_info = None
subject = Templating().format_template(
notification_settings['pgp_alert_mail_title'], fakeevent)
body = Templating().format_template(
notification_settings['pgp_alert_mail_template'], fakeevent)
yield sendmail(receiver_desc['mail_address'], subject, body)
def send_admin_pgp_alerts(self, admin_desc, expired_or_expiring):
user_language = admin_desc["language"]
node_desc = yield admin_serialize_node(user_language)
notification_settings = yield get_notification(user_language)
fakeevent = OD()
fakeevent.type = u"admin_pgp_expiration_alert"
fakeevent.node_info = node_desc
fakeevent.context_info = None
fakeevent.receiver_info = None
fakeevent.tip_info = None
fakeevent.subevent_info = {"expired_or_expiring": expired_or_expiring}
subject = Templating().format_template(notification_settings["admin_pgp_alert_mail_title"], fakeevent)
body = Templating().format_template(notification_settings["admin_pgp_alert_mail_template"], fakeevent)
admin_users = yield get_admin_users()
for u in admin_users:
yield sendmail(u["mail_address"], subject, body)
def ping_mail_flush(self, notification_settings, receivers_synthesis):
for _, data in receivers_synthesis.iteritems():
receiver_dict, winks = data
receiver_name = receiver_dict['name']
receiver_email = receiver_dict['ping_mail_address']
fakeevent = OD()
fakeevent.type = u'ping_mail'
fakeevent.node_info = None
fakeevent.context_info = None
fakeevent.receiver_info = receiver_dict
fakeevent.tip_info = None
fakeevent.subevent_info = {'counter': winks}
subject = Templating().format_template(notification_settings['ping_mail_template'], fakeevent)
body = Templating().format_template(notification_settings['ping_mail_title'], fakeevent)
return sendmail(receiver_email, subject, body)
def send_admin_pgp_alerts(self, admin_desc, expired_or_expiring):
user_language = admin_desc['language']
node_desc = yield admin_serialize_node(user_language)
notification_settings = yield get_notification(user_language)
fakeevent = OD()
fakeevent.type = u'admin_pgp_expiration_alert'
fakeevent.node_info = node_desc
fakeevent.context_info = None
fakeevent.receiver_info = None
fakeevent.tip_info = None
fakeevent.subevent_info = {'expired_or_expiring': expired_or_expiring}
subject = Templating().format_template(
notification_settings['admin_pgp_alert_mail_title'], fakeevent)
body = Templating().format_template(
notification_settings['admin_pgp_alert_mail_template'], fakeevent)
admin_users = yield get_admin_users()
for u in admin_users:
yield sendmail(u['mail_address'], subject, body)
def operation(self):
@transact
def delete_sent_mail(store, mail_id):
store.find(models.Mail, models.Mail.id == mail_id).remove()
@inlineCallbacks
def _success_callback(result, mail_id):
yield delete_sent_mail(mail_id)
def _failure_callback(failure, mail_id):
pass
mail_generator = MailGenerator()
for trigger in ['ReceiverTip', 'Comment', 'Message', 'ReceiverFile']:
yield mail_generator.process_data(trigger)
mails = yield self.get_mails_from_the_pool()
for mail in mails:
sendmail_deferred = sendmail(mail['address'], mail['subject'], mail['body'])
sendmail_deferred.addCallbacks(_success_callback, _failure_callback,
callbackArgs=(mail['id'],), errbackArgs=(mail['id'],))
yield sendmail_deferred
def post(self):
"""
Parameters: None
Response: None
"""
user = yield get_user_settings(self.current_user.user_id,
GLSettings.memory_copy.default_language)
notif = yield get_notification(user['language'])
send_to = user['mail_address']
# Get the test emails subject line and body internationalized from notif
subject = notif['admin_test_static_mail_title']
msg = notif['admin_test_static_mail_template']
log.debug("Attempting to send test email to: %s" % send_to)
# If sending the email fails the exception mail address will be mailed.
# If the failure is due to a bad SMTP config that will fail too, but it
# doesn't hurt to try!
try:
yield sendmail(send_to, subject, msg)
except Exception as e:
log.debug("Sending to admin failed. Trying an exception mail")
raise e
def post(self):
"""
Parameters: None
Response: None
"""
user = yield get_user_settings(self.current_user.user_id,
GLSettings.memory_copy.default_language)
notif = yield get_notification(user['language'])
send_to = user['mail_address']
# Get the test emails subject line and body internationalized from notif
subject = notif['admin_test_static_mail_title']
msg = notif['admin_test_static_mail_template']
log.debug("Attempting to send test email to: %s" % send_to)
# If sending the email fails the exception mail address will be mailed.
# If the failure is due to a bad SMTP config that will fail too, but it
# doesn't hurt to try!
try:
yield sendmail(send_to, subject, msg)
except Exception as e:
log.debug("Sending to admin failed. Trying an exception mail")
raise e
def admin_alarm_notification(event_matrix):
"""
This function put a mail in queue for the Admin, if the
configured threshold has been reached for Alarm notification.
TODO put a GLSetting + Admin configuration variable,
now is hardcoded to notice at >= 1
"""
@transact_ro
def _get_admin_email(store):
node = store.find(models.Node).one()
return node.email
@transact_ro
def _get_message_template(store):
admin_user = store.find(models.User, models.User.username == u'admin').one()
notif = store.find(models.Notification).one()
template = notif.admin_anomaly_template
if admin_user.language in template:
return template[admin_user.language]
elif GLSetting.memory_copy.default_language in template:
return template[GLSetting.memory_copy.default_language]
else:
raise Exception("Cannot find any language for admin notification")
def _aal():
return "%s" % Alarm.stress_levels['activity']
def _ad():
retstr = ""
for event, amount in event_matrix.iteritems():
retstr = "%s: %d\n%s" % (event, amount, retstr)
return retstr
def _dal():
return "%s" % Alarm.stress_levels['disk_space']
def _dd():
return "%s Megabytes" % Alarm.latest_measured_freespace
message_required = False
if Alarm.stress_levels['activity'] >= 1:
message_required = True
if Alarm.stress_levels['disk_space'] >= 1:
message_required = True
if not message_required:
# luckly, no mail needed
return
KeyWordTemplate = {
"%ActivityAlarmLevel%" : _aal,
"%ActivityDump%" : _ad,
"%DiskAlarmLevel%" : _dal,
"%DiskDump%" : _dd,
}
message = yield _get_message_template()
for keyword, function in KeyWordTemplate.iteritems():
where = message.find(keyword)
message = "%s%s%s" % (
message[:where],
function(),
message[where + len(keyword):])
if Alarm.last_alarm_email:
if not is_expired(Alarm.last_alarm_email, minutes=10):
log.debug("Alert email want be send, but the threshold of 10 minutes is not yet reached since %s" %
datetime_to_ISO8601(Alarm.last_alarm_email))
return
to_address = yield _get_admin_email()
message = MIME_mail_build(GLSetting.memory_copy.notif_source_name,
GLSetting.memory_copy.notif_source_email,
"Tester",
to_address,
"ALERT: Anomaly detection",
message)
log.debug('Alarm Email for admin: connecting to [%s:%d]' %
(GLSetting.memory_copy.notif_server,
GLSetting.memory_copy.notif_port) )
Alarm.last_alarm_email = datetime_now()
yield sendmail(authentication_username=GLSetting.memory_copy.notif_username,
authentication_password=GLSetting.memory_copy.notif_password,
from_address=GLSetting.memory_copy.notif_source_email,
to_address=to_address,
message_file=message,
smtp_host=GLSetting.memory_copy.notif_server,
smtp_port=GLSetting.memory_copy.notif_port,
security=GLSetting.memory_copy.notif_security,
event=None)
def sendmail(self, mail):
d = sendmail(mail['address'], mail['subject'], mail['body'])
d.addCallback(delete_sent_mail, mail['id'])
return d
def compute_activity_level():
"""
This function is called by the scheduled task, to update the
Alarm level.
At the end of the execution, reset to 0 the counters,
this is why the content are copied for the statistic
acquiring later.
"""
Alarm.number_of_anomalies = 0
current_event_matrix = {}
requests_timing = []
for _, event_obj in event.EventTrackQueue.queue.iteritems():
current_event_matrix.setdefault(event_obj.event_type, 0)
current_event_matrix[event_obj.event_type] += 1
requests_timing.append(event_obj.request_time)
if len(requests_timing) > 2:
log.info("In latest %d seconds: worst RTT %f, best %f" %
(GLSettings.anomaly_delta,
round(max(requests_timing), 2),
round(min(requests_timing), 2)))
for event_name, threshold in Alarm.OUTCOMING_ANOMALY_MAP.iteritems():
if event_name in current_event_matrix:
if current_event_matrix[event_name] > threshold:
Alarm.number_of_anomalies += 1
else:
log.debug("[compute_activity_level] %s %d < %d: it's OK (Anomalies recorded so far %d)" %
(event_name,
current_event_matrix[event_name],
threshold, Alarm.number_of_anomalies))
previous_activity_sl = Alarm.stress_levels['activity']
# Behavior: once the activity has reach a peek, the stress level
# is raised at RED (two), and then is decremented at YELLOW (one) in the
# next evaluation.
if Alarm.number_of_anomalies >= 2:
report_function = log.msg
Alarm.stress_levels['activity'] = 2
elif Alarm.number_of_anomalies == 1:
report_function = log.info
Alarm.stress_levels['activity'] = 1
else:
report_function = log.debug
Alarm.stress_levels['activity'] = 0
# slow downgrade, if something has triggered a two, next step to 1
if previous_activity_sl == 2 and not Alarm.stress_levels['activity']:
Alarm.stress_levels['activity'] = 1
# if there are some anomaly or we're nearby, record it.
if Alarm.number_of_anomalies >= 1 or Alarm.stress_levels['activity'] >= 1:
update_AnomalyQ(current_event_matrix,
Alarm.stress_levels['activity'])
if previous_activity_sl or Alarm.stress_levels['activity']:
report_function("in Activity stress level switch from %d => %d" %
(previous_activity_sl,
Alarm.stress_levels['activity']))
mailinfos = yield Alarm.admin_alarm_generate_mail(current_event_matrix)
for mailinfo in mailinfos:
Alarm.last_alarm_email = datetime_now()
yield sendmail(mailinfo['mail_address'], mailinfo['subject'], mailinfo['body'])
defer.returnValue(Alarm.stress_levels['activity'] - previous_activity_sl)
def compute_activity_level():
"""
This function is called by the scheduled task, to update the
Alarm level.
At the end of the execution, reset to 0 the counters,
this is why the content are copied for the statistic
acquiring later.
"""
Alarm.number_of_anomalies = 0
current_event_matrix = {}
requests_timing = []
for _, event_obj in event.EventTrackQueue.queue.iteritems():
current_event_matrix.setdefault(event_obj.event_type, 0)
current_event_matrix[event_obj.event_type] += 1
requests_timing.append(event_obj.request_time)
if len(requests_timing) > 2:
log.info("In latest %d seconds: worst RTT %f, best %f" %
(GLSettings.anomaly_delta, round(
max(requests_timing), 2), round(min(requests_timing), 2)))
for event_name, threshold in Alarm.OUTCOMING_ANOMALY_MAP.iteritems():
if event_name in current_event_matrix:
if current_event_matrix[event_name] > threshold:
Alarm.number_of_anomalies += 1
else:
log.debug(
"[compute_activity_level] %s %d < %d: it's OK (Anomalies recorded so far %d)"
% (event_name, current_event_matrix[event_name], threshold,
Alarm.number_of_anomalies))
previous_activity_sl = Alarm.stress_levels['activity']
# Behavior: once the activity has reach a peek, the stress level
# is raised at RED (two), and then is decremented at YELLOW (one) in the
# next evaluation.
if Alarm.number_of_anomalies >= 2:
report_function = log.msg
Alarm.stress_levels['activity'] = 2
elif Alarm.number_of_anomalies == 1:
report_function = log.info
Alarm.stress_levels['activity'] = 1
else:
report_function = log.debug
Alarm.stress_levels['activity'] = 0
# slow downgrade, if something has triggered a two, next step to 1
if previous_activity_sl == 2 and not Alarm.stress_levels['activity']:
Alarm.stress_levels['activity'] = 1
# if there are some anomaly or we're nearby, record it.
if Alarm.number_of_anomalies >= 1 or Alarm.stress_levels['activity'] >= 1:
update_AnomalyQ(current_event_matrix, Alarm.stress_levels['activity'])
if previous_activity_sl or Alarm.stress_levels['activity']:
report_function(
"in Activity stress level switch from %d => %d" %
(previous_activity_sl, Alarm.stress_levels['activity']))
mailinfos = yield Alarm.admin_alarm_generate_mail(current_event_matrix)
for mailinfo in mailinfos:
Alarm.last_alarm_email = datetime_now()
yield sendmail(mailinfo['mail_address'], mailinfo['subject'],
mailinfo['body'])
defer.returnValue(Alarm.stress_levels['activity'] - previous_activity_sl)
def sendmail(self, mail):
success = yield sendmail(mail['address'], mail['subject'],
mail['body'])
if success:
self.mails_to_delete.append(mail['id'])
def admin_alarm_notification(event_matrix):
"""
This function put a mail in queue for the Admin, if the
configured threshold has been reached for Alarm notification.
TODO put a GLSetting + Admin configuration variable,
now is hardcoded to notice at >= 1
"""
@transact_ro
def _get_node_name(store):
node = store.find(models.Node).one()
return node.email
@transact_ro
def _get_admin_email(store):
node = store.find(models.Node).one()
return node.email
node_name = yield _get_node_name()
admin_email = yield _get_admin_email()
@transact_ro
def _get_message_template(store):
admin_user = store.find(models.User,
models.User.username == u'admin').one()
notif = store.find(models.Notification).one()
template = notif.admin_anomaly_template
if admin_user.language in template:
return template[admin_user.language]
elif GLSetting.memory_copy.language in template:
return template[GLSetting.memory_copy.language]
else:
raise Exception(
"Cannot find any language for admin notification")
def _aal():
return "%s" % Alarm.stress_levels['activity']
def _ad():
retstr = ""
for event, amount in event_matrix.iteritems():
retstr = "%s: %d\n%s" % (event, amount, retstr)
return retstr
def _dal():
return "%s" % Alarm.stress_levels['disk_space']
def _dd():
return "%s" % bytes_to_pretty_str(Alarm.latest_measured_freespace)
def _nn():
return "%s" % node_name
message_required = False
if Alarm.stress_levels['activity'] >= 1:
message_required = True
if Alarm.stress_levels['disk_space'] >= 1:
message_required = True
if not message_required:
# luckly, no mail needed
return
KeyWordTemplate = {
"%ActivityAlarmLevel%": _aal,
"%ActivityDump%": _ad,
"%DiskAlarmLevel%": _dal,
"%DiskDump%": _dd,
"%NodeSignature%": _nn
}
message = yield _get_message_template()
for keyword, function in KeyWordTemplate.iteritems():
where = message.find(keyword)
message = "%s%s%s" % (message[:where], function(),
message[where + len(keyword):])
if Alarm.last_alarm_email:
if not is_expired(Alarm.last_alarm_email, minutes=10):
log.debug(
"Alert email want be send, but the threshold of 10 minutes is not yet reached since %s"
% datetime_to_ISO8601(Alarm.last_alarm_email))
return
message = MIME_mail_build(GLSetting.memory_copy.notif_source_name,
GLSetting.memory_copy.notif_source_email,
"Admin", admin_email,
"ALERT: Anomaly detection", message)
log.debug('Alarm Email for admin: connecting to [%s:%d]' %
(GLSetting.memory_copy.notif_server,
GLSetting.memory_copy.notif_port))
Alarm.last_alarm_email = datetime_now()
yield sendmail(
authentication_username=GLSetting.memory_copy.notif_username,
authentication_password=GLSetting.memory_copy.notif_password,
from_address=GLSetting.memory_copy.notif_source_email,
to_address=admin_email,
message_file=message,
smtp_host=GLSetting.memory_copy.notif_server,
smtp_port=GLSetting.memory_copy.notif_port,
security=GLSetting.memory_copy.notif_security,
event=None)
def sendmail(self, mail):
d = sendmail(mail['address'], mail['subject'], mail['body'])
d.addCallback(delete_sent_mail, mail['id'])
return d