def set_target_fallback_mode(message):
try:
connection = db.engine.raw_connection()
cursor = connection.cursor()
cursor.execute('''SELECT `destination`, `mode`.`name`, `mode`.`id`
FROM `target`
JOIN `target_contact` ON `target_contact`.`target_id` = `target`.`id`
JOIN `mode` ON `mode`.`id` = `target_contact`.`mode_id`
WHERE `target`.`name` = %s AND `mode`.`name` = %s''',
(message['target'], target_fallback_mode))
[(destination, mode, mode_id)] = cursor
cursor.close()
connection.close()
old_mode = message.get('mode', '')
message['destination'] = destination
message['mode'] = mode
message['mode_id'] = mode_id
update_message_mode(message)
message_id = message.get('message_id')
if message_id:
auditlog.message_change(
message_id, auditlog.MODE_CHANGE, old_mode, message['mode'],
'Changing mode due to original mode failure')
return True
# target doesn't have email either - bail
except ValueError:
logger.exception('target does not have mode(%s) %r', target_fallback_mode, message)
message['destination'] = message['mode'] = message['mode_id'] = None
return False
def mark_message_has_no_contact(message):
message_id = message.get('message_id')
if not message_id:
logger.warn('Cannot mark message "%s" as not having contact as message_id is missing', message)
return
connection = db.engine.raw_connection()
cursor = connection.cursor()
cursor.execute('UPDATE `message` set `active`=0 WHERE `id`=%s',
message_id)
connection.commit()
cursor.close()
connection.close()
auditlog.message_change(
message_id, auditlog.MODE_CHANGE, target_fallback_mode, 'invalid',
'Ignore message as we failed to resolve target contact')
def fetch_and_send_message():
message = send_queue.get()
has_contact = set_target_contact(message)
if not has_contact:
mark_message_has_no_contact(message)
return
if 'message_id' not in message:
message['message_id'] = None
drop_mode_id = api_cache.modes.get('drop')
# If this app breaches hard quota, drop message on floor, and update in UI if it has an ID
if not quota.allow_send(message):
logger.warn(
'Hard message quota exceeded; Dropping this message on floor: %s',
message)
if message['message_id']:
spawn(auditlog.message_change, message['message_id'],
auditlog.MODE_CHANGE, message.get('mode', '?'), 'drop',
'Dropping due to hard quota violation.')
# If we know the ID for the mode drop, reflect that for the message
if drop_mode_id:
message['mode'] = 'drop'
message['mode_id'] = drop_mode_id
else:
logger.error(
'Can\'t mark message %s as dropped as we don\'t know the mode ID for %s',
message, 'drop')
# Render, so we're able to populate the message table with the proper subject/etc as well as
# information that it was dropped.
render(message)
mark_message_as_sent(message)
return
# If we're set to drop this message, no-op this before message gets sent to a vendor
if message.get('mode') == 'drop':
if message['message_id']:
render(message)
mark_message_as_sent(message)
add_mode_stat('drop', 0)
metrics_key = 'app_%(application)s_mode_drop_cnt' % message
metrics.add_new_metrics({metrics_key: 0})
metrics.incr(metrics_key)
return
render(message)
# Drop this message, and mark it as dropped, rather than sending it, if its body is too long and we were normally
# going to send it anyway.
body_length = len(message['body'])
if body_length > MAX_MESSAGE_BODY_LENGTH:
logger.warn(
'Message id %s has a ridiculously long body (%s chars). Dropping it.',
message['message_id'], body_length)
spawn(
auditlog.message_change, message['message_id'],
auditlog.MODE_CHANGE, message.get('mode', '?'), 'drop',
'Dropping due to excessive body length (%s > %s chars)' %
(body_length, MAX_MESSAGE_BODY_LENGTH))
metrics.incr('msg_drop_length_cnt')
# Truncate this here to avoid a duplicate log message in mark_message_as_sent(), as we still need to call
# that to update the body/subject
message['body'] = message['body'][:MAX_MESSAGE_BODY_LENGTH]
if drop_mode_id:
message['mode'] = 'drop'
message['mode_id'] = drop_mode_id
mark_message_as_sent(message)
return
success = None
try:
success = distributed_send_message(message)
except Exception:
logger.exception('Failed to send message: %s', message)
if message['mode'] == 'email':
logger.error(
'unable to send %(mode)s %(message_id)s %(application)s %(destination)s %(subject)s %(body)s',
message)
metrics.incr('task_failure')
else:
logger.error(
'reclassifying as email %(mode)s %(message_id)s %(application)s %(destination)s %(subject)s %(body)s',
message)
old_mode = message['mode']
if (set_target_fallback_mode(message)):
update_message_mode(message)
auditlog.message_change(
message['message_id'], auditlog.MODE_CHANGE, old_mode,
message['mode'],
'Changing mode due to original mode failure')
render(message)
try:
success = distributed_send_message(message)
# nope - log and bail
except Exception:
metrics.incr('task_failure')
logger.error(
'unable to send %(mode)s %(message_id)s %(application)s %(destination)s %(subject)s %(body)s',
message)
if success:
metrics.incr('message_send_cnt')
if message['message_id']:
mark_message_as_sent(message)
if message['message_id']:
update_message_sent_status(message, success)
def create_messages(incident_id, plan_notification_id):
application_id = cache.incidents[incident_id]['application_id']
plan_notification = cache.plan_notifications[plan_notification_id]
role = cache.roles[plan_notification['role_id']]['name']
target = cache.targets[plan_notification['target_id']]['name']
# find role/priority from plan_notification_id
names = cache.targets_for_role(role, target)
priority_id = plan_notification['priority_id']
changed_target = False
body = ''
if not names:
metrics.incr('role_target_lookup_error')
# Try to get creator of the plan and nag them instead
name = None
try:
name = cache.plans[plan_notification['plan_id']]['creator']
except (KeyError, TypeError):
pass
if not name:
logger.error((
'Failed to find targets for incident %s, plan_notification_id: %s, role: %s, target: %s, result: %s and failed looking '
'up the plan\'s creator'), incident_id, plan_notification_id,
role, target, names)
return False
try:
priority_id = api_cache.priorities['low']['id']
except KeyError:
logger.error((
'Failed to find targets for incident %s, plan_notification_id: %s, role: %s, target: %s, result: %s and failed looking '
'up ID for low priority'), incident_id, plan_notification_id,
role, target, names)
return False
logger.error((
'Failed to find targets for incident %s, plan_notification_id: %s, role: %s, target: %s, result: %s. '
'Reaching out to %s instead and lowering priority to low (%s)'),
incident_id, plan_notification_id, role, target, names,
name, priority_id)
body = 'You are receiving this as you created this plan and we can\'t resolve %s of %s at this time.\n\n' % (
role, target)
names = [name]
changed_target = True
connection = db.engine.raw_connection()
cursor = connection.cursor()
for name in names:
t = cache.target_names[name]
if t:
target_id = t['id']
cursor.execute(
INSERT_MESSAGE_SQL,
(plan_notification['plan_id'], plan_notification_id,
incident_id, application_id, target_id, priority_id, body))
if changed_target:
connection.commit(
) # needed for the lastrowid to exist in the DB to satsify the constraint
auditlog.message_change(
cursor.lastrowid, auditlog.TARGET_CHANGE,
role + '|' + target, name,
'Changing target as we failed resolving original target')
else:
metrics.incr('target_not_found')
logger.error('No target found: %s', name)
connection.commit()
cursor.close()
connection.close()
return True