def distributed_send_message(message):
if rpc.num_slaves and rpc.sender_slaves:
for i, address in enumerate(rpc.sender_slaves):
if i >= rpc.num_slaves:
logger.error(
'Failed using all configured slaves; resorting to local send_message'
)
break
if rpc.send_message_to_slave(message, address):
return True
logger.info('Sending message (ID %s) locally',
message.get('message_id', '?'))
runtime = send_message(message)
add_mode_stat(message['mode'], runtime)
metrics_key = 'app_%(application)s_mode_%(mode)s_cnt' % message
metrics.add_new_metrics({metrics_key: 0})
metrics.incr(metrics_key)
if runtime is not None:
return True
raise Exception('Failed sending message')
def distributed_send_message(message):
# If I am the master, attempt sending my messages through my slaves.
if coordinator.am_i_master():
try:
if coordinator.slave_count and coordinator.slaves:
for i, address in enumerate(coordinator.slaves):
if i >= coordinator.slave_count:
logger.error('Failed using all configured slaves; resorting to local send_message')
break
if rpc.send_message_to_slave(message, address):
return True
except StopIteration:
logger.warning('No more slaves. Sending locally.')
logger.info('Sending message (ID %s) locally', message.get('message_id', '?'))
runtime = send_message(message)
add_mode_stat(message['mode'], runtime)
metrics_key = 'app_%(application)s_mode_%(mode)s_cnt' % message
metrics.add_new_metrics({metrics_key: 0})
metrics.incr(metrics_key)
if runtime is not None:
return True
raise Exception('Failed sending message')
def fetch_and_send_message():
message = send_queue.get()
has_contact = set_target_contact(message)
if not has_contact:
mark_message_has_no_contact(message)
return
if 'message_id' not in message:
message['message_id'] = None
drop_mode_id = api_cache.modes.get('drop')
# If this app breaches hard quota, drop message on floor, and update in UI if it has an ID
if not quota.allow_send(message):
logger.warn(
'Hard message quota exceeded; Dropping this message on floor: %s',
message)
if message['message_id']:
spawn(auditlog.message_change, message['message_id'],
auditlog.MODE_CHANGE, message.get('mode', '?'), 'drop',
'Dropping due to hard quota violation.')
# If we know the ID for the mode drop, reflect that for the message
if drop_mode_id:
message['mode'] = 'drop'
message['mode_id'] = drop_mode_id
else:
logger.error(
'Can\'t mark message %s as dropped as we don\'t know the mode ID for %s',
message, 'drop')
# Render, so we're able to populate the message table with the proper subject/etc as well as
# information that it was dropped.
render(message)
mark_message_as_sent(message)
return
# If we're set to drop this message, no-op this before message gets sent to a vendor
if message.get('mode') == 'drop':
if message['message_id']:
render(message)
mark_message_as_sent(message)
add_mode_stat('drop', 0)
metrics_key = 'app_%(application)s_mode_drop_cnt' % message
metrics.add_new_metrics({metrics_key: 0})
metrics.incr(metrics_key)
return
render(message)
# Drop this message, and mark it as dropped, rather than sending it, if its body is too long and we were normally
# going to send it anyway.
body_length = len(message['body'])
if body_length > MAX_MESSAGE_BODY_LENGTH:
logger.warn(
'Message id %s has a ridiculously long body (%s chars). Dropping it.',
message['message_id'], body_length)
spawn(
auditlog.message_change, message['message_id'],
auditlog.MODE_CHANGE, message.get('mode', '?'), 'drop',
'Dropping due to excessive body length (%s > %s chars)' %
(body_length, MAX_MESSAGE_BODY_LENGTH))
metrics.incr('msg_drop_length_cnt')
# Truncate this here to avoid a duplicate log message in mark_message_as_sent(), as we still need to call
# that to update the body/subject
message['body'] = message['body'][:MAX_MESSAGE_BODY_LENGTH]
if drop_mode_id:
message['mode'] = 'drop'
message['mode_id'] = drop_mode_id
mark_message_as_sent(message)
return
success = None
try:
success = distributed_send_message(message)
except Exception:
logger.exception('Failed to send message: %s', message)
if message['mode'] == 'email':
logger.error(
'unable to send %(mode)s %(message_id)s %(application)s %(destination)s %(subject)s %(body)s',
message)
metrics.incr('task_failure')
else:
logger.error(
'reclassifying as email %(mode)s %(message_id)s %(application)s %(destination)s %(subject)s %(body)s',
message)
old_mode = message['mode']
if (set_target_fallback_mode(message)):
update_message_mode(message)
auditlog.message_change(
message['message_id'], auditlog.MODE_CHANGE, old_mode,
message['mode'],
'Changing mode due to original mode failure')
render(message)
try:
success = distributed_send_message(message)
# nope - log and bail
except Exception:
metrics.incr('task_failure')
logger.error(
'unable to send %(mode)s %(message_id)s %(application)s %(destination)s %(subject)s %(body)s',
message)
if success:
metrics.incr('message_send_cnt')
if message['message_id']:
mark_message_as_sent(message)
if message['message_id']:
update_message_sent_status(message, success)