def test_sanitize_unicode_dict():
import pytest
from jinja2.sandbox import SandboxedEnvironment
from iris.utils import sanitize_unicode_dict
# Use jinja the same way as in sender
env = SandboxedEnvironment(autoescape=False)
template = env.from_string('{{var}} {{var2}} {{nested.nest}}')
bad_context = {'var': '\xe2\x80\x99', 'var2': 2, 'nested': {'nest': '\xe2\x80\x99'}}
with pytest.raises(UnicodeDecodeError):
template.render(**bad_context)
template.render(**sanitize_unicode_dict(bad_context))
def test_sanitize_unicode_dict():
from jinja2.sandbox import SandboxedEnvironment
from iris.utils import sanitize_unicode_dict
# Use jinja the same way as in sender
env = SandboxedEnvironment(autoescape=False)
template = env.from_string('{{var}} {{var2}} {{nested.nest}}')
bad_context = {'var': b'\xe2\x80\x99', 'var2': 2, 'nested': {'nest': b'\xe2\x80\x99'}}
assert '\\xe2' in template.render(**bad_context)
good_render = template.render(**sanitize_unicode_dict(bad_context))
assert '\\xe2' not in good_render
assert b'\xe2\x80\x99'.decode('utf-8') in good_render
def fetch_and_send_message():
message = send_queue.get()
sanitize_unicode_dict(message)
has_contact = set_target_contact(message)
if not has_contact:
mark_message_has_no_contact(message)
metrics.incr('task_failure')
logger.error('Failed to send message, no contact found: %s', message)
return
if 'message_id' not in message:
message['message_id'] = None
drop_mode_id = api_cache.modes.get('drop')
# If this app breaches hard quota, drop message on floor, and update in UI if it has an ID
if not quota.allow_send(message):
logger.warn('Hard message quota exceeded; Dropping this message on floor: %s', message)
if message['message_id']:
spawn(auditlog.message_change,
message['message_id'], auditlog.MODE_CHANGE, message.get('mode', '?'), 'drop',
'Dropping due to hard quota violation.')
# If we know the ID for the mode drop, reflect that for the message
if drop_mode_id:
message['mode'] = 'drop'
message['mode_id'] = drop_mode_id
else:
logger.error('Can\'t mark message %s as dropped as we don\'t know the mode ID for %s',
message, 'drop')
# Render, so we're able to populate the message table with the
# proper subject/etc as well as information that it was dropped.
render(message)
mark_message_as_sent(message)
return
# If we're set to drop this message, no-op this before message gets sent to a vendor
if message.get('mode') == 'drop':
if message['message_id']:
render(message)
mark_message_as_sent(message)
add_mode_stat('drop', 0)
metrics_key = 'app_%(application)s_mode_drop_cnt' % message
metrics.add_new_metrics({metrics_key: 0})
metrics.incr(metrics_key)
return
render(message)
if message.get('body') is None:
message['body'] = ''
# Drop this message, and mark it as dropped, rather than sending it, if its
# body is too long and we were normally going to send it anyway.
body_length = len(message['body'])
if body_length > MAX_MESSAGE_BODY_LENGTH:
logger.warn('Message id %s has a ridiculously long body (%s chars). Dropping it.',
message['message_id'], body_length)
spawn(auditlog.message_change,
message['message_id'], auditlog.MODE_CHANGE, message.get('mode', '?'), 'drop',
'Dropping due to excessive body length (%s > %s chars)' % (
body_length, MAX_MESSAGE_BODY_LENGTH))
metrics.incr('msg_drop_length_cnt')
# Truncate this here to avoid a duplicate log message in
# mark_message_as_sent(), as we still need to call that to update the body/subject
message['body'] = message['body'][:MAX_MESSAGE_BODY_LENGTH]
if drop_mode_id:
message['mode'] = 'drop'
message['mode_id'] = drop_mode_id
mark_message_as_sent(message)
return
success = False
try:
success = distributed_send_message(message)
except Exception:
logger.exception('Failed to send message: %s', message)
if message['mode'] == 'email':
metrics.incr('task_failure')
logger.error('unable to send %(mode)s %(message_id)s %(application)s %(destination)s %(subject)s %(body)s', message)
else:
logger.error('reclassifying as email %(mode)s %(message_id)s %(application)s %(destination)s %(subject)s %(body)s', message)
set_target_fallback_mode(message)
render(message)
try:
success = distributed_send_message(message)
# nope - log and bail
except Exception:
metrics.incr('task_failure')
logger.error('unable to send %(mode)s %(message_id)s %(application)s %(destination)s %(subject)s %(body)s', message)
if success:
metrics.incr('message_send_cnt')
if message['message_id']:
mark_message_as_sent(message)
if message['message_id']:
update_message_sent_status(message, success)