def _serialize_check_and_chunk_message(
self,
protocol_version, # type: ProtocolVersion
message, # type: Dict[six.text_type, Any]
serializer, # type: Serializer
):
# type: (...) -> List[six.binary_type]
with self._get_timer('send.serialize'):
serialized_message = serializer.dict_to_blob(message)
message_size_in_bytes = len(serialized_message)
self._get_histogram('send.message_size').set(message_size_in_bytes)
if message_size_in_bytes > self.maximum_message_size_in_bytes:
self._get_counter('send.error.message_too_large').increment()
raise MessageTooLarge(message_size_in_bytes, 'Message exceeds maximum message size')
if self.log_messages_larger_than_bytes and message_size_in_bytes > self.log_messages_larger_than_bytes:
_oversized_message_logger.warning(
'Oversized message sent for PySOA service {}'.format(self.service_name),
extra={'data': {
'message': RecursivelyCensoredDictWrapper(message),
'serialized_length_in_bytes': message_size_in_bytes,
'threshold': self.log_messages_larger_than_bytes,
}},
)
content_type_header = 'content-type:{};'.format(serializer.mime_type).encode('utf-8')
if self.is_server and 0 < self.chunk_messages_larger_than_bytes < message_size_in_bytes:
# chunking is enabled on the server and the message is big enough to chunk
if not ProtocolFeature.CHUNKED_RESPONSES.supported_in(protocol_version):
self._get_counter('send.error.message_too_large').increment()
raise MessageTooLarge(
message_size_in_bytes,
'Message exceeds chunking threshold but client does not support chunking',
)
chunk_count = int(math.ceil(message_size_in_bytes / self.chunk_messages_larger_than_bytes))
self._get_histogram('send.chunk_count').set(chunk_count)
headers = protocol_version.prefix + content_type_header + (b'chunk-count:%d;' % (chunk_count, ))
return [
headers + (b'chunk-id:%d;' % (i + 1, )) + serialized_message[
i * self.chunk_messages_larger_than_bytes:
(i + 1) * self.chunk_messages_larger_than_bytes
]
for i in range(chunk_count)
]
if ProtocolFeature.CONTENT_TYPE_HEADER.supported_in(protocol_version):
serialized_message = content_type_header + serialized_message
if ProtocolFeature.VERSION_MARKER.supported_in(protocol_version):
serialized_message = protocol_version.prefix + serialized_message
return [serialized_message]
def test_simple_dict(self):
original = {
'hello': 'world',
'password': '******',
'credit_card': '1234567890123456',
'passphrase': True,
'cvv': 938,
}
wrapped = RecursivelyCensoredDictWrapper(original)
expected = {
'hello': 'world',
'password': '******',
'credit_card': '**********',
'passphrase': True,
'cvv': '**********',
}
self.assertEqual(expected, eval(repr(wrapped)))
self.assertEqual(repr(wrapped), str(wrapped))
if six.PY2:
self.assertEqual(six.text_type(repr(wrapped)),
six.text_type(wrapped))
else:
self.assertEqual(six.binary_type(repr(wrapped), 'utf-8'),
six.binary_type(wrapped))
# Make sure the original dict wasn't modified
self.assertEqual(
{
'hello': 'world',
'password': '******',
'credit_card': '1234567890123456',
'passphrase': True,
'cvv': 938,
},
original,
)
def send_message(self, request_id, meta, body, message_expiry_in_seconds=None):
protocol_key = meta.get('protocol_key')
stream_id = meta.get('stream_id')
if request_id is None:
raise InvalidMessageError('No request ID')
if message_expiry_in_seconds:
message_expiry = time.time() + message_expiry_in_seconds
else:
message_expiry = time.time() + self.message_expiry_in_seconds
meta['__expiry__'] = message_expiry
message = {'request_id': request_id, 'meta': meta, 'body': body}
with self._get_timer('send.serialize'):
serializer = self.default_serializer
if 'serializer' in meta:
# TODO: Breaking change: Assume a MIME type is always specified. This should not be done until all
# TODO servers and clients have Step 2 code. This will be a Step 3 breaking change.
serializer = meta.pop('serializer')
serialized_message = (
'content-type:{};'.format(serializer.mime_type).encode('utf-8') + serializer.dict_to_blob(message)
)
message_size_in_bytes = len(serialized_message)
response_headers = [
(':status', '200'),
('content-type', 'application/json'),
('content-length', str(message_size_in_bytes)),
('server', 'pysoa-h2'),
]
if self.log_messages_larger_than_bytes and message_size_in_bytes > self.log_messages_larger_than_bytes:
_oversized_message_logger.warning(
'Oversized message sent for PySOA service {}'.format(self.service_name),
extra={'data': {
'message': RecursivelyCensoredDictWrapper(message),
'serialized_length_in_bytes': message_size_in_bytes,
'threshold': self.log_messages_larger_than_bytes,
}},
)
for i in range(-1, self.queue_full_retries):
if i >= 0:
time.sleep((2 ** i + random.random()) / self.EXPONENTIAL_BACK_OFF_FACTOR)
self._get_counter('send.responses_queue_full_retry').increment()
self._get_counter('send.responses_queue_full_retry.retry_{}'.format(i + 1)).increment()
try:
with self._get_timer('send.send_message_response_http2_queue'):
self.responses_queue.put((
protocol_key,
stream_id,
request_id,
serialized_message,
response_headers,
), timeout=0)
return
except six.moves.queue.Full:
continue
except Exception as e:
self._get_counter('send.error.unknown').increment()
raise MessageSendError(
'Unknown error sending message for service {}'.format(self.service_name),
six.text_type(type(e).__name__),
*e.args
)
self._get_counter('send.error.responses_queue_full').increment()
raise MessageSendError(
'Http2 responses queue was full after {retries} retries'.format(
retries=self.queue_full_retries,
)
)
def test_complex_dict(self):
original = {
'a_list': [
'a',
True,
109.8277,
{
'username': '******',
'passphrase': 'this should be censored'
},
{
'username': '******',
'passphrase': ''
},
],
'a_set': {
'b',
False,
18273,
},
'a_tuple': (
'c',
True,
42,
{
'cc_number': '9876543210987654',
'cvv': '987',
'expiration': '12-20',
'pin': '4096'
},
),
'passwords': ['Make It Censored', None, '', 'Hello, World!'],
'credit_card_numbers': ('1234', '5678', '9012'),
'cvv2': {'a', None, '', 'b'},
'pin':
frozenset({'c', 'd', ''}),
'foo':
'bar',
'passphrases': {
'not_sensitive': 'not censored',
'bankAccount': 'this should also be censored',
}
}
wrapped = RecursivelyCensoredDictWrapper(original)
expected = {
'a_list': [
'a',
True,
109.8277,
{
'username': '******',
'passphrase': '**********'
},
{
'username': '******',
'passphrase': ''
},
],
'a_set': {
'b',
False,
18273,
},
'a_tuple': (
'c',
True,
42,
{
'cc_number': '**********',
'cvv': '**********',
'expiration': '12-20',
'pin': '**********'
},
),
'passwords': ['**********', None, '', '**********'],
'credit_card_numbers': ('**********', '**********', '**********'),
'cvv2': {'**********', None, '', '**********'},
'pin':
frozenset({'**********', '**********', ''}),
'foo':
'bar',
'passphrases': {
'not_sensitive': 'not censored',
'bankAccount': '**********',
}
}
self.assertEqual(expected, eval(repr(wrapped)))
self.assertEqual(repr(wrapped), str(wrapped))
if six.PY2:
self.assertEqual(six.text_type(repr(wrapped)),
six.text_type(wrapped))
else:
self.assertEqual(six.binary_type(repr(wrapped), 'utf-8'),
six.binary_type(wrapped))
self.assertEqual(
{
'a_list': [
'a',
True,
109.8277,
{
'username': '******',
'passphrase': 'this should be censored'
},
{
'username': '******',
'passphrase': ''
},
],
'a_set': {
'b',
False,
18273,
},
'a_tuple': (
'c',
True,
42,
{
'cc_number': '9876543210987654',
'cvv': '987',
'expiration': '12-20',
'pin': '4096'
},
),
'passwords': ['Make It Censored', None, '', 'Hello, World!'],
'credit_card_numbers': ('1234', '5678', '9012'),
'cvv2': {'a', None, '', 'b'},
'pin':
frozenset({'c', 'd', ''}),
'foo':
'bar',
'passphrases': {
'not_sensitive': 'not censored',
'bankAccount': 'this should also be censored',
}
},
original,
)
def test_non_dict(self):
with self.assertRaises(ValueError):
# noinspection PyTypeChecker
RecursivelyCensoredDictWrapper(['this', 'is', 'a', 'list'])
def send_message(self,
queue_name,
request_id,
meta,
body,
message_expiry_in_seconds=None):
"""
Send a message to the specified queue in Redis.
:param queue_name: The name of the queue to which to send the message
:type queue_name: union(str, unicode)
:param request_id: The message's request ID
:type request_id: int
:param meta: The message meta information, if any (should be an empty dict if no metadata)
:type meta: dict
:param body: The message body (should be a dict)
:type body: dict
:param message_expiry_in_seconds: The optional message expiry, which defaults to the setting with the same name
:type message_expiry_in_seconds: int
:raise: InvalidMessageError, MessageTooLarge, MessageSendError
"""
if request_id is None:
raise InvalidMessageError('No request ID')
if message_expiry_in_seconds:
message_expiry = time.time() + message_expiry_in_seconds
redis_expiry = message_expiry_in_seconds + 10
else:
message_expiry = time.time() + self.message_expiry_in_seconds
redis_expiry = self.message_expiry_in_seconds
meta['__expiry__'] = message_expiry
message = {'request_id': request_id, 'meta': meta, 'body': body}
with self._get_timer('send.serialize'):
serialized_message = self.serializer.dict_to_blob(message)
message_size_in_bytes = len(serialized_message)
if message_size_in_bytes > self.maximum_message_size_in_bytes:
self._get_counter('send.error.message_too_large').increment()
raise MessageTooLarge(message_size_in_bytes)
elif self.log_messages_larger_than_bytes and message_size_in_bytes > self.log_messages_larger_than_bytes:
_oversized_message_logger.warning(
'Oversized message sent for PySOA service {}'.format(
self.service_name),
extra={
'data': {
'message': RecursivelyCensoredDictWrapper(message),
'serialized_length_in_bytes': message_size_in_bytes,
'threshold': self.log_messages_larger_than_bytes,
}
},
)
queue_key = self.QUEUE_NAME_PREFIX + queue_name
# Try at least once, up to queue_full_retries times, then error
for i in range(-1, self.queue_full_retries):
if i >= 0:
time.sleep((2**i + random.random()) /
self.EXPONENTIAL_BACK_OFF_FACTOR)
self._get_counter('send.queue_full_retry').increment()
self._get_counter(
'send.queue_full_retry.retry_{}'.format(i +
1)).increment()
try:
with self._get_timer('send.get_redis_connection'):
connection = self.backend_layer.get_connection(queue_key)
with self._get_timer('send.send_message_to_redis_queue'):
self.backend_layer.send_message_to_queue(
queue_key=queue_key,
message=serialized_message,
expiry=redis_expiry,
capacity=self.queue_capacity,
connection=connection,
)
return
except redis.exceptions.ResponseError as e:
# The Lua script handles capacity checking and sends the "full" error back
if e.args[0] == 'queue full':
continue
self._get_counter('send.error.response').increment()
raise MessageSendError(
'Redis error sending message for service {}'.format(
self.service_name), *e.args)
except CannotGetConnectionError as e:
self._get_counter('send.error.connection').increment()
raise MessageSendError('Cannot get connection: {}'.format(
e.args[0]))
except Exception as e:
self._get_counter('send.error.unknown').increment()
raise MessageSendError(
'Unknown error sending message for service {}'.format(
self.service_name), six.text_type(type(e).__name__),
*e.args)
self._get_counter('send.error.redis_queue_full').increment()
raise MessageSendError(
'Redis queue {queue_name} was full after {retries} retries'.format(
queue_name=queue_name,
retries=self.queue_full_retries,
))